diff options
Diffstat (limited to 'qemu/migration')
-rw-r--r-- | qemu/migration/Makefile.objs | 2 | ||||
-rw-r--r-- | qemu/migration/block.c | 106 | ||||
-rw-r--r-- | qemu/migration/exec.c | 7 | ||||
-rw-r--r-- | qemu/migration/fd.c | 6 | ||||
-rw-r--r-- | qemu/migration/migration.c | 1016 | ||||
-rw-r--r-- | qemu/migration/postcopy-ram.c | 761 | ||||
-rw-r--r-- | qemu/migration/qemu-file-buf.c | 12 | ||||
-rw-r--r-- | qemu/migration/qemu-file-stdio.c | 18 | ||||
-rw-r--r-- | qemu/migration/qemu-file-unix.c | 129 | ||||
-rw-r--r-- | qemu/migration/qemu-file.c | 91 | ||||
-rw-r--r-- | qemu/migration/ram.c | 1330 | ||||
-rw-r--r-- | qemu/migration/rdma.c | 44 | ||||
-rw-r--r-- | qemu/migration/savevm.c | 1226 | ||||
-rw-r--r-- | qemu/migration/tcp.c | 13 | ||||
-rw-r--r-- | qemu/migration/unix.c | 6 | ||||
-rw-r--r-- | qemu/migration/vmstate.c | 28 | ||||
-rw-r--r-- | qemu/migration/xbzrle.c | 3 |
17 files changed, 4046 insertions, 752 deletions
diff --git a/qemu/migration/Makefile.objs b/qemu/migration/Makefile.objs index d929e969a..0cac6d707 100644 --- a/qemu/migration/Makefile.objs +++ b/qemu/migration/Makefile.objs @@ -1,7 +1,7 @@ common-obj-y += migration.o tcp.o common-obj-y += vmstate.o common-obj-y += qemu-file.o qemu-file-buf.o qemu-file-unix.o qemu-file-stdio.o -common-obj-y += xbzrle.o +common-obj-y += xbzrle.o postcopy-ram.o common-obj-$(CONFIG_RDMA) += rdma.o common-obj-$(CONFIG_POSIX) += exec.o unix.o fd.o diff --git a/qemu/migration/block.c b/qemu/migration/block.c index 85496fd80..174331728 100644 --- a/qemu/migration/block.c +++ b/qemu/migration/block.c @@ -13,18 +13,20 @@ * GNU GPL, version 2 or (at your option) any later version. */ +#include "qemu/osdep.h" +#include "qapi/error.h" #include "qemu-common.h" #include "block/block.h" #include "qemu/error-report.h" #include "qemu/main-loop.h" #include "hw/hw.h" +#include "qemu/cutils.h" #include "qemu/queue.h" #include "qemu/timer.h" #include "migration/block.h" #include "migration/migration.h" #include "sysemu/blockdev.h" #include "sysemu/block-backend.h" -#include <assert.h> #define BLOCK_SIZE (1 << 20) #define BDRV_SECTORS_PER_DIRTY_CHUNK (BLOCK_SIZE >> BDRV_SECTOR_BITS) @@ -36,6 +38,8 @@ #define MAX_IS_ALLOCATED_SEARCH 65536 +#define MAX_INFLIGHT_IO 512 + //#define DEBUG_BLK_MIGRATION #ifdef DEBUG_BLK_MIGRATION @@ -52,17 +56,25 @@ typedef struct BlkMigDevState { int shared_base; int64_t total_sectors; QSIMPLEQ_ENTRY(BlkMigDevState) entry; + Error *blocker; /* Only used by migration thread. Does not need a lock. */ int bulk_completed; int64_t cur_sector; int64_t cur_dirty; - /* Protected by block migration lock. */ + /* Data in the aio_bitmap is protected by block migration lock. + * Allocation and free happen during setup and cleanup respectively. + */ unsigned long *aio_bitmap; + + /* Protected by block migration lock. */ int64_t completed_sectors; + + /* During migration this is protected by iothread lock / AioContext. + * Allocation and free happen during setup and cleanup respectively. + */ BdrvDirtyBitmap *dirty_bitmap; - Error *blocker; } BlkMigDevState; typedef struct BlkMigBlock { @@ -98,7 +110,7 @@ typedef struct BlkMigState { int prev_progress; int bulk_completed; - /* Lock must be taken _inside_ the iothread lock. */ + /* Lock must be taken _inside_ the iothread lock and any AioContexts. */ QemuMutex lock; } BlkMigState; @@ -262,11 +274,13 @@ static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds) if (bmds->shared_base) { qemu_mutex_lock_iothread(); + aio_context_acquire(bdrv_get_aio_context(bs)); while (cur_sector < total_sectors && !bdrv_is_allocated(bs, cur_sector, MAX_IS_ALLOCATED_SEARCH, &nr_sectors)) { cur_sector += nr_sectors; } + aio_context_release(bdrv_get_aio_context(bs)); qemu_mutex_unlock_iothread(); } @@ -300,11 +314,21 @@ static int mig_save_device_bulk(QEMUFile *f, BlkMigDevState *bmds) block_mig_state.submitted++; blk_mig_unlock(); + /* We do not know if bs is under the main thread (and thus does + * not acquire the AioContext when doing AIO) or rather under + * dataplane. Thus acquire both the iothread mutex and the + * AioContext. + * + * This is ugly and will disappear when we make bdrv_* thread-safe, + * without the need to acquire the AioContext. + */ qemu_mutex_lock_iothread(); + aio_context_acquire(bdrv_get_aio_context(bmds->bs)); blk->aiocb = bdrv_aio_readv(bs, cur_sector, &blk->qiov, nr_sectors, blk_mig_read_cb, blk); bdrv_reset_dirty_bitmap(bmds->dirty_bitmap, cur_sector, nr_sectors); + aio_context_release(bdrv_get_aio_context(bmds->bs)); qemu_mutex_unlock_iothread(); bmds->cur_sector = cur_sector + nr_sectors; @@ -319,8 +343,10 @@ static int set_dirty_tracking(void) int ret; QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + aio_context_acquire(bdrv_get_aio_context(bmds->bs)); bmds->dirty_bitmap = bdrv_create_dirty_bitmap(bmds->bs, BLOCK_SIZE, NULL, NULL); + aio_context_release(bdrv_get_aio_context(bmds->bs)); if (!bmds->dirty_bitmap) { ret = -errno; goto fail; @@ -331,18 +357,24 @@ static int set_dirty_tracking(void) fail: QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { if (bmds->dirty_bitmap) { + aio_context_acquire(bdrv_get_aio_context(bmds->bs)); bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap); + aio_context_release(bdrv_get_aio_context(bmds->bs)); } } return ret; } +/* Called with iothread lock taken. */ + static void unset_dirty_tracking(void) { BlkMigDevState *bmds; QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + aio_context_acquire(bdrv_get_aio_context(bmds->bs)); bdrv_release_dirty_bitmap(bmds->bs, bmds->dirty_bitmap); + aio_context_release(bdrv_get_aio_context(bmds->bs)); } } @@ -442,7 +474,7 @@ static void blk_mig_reset_dirty_cursor(void) } } -/* Called with iothread lock taken. */ +/* Called with iothread lock and AioContext taken. */ static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds, int is_async) @@ -525,7 +557,9 @@ static int blk_mig_save_dirty_block(QEMUFile *f, int is_async) int ret = 1; QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + aio_context_acquire(bdrv_get_aio_context(bmds->bs)); ret = mig_save_device_dirty(f, bmds, is_async); + aio_context_release(bdrv_get_aio_context(bmds->bs)); if (ret <= 0) { break; } @@ -583,7 +617,9 @@ static int64_t get_remaining_dirty(void) int64_t dirty = 0; QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) { + aio_context_acquire(bdrv_get_aio_context(bmds->bs)); dirty += bdrv_get_dirty_count(bmds->dirty_bitmap); + aio_context_release(bdrv_get_aio_context(bmds->bs)); } return dirty << BDRV_SECTOR_BITS; @@ -591,25 +627,32 @@ static int64_t get_remaining_dirty(void) /* Called with iothread lock taken. */ -static void blk_mig_cleanup(void) +static void block_migration_cleanup(void *opaque) { BlkMigDevState *bmds; BlkMigBlock *blk; + AioContext *ctx; bdrv_drain_all(); unset_dirty_tracking(); - blk_mig_lock(); while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) { QSIMPLEQ_REMOVE_HEAD(&block_mig_state.bmds_list, entry); bdrv_op_unblock_all(bmds->bs, bmds->blocker); error_free(bmds->blocker); + + /* Save ctx, because bmds->bs can disappear during bdrv_unref. */ + ctx = bdrv_get_aio_context(bmds->bs); + aio_context_acquire(ctx); bdrv_unref(bmds->bs); + aio_context_release(ctx); + g_free(bmds->aio_bitmap); g_free(bmds); } + blk_mig_lock(); while ((blk = QSIMPLEQ_FIRST(&block_mig_state.blk_list)) != NULL) { QSIMPLEQ_REMOVE_HEAD(&block_mig_state.blk_list, entry); g_free(blk->buf); @@ -618,11 +661,6 @@ static void blk_mig_cleanup(void) blk_mig_unlock(); } -static void block_migration_cancel(void *opaque) -{ - blk_mig_cleanup(); -} - static int block_save_setup(QEMUFile *f, void *opaque) { int ret; @@ -636,13 +674,12 @@ static int block_save_setup(QEMUFile *f, void *opaque) /* start track dirty blocks */ ret = set_dirty_tracking(); + qemu_mutex_unlock_iothread(); + if (ret) { - qemu_mutex_unlock_iothread(); return ret; } - qemu_mutex_unlock_iothread(); - ret = flush_blks(f); blk_mig_reset_dirty_cursor(); qemu_put_be64(f, BLK_MIG_FLAG_EOS); @@ -670,7 +707,10 @@ static int block_save_iterate(QEMUFile *f, void *opaque) blk_mig_lock(); while ((block_mig_state.submitted + block_mig_state.read_done) * BLOCK_SIZE < - qemu_file_get_rate_limit(f)) { + qemu_file_get_rate_limit(f) && + (block_mig_state.submitted + + block_mig_state.read_done) < + MAX_INFLIGHT_IO) { blk_mig_unlock(); if (block_mig_state.bulk_completed == 0) { /* first finish the bulk phase */ @@ -753,26 +793,30 @@ static int block_save_complete(QEMUFile *f, void *opaque) return 0; } -static uint64_t block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size) +static void block_save_pending(QEMUFile *f, void *opaque, uint64_t max_size, + uint64_t *non_postcopiable_pending, + uint64_t *postcopiable_pending) { /* Estimate pending number of bytes to send */ uint64_t pending; qemu_mutex_lock_iothread(); + pending = get_remaining_dirty(); + qemu_mutex_unlock_iothread(); + blk_mig_lock(); - pending = get_remaining_dirty() + - block_mig_state.submitted * BLOCK_SIZE + - block_mig_state.read_done * BLOCK_SIZE; + pending += block_mig_state.submitted * BLOCK_SIZE + + block_mig_state.read_done * BLOCK_SIZE; + blk_mig_unlock(); /* Report at least one block pending during bulk phase */ if (pending <= max_size && !block_mig_state.bulk_completed) { pending = max_size + BLOCK_SIZE; } - blk_mig_unlock(); - qemu_mutex_unlock_iothread(); DPRINTF("Enter save live pending %" PRIu64 "\n", pending); - return pending; + /* We don't do postcopy */ + *non_postcopiable_pending += pending; } static int block_load(QEMUFile *f, void *opaque, int version_id) @@ -783,6 +827,7 @@ static int block_load(QEMUFile *f, void *opaque, int version_id) int64_t addr; BlockDriverState *bs, *bs_prev = NULL; BlockBackend *blk; + Error *local_err = NULL; uint8_t *buf; int64_t total_sectors = 0; int nr_sectors; @@ -807,6 +852,11 @@ static int block_load(QEMUFile *f, void *opaque, int version_id) return -EINVAL; } bs = blk_bs(blk); + if (!bs) { + fprintf(stderr, "Block device %s has no medium\n", + device_name); + return -EINVAL; + } if (bs != bs_prev) { bs_prev = bs; @@ -816,6 +866,12 @@ static int block_load(QEMUFile *f, void *opaque, int version_id) device_name); return -EINVAL; } + + bdrv_invalidate_cache(bs, &local_err); + if (local_err) { + error_report_err(local_err); + return -EINVAL; + } } if (total_sectors - addr < BDRV_SECTORS_PER_DIRTY_CHUNK) { @@ -876,10 +932,10 @@ static SaveVMHandlers savevm_block_handlers = { .set_params = block_set_params, .save_live_setup = block_save_setup, .save_live_iterate = block_save_iterate, - .save_live_complete = block_save_complete, + .save_live_complete_precopy = block_save_complete, .save_live_pending = block_save_pending, .load_state = block_load, - .cancel = block_migration_cancel, + .cleanup = block_migration_cleanup, .is_active = block_is_active, }; diff --git a/qemu/migration/exec.c b/qemu/migration/exec.c index 8406d2bbd..559420969 100644 --- a/qemu/migration/exec.c +++ b/qemu/migration/exec.c @@ -15,13 +15,14 @@ * GNU GPL, version 2 or (at your option) any later version. */ +#include "qemu/osdep.h" +#include "qapi/error.h" #include "qemu-common.h" #include "qemu/sockets.h" #include "qemu/main-loop.h" #include "migration/migration.h" #include "migration/qemu-file.h" #include "block/block.h" -#include <sys/types.h> #include <sys/wait.h> //#define DEBUG_MIGRATION_EXEC @@ -36,8 +37,8 @@ void exec_start_outgoing_migration(MigrationState *s, const char *command, Error **errp) { - s->file = qemu_popen_cmd(command, "w"); - if (s->file == NULL) { + s->to_dst_file = qemu_popen_cmd(command, "w"); + if (s->to_dst_file == NULL) { error_setg_errno(errp, errno, "failed to popen the migration target"); return; } diff --git a/qemu/migration/fd.c b/qemu/migration/fd.c index 3e4bed0e0..3d788bb29 100644 --- a/qemu/migration/fd.c +++ b/qemu/migration/fd.c @@ -13,6 +13,8 @@ * GNU GPL, version 2 or (at your option) any later version. */ +#include "qemu/osdep.h" +#include "qapi/error.h" #include "qemu-common.h" #include "qemu/main-loop.h" #include "qemu/sockets.h" @@ -50,9 +52,9 @@ void fd_start_outgoing_migration(MigrationState *s, const char *fdname, Error ** } if (fd_is_socket(fd)) { - s->file = qemu_fopen_socket(fd, "wb"); + s->to_dst_file = qemu_fopen_socket(fd, "wb"); } else { - s->file = qemu_fdopen(fd, "wb"); + s->to_dst_file = qemu_fdopen(fd, "wb"); } migrate_fd_connect(s); diff --git a/qemu/migration/migration.c b/qemu/migration/migration.c index 4ddb9adce..991313a86 100644 --- a/qemu/migration/migration.c +++ b/qemu/migration/migration.c @@ -13,7 +13,8 @@ * GNU GPL, version 2 or (at your option) any later version. */ -#include "qemu-common.h" +#include "qemu/osdep.h" +#include "qemu/cutils.h" #include "qemu/error-report.h" #include "qemu/main-loop.h" #include "migration/migration.h" @@ -21,16 +22,20 @@ #include "sysemu/sysemu.h" #include "block/block.h" #include "qapi/qmp/qerror.h" +#include "qapi/util.h" #include "qemu/sockets.h" #include "qemu/rcu.h" #include "migration/block.h" +#include "migration/postcopy-ram.h" #include "qemu/thread.h" #include "qmp-commands.h" #include "trace.h" -#include "qapi/util.h" #include "qapi-event.h" +#include "qom/cpu.h" +#include "exec/memory.h" +#include "exec/address-spaces.h" -#define MAX_THROTTLE (32 << 20) /* Migration speed throttling */ +#define MAX_THROTTLE (32 << 20) /* Migration transfer speed throttling */ /* Amount of time to allocate to each "chunk" of bandwidth-throttled * data. */ @@ -44,6 +49,9 @@ #define DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT 2 /*0: means nocompress, 1: best speed, ... 9: best compress ratio */ #define DEFAULT_MIGRATE_COMPRESS_LEVEL 1 +/* Define default autoconverge cpu throttle migration parameters */ +#define DEFAULT_MIGRATE_X_CPU_THROTTLE_INITIAL 20 +#define DEFAULT_MIGRATE_X_CPU_THROTTLE_INCREMENT 10 /* Migration XBZRLE default cache size */ #define DEFAULT_MIGRATE_CACHE_SIZE (64 * 1024 * 1024) @@ -53,6 +61,13 @@ static NotifierList migration_state_notifiers = static bool deferred_incoming; +/* + * Current state of incoming postcopy; note this is not part of + * MigrationIncomingState since it's state is used during cleanup + * at the end as MIS is being freed. + */ +static PostcopyState incoming_postcopy_state; + /* When we add fault tolerance, we could have several migrations at once. For now we don't need to add dynamic creation of migration */ @@ -60,6 +75,7 @@ static bool deferred_incoming; /* For outgoing */ MigrationState *migrate_get_current(void) { + static bool once; static MigrationState current_migration = { .state = MIGRATION_STATUS_NONE, .bandwidth_limit = MAX_THROTTLE, @@ -71,8 +87,16 @@ MigrationState *migrate_get_current(void) DEFAULT_MIGRATE_COMPRESS_THREAD_COUNT, .parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = DEFAULT_MIGRATE_DECOMPRESS_THREAD_COUNT, + .parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL] = + DEFAULT_MIGRATE_X_CPU_THROTTLE_INITIAL, + .parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT] = + DEFAULT_MIGRATE_X_CPU_THROTTLE_INCREMENT, }; + if (!once) { + qemu_mutex_init(¤t_migration.src_page_req_mutex); + once = true; + } return ¤t_migration; } @@ -86,15 +110,19 @@ MigrationIncomingState *migration_incoming_get_current(void) MigrationIncomingState *migration_incoming_state_new(QEMUFile* f) { - mis_current = g_malloc0(sizeof(MigrationIncomingState)); - mis_current->file = f; + mis_current = g_new0(MigrationIncomingState, 1); + mis_current->from_src_file = f; + mis_current->state = MIGRATION_STATUS_NONE; QLIST_INIT(&mis_current->loadvm_handlers); + qemu_mutex_init(&mis_current->rp_mutex); + qemu_event_init(&mis_current->main_thread_load_event, false); return mis_current; } void migration_incoming_state_destroy(void) { + qemu_event_destroy(&mis_current->main_thread_load_event); loadvm_free_handlers(mis_current); g_free(mis_current); mis_current = NULL; @@ -176,7 +204,7 @@ static int global_state_post_load(void *opaque, int version_id) s->received = true; trace_migrate_global_state_post_load(runstate); - r = qapi_enum_parse(RunState_lookup, runstate, RUN_STATE_MAX, + r = qapi_enum_parse(RunState_lookup, runstate, RUN_STATE__MAX, -1, &local_err); if (r == -1) { @@ -240,6 +268,35 @@ static void deferred_incoming_migration(Error **errp) deferred_incoming = true; } +/* Request a range of pages from the source VM at the given + * start address. + * rbname: Name of the RAMBlock to request the page in, if NULL it's the same + * as the last request (a name must have been given previously) + * Start: Address offset within the RB + * Len: Length in bytes required - must be a multiple of pagesize + */ +void migrate_send_rp_req_pages(MigrationIncomingState *mis, const char *rbname, + ram_addr_t start, size_t len) +{ + uint8_t bufc[12 + 1 + 255]; /* start (8), len (4), rbname upto 256 */ + size_t msglen = 12; /* start + len */ + + *(uint64_t *)bufc = cpu_to_be64((uint64_t)start); + *(uint32_t *)(bufc + 8) = cpu_to_be32((uint32_t)len); + + if (rbname) { + int rbname_len = strlen(rbname); + assert(rbname_len < 256); + + bufc[msglen++] = rbname_len; + memcpy(bufc + msglen, rbname, rbname_len); + msglen += rbname_len; + migrate_send_rp_message(mis, MIG_RP_MSG_REQ_PAGES_ID, msglen, bufc); + } else { + migrate_send_rp_message(mis, MIG_RP_MSG_REQ_PAGES, msglen, bufc); + } +} + void qemu_start_incoming_migration(const char *uri, Error **errp) { const char *p; @@ -266,37 +323,27 @@ void qemu_start_incoming_migration(const char *uri, Error **errp) } } -static void process_incoming_migration_co(void *opaque) +static void process_incoming_migration_bh(void *opaque) { - QEMUFile *f = opaque; Error *local_err = NULL; - int ret; - - migration_incoming_state_new(f); - migrate_generate_event(MIGRATION_STATUS_ACTIVE); - ret = qemu_loadvm_state(f); - - qemu_fclose(f); - free_xbzrle_decoded_buf(); - migration_incoming_state_destroy(); - - if (ret < 0) { - migrate_generate_event(MIGRATION_STATUS_FAILED); - error_report("load of migration failed: %s", strerror(-ret)); - migrate_decompress_threads_join(); - exit(EXIT_FAILURE); - } - migrate_generate_event(MIGRATION_STATUS_COMPLETED); - qemu_announce_self(); + MigrationIncomingState *mis = opaque; /* Make sure all file formats flush their mutable metadata */ bdrv_invalidate_cache_all(&local_err); if (local_err) { + migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_FAILED); error_report_err(local_err); migrate_decompress_threads_join(); exit(EXIT_FAILURE); } + /* + * This must happen after all error conditions are dealt with and + * we're sure the VM is going to be running on this host. + */ + qemu_announce_self(); + /* If global state section was not received or we are in running state, we need to obey autostart. Any other state is set with runstate_set. */ @@ -312,6 +359,64 @@ static void process_incoming_migration_co(void *opaque) runstate_set(global_state_get_runstate()); } migrate_decompress_threads_join(); + /* + * This must happen after any state changes since as soon as an external + * observer sees this event they might start to prod at the VM assuming + * it's ready to use. + */ + migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_COMPLETED); + qemu_bh_delete(mis->bh); + migration_incoming_state_destroy(); +} + +static void process_incoming_migration_co(void *opaque) +{ + QEMUFile *f = opaque; + MigrationIncomingState *mis; + PostcopyState ps; + int ret; + + mis = migration_incoming_state_new(f); + postcopy_state_set(POSTCOPY_INCOMING_NONE); + migrate_set_state(&mis->state, MIGRATION_STATUS_NONE, + MIGRATION_STATUS_ACTIVE); + ret = qemu_loadvm_state(f); + + ps = postcopy_state_get(); + trace_process_incoming_migration_co_end(ret, ps); + if (ps != POSTCOPY_INCOMING_NONE) { + if (ps == POSTCOPY_INCOMING_ADVISE) { + /* + * Where a migration had postcopy enabled (and thus went to advise) + * but managed to complete within the precopy period, we can use + * the normal exit. + */ + postcopy_ram_incoming_cleanup(mis); + } else if (ret >= 0) { + /* + * Postcopy was started, cleanup should happen at the end of the + * postcopy thread. + */ + trace_process_incoming_migration_co_postcopy_end_main(); + return; + } + /* Else if something went wrong then just fall out of the normal exit */ + } + + qemu_fclose(f); + free_xbzrle_decoded_buf(); + + if (ret < 0) { + migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_FAILED); + error_report("load of migration failed: %s", strerror(-ret)); + migrate_decompress_threads_join(); + exit(EXIT_FAILURE); + } + + mis->bh = qemu_bh_new(process_incoming_migration_bh, mis); + qemu_bh_schedule(mis->bh); } void process_incoming_migration(QEMUFile *f) @@ -325,6 +430,50 @@ void process_incoming_migration(QEMUFile *f) qemu_coroutine_enter(co, f); } +/* + * Send a message on the return channel back to the source + * of the migration. + */ +void migrate_send_rp_message(MigrationIncomingState *mis, + enum mig_rp_message_type message_type, + uint16_t len, void *data) +{ + trace_migrate_send_rp_message((int)message_type, len); + qemu_mutex_lock(&mis->rp_mutex); + qemu_put_be16(mis->to_src_file, (unsigned int)message_type); + qemu_put_be16(mis->to_src_file, len); + qemu_put_buffer(mis->to_src_file, data, len); + qemu_fflush(mis->to_src_file); + qemu_mutex_unlock(&mis->rp_mutex); +} + +/* + * Send a 'SHUT' message on the return channel with the given value + * to indicate that we've finished with the RP. Non-0 value indicates + * error. + */ +void migrate_send_rp_shut(MigrationIncomingState *mis, + uint32_t value) +{ + uint32_t buf; + + buf = cpu_to_be32(value); + migrate_send_rp_message(mis, MIG_RP_MSG_SHUT, sizeof(buf), &buf); +} + +/* + * Send a 'PONG' message on the return channel with the given value + * (normally in response to a 'PING') + */ +void migrate_send_rp_pong(MigrationIncomingState *mis, + uint32_t value) +{ + uint32_t buf; + + buf = cpu_to_be32(value); + migrate_send_rp_message(mis, MIG_RP_MSG_PONG, sizeof(buf), &buf); +} + /* amount of nanoseconds we are willing to wait for migration to be down. * the choice of nanoseconds is because it is the maximum resolution that * get_clock() can achieve. It is an internal measure. All user-visible @@ -344,7 +493,7 @@ MigrationCapabilityStatusList *qmp_query_migrate_capabilities(Error **errp) int i; caps = NULL; /* silence compiler warning */ - for (i = 0; i < MIGRATION_CAPABILITY_MAX; i++) { + for (i = 0; i < MIGRATION_CAPABILITY__MAX; i++) { if (head == NULL) { head = g_malloc0(sizeof(*caps)); caps = head; @@ -372,10 +521,32 @@ MigrationParameters *qmp_query_migrate_parameters(Error **errp) s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; params->decompress_threads = s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; + params->x_cpu_throttle_initial = + s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL]; + params->x_cpu_throttle_increment = + s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT]; return params; } +/* + * Return true if we're already in the middle of a migration + * (i.e. any of the active or setup states) + */ +static bool migration_is_setup_or_active(int state) +{ + switch (state) { + case MIGRATION_STATUS_ACTIVE: + case MIGRATION_STATUS_POSTCOPY_ACTIVE: + case MIGRATION_STATUS_SETUP: + return true; + + default: + return false; + + } +} + static void get_xbzrle_cache_stats(MigrationInfo *info) { if (migrate_use_xbzrle()) { @@ -435,6 +606,45 @@ MigrationInfo *qmp_query_migrate(Error **errp) info->disk->total = blk_mig_bytes_total(); } + if (cpu_throttle_active()) { + info->has_x_cpu_throttle_percentage = true; + info->x_cpu_throttle_percentage = cpu_throttle_get_percentage(); + } + + get_xbzrle_cache_stats(info); + break; + case MIGRATION_STATUS_POSTCOPY_ACTIVE: + /* Mostly the same as active; TODO add some postcopy stats */ + info->has_status = true; + info->has_total_time = true; + info->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) + - s->total_time; + info->has_expected_downtime = true; + info->expected_downtime = s->expected_downtime; + info->has_setup_time = true; + info->setup_time = s->setup_time; + + info->has_ram = true; + info->ram = g_malloc0(sizeof(*info->ram)); + info->ram->transferred = ram_bytes_transferred(); + info->ram->remaining = ram_bytes_remaining(); + info->ram->total = ram_bytes_total(); + info->ram->duplicate = dup_mig_pages_transferred(); + info->ram->skipped = skipped_mig_pages_transferred(); + info->ram->normal = norm_mig_pages_transferred(); + info->ram->normal_bytes = norm_mig_bytes_transferred(); + info->ram->dirty_pages_rate = s->dirty_pages_rate; + info->ram->mbps = s->mbps; + info->ram->dirty_sync_count = s->dirty_sync_count; + + if (blk_mig_active()) { + info->has_disk = true; + info->disk = g_malloc0(sizeof(*info->disk)); + info->disk->transferred = blk_mig_bytes_transferred(); + info->disk->remaining = blk_mig_bytes_remaining(); + info->disk->total = blk_mig_bytes_total(); + } + get_xbzrle_cache_stats(info); break; case MIGRATION_STATUS_COMPLETED: @@ -478,8 +688,7 @@ void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params, MigrationState *s = migrate_get_current(); MigrationCapabilityStatusList *cap; - if (s->state == MIGRATION_STATUS_ACTIVE || - s->state == MIGRATION_STATUS_SETUP) { + if (migration_is_setup_or_active(s->state)) { error_setg(errp, QERR_MIGRATION_ACTIVE); return; } @@ -487,6 +696,20 @@ void qmp_migrate_set_capabilities(MigrationCapabilityStatusList *params, for (cap = params; cap; cap = cap->next) { s->enabled_capabilities[cap->value->capability] = cap->value->state; } + + if (migrate_postcopy_ram()) { + if (migrate_use_compression()) { + /* The decompression threads asynchronously write into RAM + * rather than use the atomic copies needed to avoid + * userfaulting. It should be possible to fix the decompression + * threads for compatibility in future. + */ + error_report("Postcopy is not currently compatible with " + "compression"); + s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM] = + false; + } + } } void qmp_migrate_set_parameters(bool has_compress_level, @@ -494,7 +717,11 @@ void qmp_migrate_set_parameters(bool has_compress_level, bool has_compress_threads, int64_t compress_threads, bool has_decompress_threads, - int64_t decompress_threads, Error **errp) + int64_t decompress_threads, + bool has_x_cpu_throttle_initial, + int64_t x_cpu_throttle_initial, + bool has_x_cpu_throttle_increment, + int64_t x_cpu_throttle_increment, Error **errp) { MigrationState *s = migrate_get_current(); @@ -517,6 +744,18 @@ void qmp_migrate_set_parameters(bool has_compress_level, "is invalid, it should be in the range of 1 to 255"); return; } + if (has_x_cpu_throttle_initial && + (x_cpu_throttle_initial < 1 || x_cpu_throttle_initial > 99)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "x_cpu_throttle_initial", + "an integer in the range of 1 to 99"); + } + if (has_x_cpu_throttle_increment && + (x_cpu_throttle_increment < 1 || x_cpu_throttle_increment > 99)) { + error_setg(errp, QERR_INVALID_PARAMETER_VALUE, + "x_cpu_throttle_increment", + "an integer in the range of 1 to 99"); + } if (has_compress_level) { s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level; @@ -528,13 +767,44 @@ void qmp_migrate_set_parameters(bool has_compress_level, s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = decompress_threads; } + if (has_x_cpu_throttle_initial) { + s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL] = + x_cpu_throttle_initial; + } + + if (has_x_cpu_throttle_increment) { + s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT] = + x_cpu_throttle_increment; + } +} + +void qmp_migrate_start_postcopy(Error **errp) +{ + MigrationState *s = migrate_get_current(); + + if (!migrate_postcopy_ram()) { + error_setg(errp, "Enable postcopy with migrate_set_capability before" + " the start of migration"); + return; + } + + if (s->state == MIGRATION_STATUS_NONE) { + error_setg(errp, "Postcopy must be started after migration has been" + " started"); + return; + } + /* + * we don't error if migration has finished since that would be racy + * with issuing this command. + */ + atomic_set(&s->start_postcopy, true); } /* shared migration helpers */ -static void migrate_set_state(MigrationState *s, int old_state, int new_state) +void migrate_set_state(int *state, int old_state, int new_state) { - if (atomic_cmpxchg(&s->state, old_state, new_state) == old_state) { + if (atomic_cmpxchg(state, old_state, new_state) == old_state) { trace_migrate_set_state(new_state); migrate_generate_event(new_state); } @@ -547,21 +817,27 @@ static void migrate_fd_cleanup(void *opaque) qemu_bh_delete(s->cleanup_bh); s->cleanup_bh = NULL; - if (s->file) { + flush_page_queue(s); + + if (s->to_dst_file) { trace_migrate_fd_cleanup(); qemu_mutex_unlock_iothread(); - qemu_thread_join(&s->thread); + if (s->migration_thread_running) { + qemu_thread_join(&s->thread); + s->migration_thread_running = false; + } qemu_mutex_lock_iothread(); migrate_compress_threads_join(); - qemu_fclose(s->file); - s->file = NULL; + qemu_fclose(s->to_dst_file); + s->to_dst_file = NULL; } - assert(s->state != MIGRATION_STATUS_ACTIVE); + assert((s->state != MIGRATION_STATUS_ACTIVE) && + (s->state != MIGRATION_STATUS_POSTCOPY_ACTIVE)); if (s->state == MIGRATION_STATUS_CANCELLING) { - migrate_set_state(s, MIGRATION_STATUS_CANCELLING, + migrate_set_state(&s->state, MIGRATION_STATUS_CANCELLING, MIGRATION_STATUS_CANCELLED); } @@ -571,24 +847,29 @@ static void migrate_fd_cleanup(void *opaque) void migrate_fd_error(MigrationState *s) { trace_migrate_fd_error(); - assert(s->file == NULL); - migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_FAILED); + assert(s->to_dst_file == NULL); + migrate_set_state(&s->state, MIGRATION_STATUS_SETUP, + MIGRATION_STATUS_FAILED); notifier_list_notify(&migration_state_notifiers, s); } static void migrate_fd_cancel(MigrationState *s) { int old_state ; - QEMUFile *f = migrate_get_current()->file; + QEMUFile *f = migrate_get_current()->to_dst_file; trace_migrate_fd_cancel(); + if (s->rp_state.from_dst_file) { + /* shutdown the rp socket, so causing the rp thread to shutdown */ + qemu_file_shutdown(s->rp_state.from_dst_file); + } + do { old_state = s->state; - if (old_state != MIGRATION_STATUS_SETUP && - old_state != MIGRATION_STATUS_ACTIVE) { + if (!migration_is_setup_or_active(old_state)) { break; } - migrate_set_state(s, old_state, MIGRATION_STATUS_CANCELLING); + migrate_set_state(&s->state, old_state, MIGRATION_STATUS_CANCELLING); } while (s->state != MIGRATION_STATUS_CANCELLING); /* @@ -629,34 +910,48 @@ bool migration_has_failed(MigrationState *s) s->state == MIGRATION_STATUS_FAILED); } -static MigrationState *migrate_init(const MigrationParams *params) +bool migration_in_postcopy(MigrationState *s) { - MigrationState *s = migrate_get_current(); - int64_t bandwidth_limit = s->bandwidth_limit; - bool enabled_capabilities[MIGRATION_CAPABILITY_MAX]; - int64_t xbzrle_cache_size = s->xbzrle_cache_size; - int compress_level = s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL]; - int compress_thread_count = - s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS]; - int decompress_thread_count = - s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS]; + return (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE); +} - memcpy(enabled_capabilities, s->enabled_capabilities, - sizeof(enabled_capabilities)); +bool migration_in_postcopy_after_devices(MigrationState *s) +{ + return migration_in_postcopy(s) && s->postcopy_after_devices; +} - memset(s, 0, sizeof(*s)); +MigrationState *migrate_init(const MigrationParams *params) +{ + MigrationState *s = migrate_get_current(); + + /* + * Reinitialise all migration state, except + * parameters/capabilities that the user set, and + * locks. + */ + s->bytes_xfer = 0; + s->xfer_limit = 0; + s->cleanup_bh = 0; + s->to_dst_file = NULL; + s->state = MIGRATION_STATUS_NONE; s->params = *params; - memcpy(s->enabled_capabilities, enabled_capabilities, - sizeof(enabled_capabilities)); - s->xbzrle_cache_size = xbzrle_cache_size; - - s->parameters[MIGRATION_PARAMETER_COMPRESS_LEVEL] = compress_level; - s->parameters[MIGRATION_PARAMETER_COMPRESS_THREADS] = - compress_thread_count; - s->parameters[MIGRATION_PARAMETER_DECOMPRESS_THREADS] = - decompress_thread_count; - s->bandwidth_limit = bandwidth_limit; - migrate_set_state(s, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP); + s->rp_state.from_dst_file = NULL; + s->rp_state.error = false; + s->mbps = 0.0; + s->downtime = 0; + s->expected_downtime = 0; + s->dirty_pages_rate = 0; + s->dirty_bytes_rate = 0; + s->setup_time = 0; + s->dirty_sync_count = 0; + s->start_postcopy = false; + s->postcopy_after_devices = false; + s->migration_thread_running = false; + s->last_req_rb = NULL; + + migrate_set_state(&s->state, MIGRATION_STATUS_NONE, MIGRATION_STATUS_SETUP); + + QSIMPLEQ_INIT(&s->src_page_requests); s->total_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); return s; @@ -709,8 +1004,7 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk, params.blk = has_blk && blk; params.shared = has_inc && inc; - if (s->state == MIGRATION_STATUS_ACTIVE || - s->state == MIGRATION_STATUS_SETUP || + if (migration_is_setup_or_active(s->state) || s->state == MIGRATION_STATUS_CANCELLING) { error_setg(errp, QERR_MIGRATION_ACTIVE); return; @@ -729,12 +1023,6 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk, return; } - /* We are starting a new migration, so we want to start in a clean - state. This change is only needed if previous migration - failed/was cancelled. We don't use migrate_set_state() because - we are setting the initial state, not changing it. */ - s->state = MIGRATION_STATUS_NONE; - s = migrate_init(¶ms); if (strstart(uri, "tcp:", &p)) { @@ -754,7 +1042,8 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk, } else { error_setg(errp, QERR_INVALID_PARAMETER_VALUE, "uri", "a valid migration protocol"); - migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_FAILED); + migrate_set_state(&s->state, MIGRATION_STATUS_SETUP, + MIGRATION_STATUS_FAILED); return; } @@ -817,8 +1106,9 @@ void qmp_migrate_set_speed(int64_t value, Error **errp) s = migrate_get_current(); s->bandwidth_limit = value; - if (s->file) { - qemu_file_set_rate_limit(s->file, s->bandwidth_limit / XFER_LIMIT_RATIO); + if (s->to_dst_file) { + qemu_file_set_rate_limit(s->to_dst_file, + s->bandwidth_limit / XFER_LIMIT_RATIO); } } @@ -829,6 +1119,15 @@ void qmp_migrate_set_downtime(double value, Error **errp) max_downtime = (uint64_t)value; } +bool migrate_postcopy_ram(void) +{ + MigrationState *s; + + s = migrate_get_current(); + + return s->enabled_capabilities[MIGRATION_CAPABILITY_POSTCOPY_RAM]; +} + bool migrate_auto_converge(void) { MigrationState *s; @@ -911,10 +1210,419 @@ int64_t migrate_xbzrle_cache_size(void) } /* migration thread support */ +/* + * Something bad happened to the RP stream, mark an error + * The caller shall print or trace something to indicate why + */ +static void mark_source_rp_bad(MigrationState *s) +{ + s->rp_state.error = true; +} + +static struct rp_cmd_args { + ssize_t len; /* -1 = variable */ + const char *name; +} rp_cmd_args[] = { + [MIG_RP_MSG_INVALID] = { .len = -1, .name = "INVALID" }, + [MIG_RP_MSG_SHUT] = { .len = 4, .name = "SHUT" }, + [MIG_RP_MSG_PONG] = { .len = 4, .name = "PONG" }, + [MIG_RP_MSG_REQ_PAGES] = { .len = 12, .name = "REQ_PAGES" }, + [MIG_RP_MSG_REQ_PAGES_ID] = { .len = -1, .name = "REQ_PAGES_ID" }, + [MIG_RP_MSG_MAX] = { .len = -1, .name = "MAX" }, +}; + +/* + * Process a request for pages received on the return path, + * We're allowed to send more than requested (e.g. to round to our page size) + * and we don't need to send pages that have already been sent. + */ +static void migrate_handle_rp_req_pages(MigrationState *ms, const char* rbname, + ram_addr_t start, size_t len) +{ + long our_host_ps = getpagesize(); + trace_migrate_handle_rp_req_pages(rbname, start, len); + + /* + * Since we currently insist on matching page sizes, just sanity check + * we're being asked for whole host pages. + */ + if (start & (our_host_ps-1) || + (len & (our_host_ps-1))) { + error_report("%s: Misaligned page request, start: " RAM_ADDR_FMT + " len: %zd", __func__, start, len); + mark_source_rp_bad(ms); + return; + } + + if (ram_save_queue_pages(ms, rbname, start, len)) { + mark_source_rp_bad(ms); + } +} + +/* + * Handles messages sent on the return path towards the source VM + * + */ +static void *source_return_path_thread(void *opaque) +{ + MigrationState *ms = opaque; + QEMUFile *rp = ms->rp_state.from_dst_file; + uint16_t header_len, header_type; + uint8_t buf[512]; + uint32_t tmp32, sibling_error; + ram_addr_t start = 0; /* =0 to silence warning */ + size_t len = 0, expected_len; + int res; + + trace_source_return_path_thread_entry(); + while (!ms->rp_state.error && !qemu_file_get_error(rp) && + migration_is_setup_or_active(ms->state)) { + trace_source_return_path_thread_loop_top(); + header_type = qemu_get_be16(rp); + header_len = qemu_get_be16(rp); + + if (header_type >= MIG_RP_MSG_MAX || + header_type == MIG_RP_MSG_INVALID) { + error_report("RP: Received invalid message 0x%04x length 0x%04x", + header_type, header_len); + mark_source_rp_bad(ms); + goto out; + } + + if ((rp_cmd_args[header_type].len != -1 && + header_len != rp_cmd_args[header_type].len) || + header_len > sizeof(buf)) { + error_report("RP: Received '%s' message (0x%04x) with" + "incorrect length %d expecting %zu", + rp_cmd_args[header_type].name, header_type, header_len, + (size_t)rp_cmd_args[header_type].len); + mark_source_rp_bad(ms); + goto out; + } + + /* We know we've got a valid header by this point */ + res = qemu_get_buffer(rp, buf, header_len); + if (res != header_len) { + error_report("RP: Failed reading data for message 0x%04x" + " read %d expected %d", + header_type, res, header_len); + mark_source_rp_bad(ms); + goto out; + } + + /* OK, we have the message and the data */ + switch (header_type) { + case MIG_RP_MSG_SHUT: + sibling_error = be32_to_cpup((uint32_t *)buf); + trace_source_return_path_thread_shut(sibling_error); + if (sibling_error) { + error_report("RP: Sibling indicated error %d", sibling_error); + mark_source_rp_bad(ms); + } + /* + * We'll let the main thread deal with closing the RP + * we could do a shutdown(2) on it, but we're the only user + * anyway, so there's nothing gained. + */ + goto out; + + case MIG_RP_MSG_PONG: + tmp32 = be32_to_cpup((uint32_t *)buf); + trace_source_return_path_thread_pong(tmp32); + break; + + case MIG_RP_MSG_REQ_PAGES: + start = be64_to_cpup((uint64_t *)buf); + len = be32_to_cpup((uint32_t *)(buf + 8)); + migrate_handle_rp_req_pages(ms, NULL, start, len); + break; + + case MIG_RP_MSG_REQ_PAGES_ID: + expected_len = 12 + 1; /* header + termination */ + + if (header_len >= expected_len) { + start = be64_to_cpup((uint64_t *)buf); + len = be32_to_cpup((uint32_t *)(buf + 8)); + /* Now we expect an idstr */ + tmp32 = buf[12]; /* Length of the following idstr */ + buf[13 + tmp32] = '\0'; + expected_len += tmp32; + } + if (header_len != expected_len) { + error_report("RP: Req_Page_id with length %d expecting %zd", + header_len, expected_len); + mark_source_rp_bad(ms); + goto out; + } + migrate_handle_rp_req_pages(ms, (char *)&buf[13], start, len); + break; + + default: + break; + } + } + if (qemu_file_get_error(rp)) { + trace_source_return_path_thread_bad_end(); + mark_source_rp_bad(ms); + } + + trace_source_return_path_thread_end(); +out: + ms->rp_state.from_dst_file = NULL; + qemu_fclose(rp); + return NULL; +} + +static int open_return_path_on_source(MigrationState *ms) +{ + + ms->rp_state.from_dst_file = qemu_file_get_return_path(ms->to_dst_file); + if (!ms->rp_state.from_dst_file) { + return -1; + } + + trace_open_return_path_on_source(); + qemu_thread_create(&ms->rp_state.rp_thread, "return path", + source_return_path_thread, ms, QEMU_THREAD_JOINABLE); + + trace_open_return_path_on_source_continue(); + + return 0; +} + +/* Returns 0 if the RP was ok, otherwise there was an error on the RP */ +static int await_return_path_close_on_source(MigrationState *ms) +{ + /* + * If this is a normal exit then the destination will send a SHUT and the + * rp_thread will exit, however if there's an error we need to cause + * it to exit. + */ + if (qemu_file_get_error(ms->to_dst_file) && ms->rp_state.from_dst_file) { + /* + * shutdown(2), if we have it, will cause it to unblock if it's stuck + * waiting for the destination. + */ + qemu_file_shutdown(ms->rp_state.from_dst_file); + mark_source_rp_bad(ms); + } + trace_await_return_path_close_on_source_joining(); + qemu_thread_join(&ms->rp_state.rp_thread); + trace_await_return_path_close_on_source_close(); + return ms->rp_state.error; +} + +/* + * Switch from normal iteration to postcopy + * Returns non-0 on error + */ +static int postcopy_start(MigrationState *ms, bool *old_vm_running) +{ + int ret; + const QEMUSizedBuffer *qsb; + int64_t time_at_stop = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + migrate_set_state(&ms->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_POSTCOPY_ACTIVE); + + trace_postcopy_start(); + qemu_mutex_lock_iothread(); + trace_postcopy_start_set_run(); + + qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER); + *old_vm_running = runstate_is_running(); + global_state_store(); + ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE); + if (ret < 0) { + goto fail; + } + + ret = bdrv_inactivate_all(); + if (ret < 0) { + goto fail; + } + + /* + * Cause any non-postcopiable, but iterative devices to + * send out their final data. + */ + qemu_savevm_state_complete_precopy(ms->to_dst_file, true); + + /* + * in Finish migrate and with the io-lock held everything should + * be quiet, but we've potentially still got dirty pages and we + * need to tell the destination to throw any pages it's already received + * that are dirty + */ + if (ram_postcopy_send_discard_bitmap(ms)) { + error_report("postcopy send discard bitmap failed"); + goto fail; + } + + /* + * send rest of state - note things that are doing postcopy + * will notice we're in POSTCOPY_ACTIVE and not actually + * wrap their state up here + */ + qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX); + /* Ping just for debugging, helps line traces up */ + qemu_savevm_send_ping(ms->to_dst_file, 2); + + /* + * While loading the device state we may trigger page transfer + * requests and the fd must be free to process those, and thus + * the destination must read the whole device state off the fd before + * it starts processing it. Unfortunately the ad-hoc migration format + * doesn't allow the destination to know the size to read without fully + * parsing it through each devices load-state code (especially the open + * coded devices that use get/put). + * So we wrap the device state up in a package with a length at the start; + * to do this we use a qemu_buf to hold the whole of the device state. + */ + QEMUFile *fb = qemu_bufopen("w", NULL); + if (!fb) { + error_report("Failed to create buffered file"); + goto fail; + } + + /* + * Make sure the receiver can get incoming pages before we send the rest + * of the state + */ + qemu_savevm_send_postcopy_listen(fb); + + qemu_savevm_state_complete_precopy(fb, false); + qemu_savevm_send_ping(fb, 3); + + qemu_savevm_send_postcopy_run(fb); + + /* <><> end of stuff going into the package */ + qsb = qemu_buf_get(fb); + + /* Now send that blob */ + if (qemu_savevm_send_packaged(ms->to_dst_file, qsb)) { + goto fail_closefb; + } + qemu_fclose(fb); + + /* Send a notify to give a chance for anything that needs to happen + * at the transition to postcopy and after the device state; in particular + * spice needs to trigger a transition now + */ + ms->postcopy_after_devices = true; + notifier_list_notify(&migration_state_notifiers, ms); + + ms->downtime = qemu_clock_get_ms(QEMU_CLOCK_REALTIME) - time_at_stop; + + qemu_mutex_unlock_iothread(); + + /* + * Although this ping is just for debug, it could potentially be + * used for getting a better measurement of downtime at the source. + */ + qemu_savevm_send_ping(ms->to_dst_file, 4); + + ret = qemu_file_get_error(ms->to_dst_file); + if (ret) { + error_report("postcopy_start: Migration stream errored"); + migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE, + MIGRATION_STATUS_FAILED); + } + + return ret; + +fail_closefb: + qemu_fclose(fb); +fail: + migrate_set_state(&ms->state, MIGRATION_STATUS_POSTCOPY_ACTIVE, + MIGRATION_STATUS_FAILED); + qemu_mutex_unlock_iothread(); + return -1; +} + +/** + * migration_completion: Used by migration_thread when there's not much left. + * The caller 'breaks' the loop when this returns. + * + * @s: Current migration state + * @current_active_state: The migration state we expect to be in + * @*old_vm_running: Pointer to old_vm_running flag + * @*start_time: Pointer to time to update + */ +static void migration_completion(MigrationState *s, int current_active_state, + bool *old_vm_running, + int64_t *start_time) +{ + int ret; + + if (s->state == MIGRATION_STATUS_ACTIVE) { + qemu_mutex_lock_iothread(); + *start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); + qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER); + *old_vm_running = runstate_is_running(); + ret = global_state_store(); + + if (!ret) { + ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE); + if (ret >= 0) { + ret = bdrv_inactivate_all(); + } + if (ret >= 0) { + qemu_file_set_rate_limit(s->to_dst_file, INT64_MAX); + qemu_savevm_state_complete_precopy(s->to_dst_file, false); + } + } + qemu_mutex_unlock_iothread(); + + if (ret < 0) { + goto fail; + } + } else if (s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) { + trace_migration_completion_postcopy_end(); + + qemu_savevm_state_complete_postcopy(s->to_dst_file); + trace_migration_completion_postcopy_end_after_complete(); + } + + /* + * If rp was opened we must clean up the thread before + * cleaning everything else up (since if there are no failures + * it will wait for the destination to send it's status in + * a SHUT command). + * Postcopy opens rp if enabled (even if it's not avtivated) + */ + if (migrate_postcopy_ram()) { + int rp_error; + trace_migration_completion_postcopy_end_before_rp(); + rp_error = await_return_path_close_on_source(s); + trace_migration_completion_postcopy_end_after_rp(rp_error); + if (rp_error) { + goto fail; + } + } + + if (qemu_file_get_error(s->to_dst_file)) { + trace_migration_completion_file_err(); + goto fail; + } + + migrate_set_state(&s->state, current_active_state, + MIGRATION_STATUS_COMPLETED); + return; + +fail: + migrate_set_state(&s->state, current_active_state, + MIGRATION_STATUS_FAILED); +} + +/* + * Master migration thread on the source VM. + * It drives the migration and pumps the data down the outgoing channel. + */ static void *migration_thread(void *opaque) { MigrationState *s = opaque; + /* Used by the bandwidth calcs, updated later */ int64_t initial_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); int64_t setup_start = qemu_clock_get_ms(QEMU_CLOCK_HOST); int64_t initial_bytes = 0; @@ -922,70 +1630,92 @@ static void *migration_thread(void *opaque) int64_t start_time = initial_time; int64_t end_time; bool old_vm_running = false; + bool entered_postcopy = false; + /* The active state we expect to be in; ACTIVE or POSTCOPY_ACTIVE */ + enum MigrationStatus current_active_state = MIGRATION_STATUS_ACTIVE; rcu_register_thread(); - qemu_savevm_state_header(s->file); - qemu_savevm_state_begin(s->file, &s->params); + qemu_savevm_state_header(s->to_dst_file); + + if (migrate_postcopy_ram()) { + /* Now tell the dest that it should open its end so it can reply */ + qemu_savevm_send_open_return_path(s->to_dst_file); + + /* And do a ping that will make stuff easier to debug */ + qemu_savevm_send_ping(s->to_dst_file, 1); + + /* + * Tell the destination that we *might* want to do postcopy later; + * if the other end can't do postcopy it should fail now, nice and + * early. + */ + qemu_savevm_send_postcopy_advise(s->to_dst_file); + } + + qemu_savevm_state_begin(s->to_dst_file, &s->params); s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start; - migrate_set_state(s, MIGRATION_STATUS_SETUP, MIGRATION_STATUS_ACTIVE); + current_active_state = MIGRATION_STATUS_ACTIVE; + migrate_set_state(&s->state, MIGRATION_STATUS_SETUP, + MIGRATION_STATUS_ACTIVE); + + trace_migration_thread_setup_complete(); - while (s->state == MIGRATION_STATUS_ACTIVE) { + while (s->state == MIGRATION_STATUS_ACTIVE || + s->state == MIGRATION_STATUS_POSTCOPY_ACTIVE) { int64_t current_time; uint64_t pending_size; - if (!qemu_file_rate_limit(s->file)) { - pending_size = qemu_savevm_state_pending(s->file, max_size); - trace_migrate_pending(pending_size, max_size); + if (!qemu_file_rate_limit(s->to_dst_file)) { + uint64_t pend_post, pend_nonpost; + + qemu_savevm_state_pending(s->to_dst_file, max_size, &pend_nonpost, + &pend_post); + pending_size = pend_nonpost + pend_post; + trace_migrate_pending(pending_size, max_size, + pend_post, pend_nonpost); if (pending_size && pending_size >= max_size) { - qemu_savevm_state_iterate(s->file); - } else { - int ret; - - qemu_mutex_lock_iothread(); - start_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); - qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER); - old_vm_running = runstate_is_running(); - - ret = global_state_store(); - if (!ret) { - ret = vm_stop_force_state(RUN_STATE_FINISH_MIGRATE); - if (ret >= 0) { - qemu_file_set_rate_limit(s->file, INT64_MAX); - qemu_savevm_state_complete(s->file); - } - } - qemu_mutex_unlock_iothread(); + /* Still a significant amount to transfer */ - if (ret < 0) { - migrate_set_state(s, MIGRATION_STATUS_ACTIVE, - MIGRATION_STATUS_FAILED); - break; - } + if (migrate_postcopy_ram() && + s->state != MIGRATION_STATUS_POSTCOPY_ACTIVE && + pend_nonpost <= max_size && + atomic_read(&s->start_postcopy)) { - if (!qemu_file_get_error(s->file)) { - migrate_set_state(s, MIGRATION_STATUS_ACTIVE, - MIGRATION_STATUS_COMPLETED); - break; + if (!postcopy_start(s, &old_vm_running)) { + current_active_state = MIGRATION_STATUS_POSTCOPY_ACTIVE; + entered_postcopy = true; + } + + continue; } + /* Just another iteration step */ + qemu_savevm_state_iterate(s->to_dst_file, entered_postcopy); + } else { + trace_migration_thread_low_pending(pending_size); + migration_completion(s, current_active_state, + &old_vm_running, &start_time); + break; } } - if (qemu_file_get_error(s->file)) { - migrate_set_state(s, MIGRATION_STATUS_ACTIVE, + if (qemu_file_get_error(s->to_dst_file)) { + migrate_set_state(&s->state, current_active_state, MIGRATION_STATUS_FAILED); + trace_migration_thread_file_err(); break; } current_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); if (current_time >= initial_time + BUFFER_DELAY) { - uint64_t transferred_bytes = qemu_ftell(s->file) - initial_bytes; + uint64_t transferred_bytes = qemu_ftell(s->to_dst_file) - + initial_bytes; uint64_t time_spent = current_time - initial_time; - double bandwidth = transferred_bytes / time_spent; + double bandwidth = (double)transferred_bytes / time_spent; max_size = bandwidth * migrate_max_downtime() / 1000000; - s->mbps = time_spent ? (((double) transferred_bytes * 8.0) / - ((double) time_spent / 1000.0)) / 1000.0 / 1000.0 : -1; + s->mbps = (((double) transferred_bytes * 8.0) / + ((double) time_spent / 1000.0)) / 1000.0 / 1000.0; trace_migrate_transferred(transferred_bytes, time_spent, bandwidth, max_size); @@ -995,31 +1725,36 @@ static void *migration_thread(void *opaque) s->expected_downtime = s->dirty_bytes_rate / bandwidth; } - qemu_file_reset_rate_limit(s->file); + qemu_file_reset_rate_limit(s->to_dst_file); initial_time = current_time; - initial_bytes = qemu_ftell(s->file); + initial_bytes = qemu_ftell(s->to_dst_file); } - if (qemu_file_rate_limit(s->file)) { + if (qemu_file_rate_limit(s->to_dst_file)) { /* usleep expects microseconds */ g_usleep((initial_time + BUFFER_DELAY - current_time)*1000); } } + trace_migration_thread_after_loop(); + /* If we enabled cpu throttling for auto-converge, turn it off. */ + cpu_throttle_stop(); end_time = qemu_clock_get_ms(QEMU_CLOCK_REALTIME); qemu_mutex_lock_iothread(); - qemu_savevm_state_cancel(); + qemu_savevm_state_cleanup(); if (s->state == MIGRATION_STATUS_COMPLETED) { - uint64_t transferred_bytes = qemu_ftell(s->file); + uint64_t transferred_bytes = qemu_ftell(s->to_dst_file); s->total_time = end_time - s->total_time; - s->downtime = end_time - start_time; + if (!entered_postcopy) { + s->downtime = end_time - start_time; + } if (s->total_time) { s->mbps = (((double) transferred_bytes * 8.0) / ((double) s->total_time)) / 1000; } runstate_set(RUN_STATE_POSTMIGRATE); } else { - if (old_vm_running) { + if (old_vm_running && !entered_postcopy) { vm_start(); } } @@ -1036,13 +1771,40 @@ void migrate_fd_connect(MigrationState *s) s->expected_downtime = max_downtime/1000000; s->cleanup_bh = qemu_bh_new(migrate_fd_cleanup, s); - qemu_file_set_rate_limit(s->file, + qemu_file_set_rate_limit(s->to_dst_file, s->bandwidth_limit / XFER_LIMIT_RATIO); /* Notify before starting migration thread */ notifier_list_notify(&migration_state_notifiers, s); + /* + * Open the return path; currently for postcopy but other things might + * also want it. + */ + if (migrate_postcopy_ram()) { + if (open_return_path_on_source(s)) { + error_report("Unable to open return-path for postcopy"); + migrate_set_state(&s->state, MIGRATION_STATUS_SETUP, + MIGRATION_STATUS_FAILED); + migrate_fd_cleanup(s); + return; + } + } + migrate_compress_threads_create(); qemu_thread_create(&s->thread, "migration", migration_thread, s, QEMU_THREAD_JOINABLE); + s->migration_thread_running = true; +} + +PostcopyState postcopy_state_get(void) +{ + return atomic_mb_read(&incoming_postcopy_state); } + +/* Set the state and return the old state */ +PostcopyState postcopy_state_set(PostcopyState new_state) +{ + return atomic_xchg(&incoming_postcopy_state, new_state); +} + diff --git a/qemu/migration/postcopy-ram.c b/qemu/migration/postcopy-ram.c new file mode 100644 index 000000000..fbd0064fc --- /dev/null +++ b/qemu/migration/postcopy-ram.c @@ -0,0 +1,761 @@ +/* + * Postcopy migration for RAM + * + * Copyright 2013-2015 Red Hat, Inc. and/or its affiliates + * + * Authors: + * Dave Gilbert <dgilbert@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + * + */ + +/* + * Postcopy is a migration technique where the execution flips from the + * source to the destination before all the data has been copied. + */ + +#include "qemu/osdep.h" +#include <glib.h> + +#include "qemu-common.h" +#include "migration/migration.h" +#include "migration/postcopy-ram.h" +#include "sysemu/sysemu.h" +#include "sysemu/balloon.h" +#include "qemu/error-report.h" +#include "trace.h" + +/* Arbitrary limit on size of each discard command, + * keeps them around ~200 bytes + */ +#define MAX_DISCARDS_PER_COMMAND 12 + +struct PostcopyDiscardState { + const char *ramblock_name; + uint64_t offset; /* Bitmap entry for the 1st bit of this RAMBlock */ + uint16_t cur_entry; + /* + * Start and length of a discard range (bytes) + */ + uint64_t start_list[MAX_DISCARDS_PER_COMMAND]; + uint64_t length_list[MAX_DISCARDS_PER_COMMAND]; + unsigned int nsentwords; + unsigned int nsentcmds; +}; + +/* Postcopy needs to detect accesses to pages that haven't yet been copied + * across, and efficiently map new pages in, the techniques for doing this + * are target OS specific. + */ +#if defined(__linux__) + +#include <poll.h> +#include <sys/mman.h> +#include <sys/ioctl.h> +#include <sys/syscall.h> +#include <asm/types.h> /* for __u64 */ +#endif + +#if defined(__linux__) && defined(__NR_userfaultfd) && defined(CONFIG_EVENTFD) +#include <sys/eventfd.h> +#include <linux/userfaultfd.h> + +static bool ufd_version_check(int ufd) +{ + struct uffdio_api api_struct; + uint64_t ioctl_mask; + + api_struct.api = UFFD_API; + api_struct.features = 0; + if (ioctl(ufd, UFFDIO_API, &api_struct)) { + error_report("postcopy_ram_supported_by_host: UFFDIO_API failed: %s", + strerror(errno)); + return false; + } + + ioctl_mask = (__u64)1 << _UFFDIO_REGISTER | + (__u64)1 << _UFFDIO_UNREGISTER; + if ((api_struct.ioctls & ioctl_mask) != ioctl_mask) { + error_report("Missing userfault features: %" PRIx64, + (uint64_t)(~api_struct.ioctls & ioctl_mask)); + return false; + } + + return true; +} + +/* + * Note: This has the side effect of munlock'ing all of RAM, that's + * normally fine since if the postcopy succeeds it gets turned back on at the + * end. + */ +bool postcopy_ram_supported_by_host(void) +{ + long pagesize = getpagesize(); + int ufd = -1; + bool ret = false; /* Error unless we change it */ + void *testarea = NULL; + struct uffdio_register reg_struct; + struct uffdio_range range_struct; + uint64_t feature_mask; + + if ((1ul << qemu_target_page_bits()) > pagesize) { + error_report("Target page size bigger than host page size"); + goto out; + } + + ufd = syscall(__NR_userfaultfd, O_CLOEXEC); + if (ufd == -1) { + error_report("%s: userfaultfd not available: %s", __func__, + strerror(errno)); + goto out; + } + + /* Version and features check */ + if (!ufd_version_check(ufd)) { + goto out; + } + + /* + * userfault and mlock don't go together; we'll put it back later if + * it was enabled. + */ + if (munlockall()) { + error_report("%s: munlockall: %s", __func__, strerror(errno)); + return -1; + } + + /* + * We need to check that the ops we need are supported on anon memory + * To do that we need to register a chunk and see the flags that + * are returned. + */ + testarea = mmap(NULL, pagesize, PROT_READ | PROT_WRITE, MAP_PRIVATE | + MAP_ANONYMOUS, -1, 0); + if (testarea == MAP_FAILED) { + error_report("%s: Failed to map test area: %s", __func__, + strerror(errno)); + goto out; + } + g_assert(((size_t)testarea & (pagesize-1)) == 0); + + reg_struct.range.start = (uintptr_t)testarea; + reg_struct.range.len = pagesize; + reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING; + + if (ioctl(ufd, UFFDIO_REGISTER, ®_struct)) { + error_report("%s userfault register: %s", __func__, strerror(errno)); + goto out; + } + + range_struct.start = (uintptr_t)testarea; + range_struct.len = pagesize; + if (ioctl(ufd, UFFDIO_UNREGISTER, &range_struct)) { + error_report("%s userfault unregister: %s", __func__, strerror(errno)); + goto out; + } + + feature_mask = (__u64)1 << _UFFDIO_WAKE | + (__u64)1 << _UFFDIO_COPY | + (__u64)1 << _UFFDIO_ZEROPAGE; + if ((reg_struct.ioctls & feature_mask) != feature_mask) { + error_report("Missing userfault map features: %" PRIx64, + (uint64_t)(~reg_struct.ioctls & feature_mask)); + goto out; + } + + /* Success! */ + ret = true; +out: + if (testarea) { + munmap(testarea, pagesize); + } + if (ufd != -1) { + close(ufd); + } + return ret; +} + +/** + * postcopy_ram_discard_range: Discard a range of memory. + * We can assume that if we've been called postcopy_ram_hosttest returned true. + * + * @mis: Current incoming migration state. + * @start, @length: range of memory to discard. + * + * returns: 0 on success. + */ +int postcopy_ram_discard_range(MigrationIncomingState *mis, uint8_t *start, + size_t length) +{ + trace_postcopy_ram_discard_range(start, length); + if (madvise(start, length, MADV_DONTNEED)) { + error_report("%s MADV_DONTNEED: %s", __func__, strerror(errno)); + return -1; + } + + return 0; +} + +/* + * Setup an area of RAM so that it *can* be used for postcopy later; this + * must be done right at the start prior to pre-copy. + * opaque should be the MIS. + */ +static int init_range(const char *block_name, void *host_addr, + ram_addr_t offset, ram_addr_t length, void *opaque) +{ + MigrationIncomingState *mis = opaque; + + trace_postcopy_init_range(block_name, host_addr, offset, length); + + /* + * We need the whole of RAM to be truly empty for postcopy, so things + * like ROMs and any data tables built during init must be zero'd + * - we're going to get the copy from the source anyway. + * (Precopy will just overwrite this data, so doesn't need the discard) + */ + if (postcopy_ram_discard_range(mis, host_addr, length)) { + return -1; + } + + return 0; +} + +/* + * At the end of migration, undo the effects of init_range + * opaque should be the MIS. + */ +static int cleanup_range(const char *block_name, void *host_addr, + ram_addr_t offset, ram_addr_t length, void *opaque) +{ + MigrationIncomingState *mis = opaque; + struct uffdio_range range_struct; + trace_postcopy_cleanup_range(block_name, host_addr, offset, length); + + /* + * We turned off hugepage for the precopy stage with postcopy enabled + * we can turn it back on now. + */ + qemu_madvise(host_addr, length, QEMU_MADV_HUGEPAGE); + + /* + * We can also turn off userfault now since we should have all the + * pages. It can be useful to leave it on to debug postcopy + * if you're not sure it's always getting every page. + */ + range_struct.start = (uintptr_t)host_addr; + range_struct.len = length; + + if (ioctl(mis->userfault_fd, UFFDIO_UNREGISTER, &range_struct)) { + error_report("%s: userfault unregister %s", __func__, strerror(errno)); + + return -1; + } + + return 0; +} + +/* + * Initialise postcopy-ram, setting the RAM to a state where we can go into + * postcopy later; must be called prior to any precopy. + * called from arch_init's similarly named ram_postcopy_incoming_init + */ +int postcopy_ram_incoming_init(MigrationIncomingState *mis, size_t ram_pages) +{ + if (qemu_ram_foreach_block(init_range, mis)) { + return -1; + } + + return 0; +} + +/* + * At the end of a migration where postcopy_ram_incoming_init was called. + */ +int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis) +{ + trace_postcopy_ram_incoming_cleanup_entry(); + + if (mis->have_fault_thread) { + uint64_t tmp64; + + if (qemu_ram_foreach_block(cleanup_range, mis)) { + return -1; + } + /* + * Tell the fault_thread to exit, it's an eventfd that should + * currently be at 0, we're going to increment it to 1 + */ + tmp64 = 1; + if (write(mis->userfault_quit_fd, &tmp64, 8) == 8) { + trace_postcopy_ram_incoming_cleanup_join(); + qemu_thread_join(&mis->fault_thread); + } else { + /* Not much we can do here, but may as well report it */ + error_report("%s: incrementing userfault_quit_fd: %s", __func__, + strerror(errno)); + } + trace_postcopy_ram_incoming_cleanup_closeuf(); + close(mis->userfault_fd); + close(mis->userfault_quit_fd); + mis->have_fault_thread = false; + } + + qemu_balloon_inhibit(false); + + if (enable_mlock) { + if (os_mlock() < 0) { + error_report("mlock: %s", strerror(errno)); + /* + * It doesn't feel right to fail at this point, we have a valid + * VM state. + */ + } + } + + postcopy_state_set(POSTCOPY_INCOMING_END); + migrate_send_rp_shut(mis, qemu_file_get_error(mis->from_src_file) != 0); + + if (mis->postcopy_tmp_page) { + munmap(mis->postcopy_tmp_page, getpagesize()); + mis->postcopy_tmp_page = NULL; + } + trace_postcopy_ram_incoming_cleanup_exit(); + return 0; +} + +/* + * Disable huge pages on an area + */ +static int nhp_range(const char *block_name, void *host_addr, + ram_addr_t offset, ram_addr_t length, void *opaque) +{ + trace_postcopy_nhp_range(block_name, host_addr, offset, length); + + /* + * Before we do discards we need to ensure those discards really + * do delete areas of the page, even if THP thinks a hugepage would + * be a good idea, so force hugepages off. + */ + qemu_madvise(host_addr, length, QEMU_MADV_NOHUGEPAGE); + + return 0; +} + +/* + * Userfault requires us to mark RAM as NOHUGEPAGE prior to discard + * however leaving it until after precopy means that most of the precopy + * data is still THPd + */ +int postcopy_ram_prepare_discard(MigrationIncomingState *mis) +{ + if (qemu_ram_foreach_block(nhp_range, mis)) { + return -1; + } + + postcopy_state_set(POSTCOPY_INCOMING_DISCARD); + + return 0; +} + +/* + * Mark the given area of RAM as requiring notification to unwritten areas + * Used as a callback on qemu_ram_foreach_block. + * host_addr: Base of area to mark + * offset: Offset in the whole ram arena + * length: Length of the section + * opaque: MigrationIncomingState pointer + * Returns 0 on success + */ +static int ram_block_enable_notify(const char *block_name, void *host_addr, + ram_addr_t offset, ram_addr_t length, + void *opaque) +{ + MigrationIncomingState *mis = opaque; + struct uffdio_register reg_struct; + + reg_struct.range.start = (uintptr_t)host_addr; + reg_struct.range.len = length; + reg_struct.mode = UFFDIO_REGISTER_MODE_MISSING; + + /* Now tell our userfault_fd that it's responsible for this area */ + if (ioctl(mis->userfault_fd, UFFDIO_REGISTER, ®_struct)) { + error_report("%s userfault register: %s", __func__, strerror(errno)); + return -1; + } + + return 0; +} + +/* + * Handle faults detected by the USERFAULT markings + */ +static void *postcopy_ram_fault_thread(void *opaque) +{ + MigrationIncomingState *mis = opaque; + struct uffd_msg msg; + int ret; + size_t hostpagesize = getpagesize(); + RAMBlock *rb = NULL; + RAMBlock *last_rb = NULL; /* last RAMBlock we sent part of */ + + trace_postcopy_ram_fault_thread_entry(); + qemu_sem_post(&mis->fault_thread_sem); + + while (true) { + ram_addr_t rb_offset; + ram_addr_t in_raspace; + struct pollfd pfd[2]; + + /* + * We're mainly waiting for the kernel to give us a faulting HVA, + * however we can be told to quit via userfault_quit_fd which is + * an eventfd + */ + pfd[0].fd = mis->userfault_fd; + pfd[0].events = POLLIN; + pfd[0].revents = 0; + pfd[1].fd = mis->userfault_quit_fd; + pfd[1].events = POLLIN; /* Waiting for eventfd to go positive */ + pfd[1].revents = 0; + + if (poll(pfd, 2, -1 /* Wait forever */) == -1) { + error_report("%s: userfault poll: %s", __func__, strerror(errno)); + break; + } + + if (pfd[1].revents) { + trace_postcopy_ram_fault_thread_quit(); + break; + } + + ret = read(mis->userfault_fd, &msg, sizeof(msg)); + if (ret != sizeof(msg)) { + if (errno == EAGAIN) { + /* + * if a wake up happens on the other thread just after + * the poll, there is nothing to read. + */ + continue; + } + if (ret < 0) { + error_report("%s: Failed to read full userfault message: %s", + __func__, strerror(errno)); + break; + } else { + error_report("%s: Read %d bytes from userfaultfd expected %zd", + __func__, ret, sizeof(msg)); + break; /* Lost alignment, don't know what we'd read next */ + } + } + if (msg.event != UFFD_EVENT_PAGEFAULT) { + error_report("%s: Read unexpected event %ud from userfaultfd", + __func__, msg.event); + continue; /* It's not a page fault, shouldn't happen */ + } + + rb = qemu_ram_block_from_host( + (void *)(uintptr_t)msg.arg.pagefault.address, + true, &in_raspace, &rb_offset); + if (!rb) { + error_report("postcopy_ram_fault_thread: Fault outside guest: %" + PRIx64, (uint64_t)msg.arg.pagefault.address); + break; + } + + rb_offset &= ~(hostpagesize - 1); + trace_postcopy_ram_fault_thread_request(msg.arg.pagefault.address, + qemu_ram_get_idstr(rb), + rb_offset); + + /* + * Send the request to the source - we want to request one + * of our host page sizes (which is >= TPS) + */ + if (rb != last_rb) { + last_rb = rb; + migrate_send_rp_req_pages(mis, qemu_ram_get_idstr(rb), + rb_offset, hostpagesize); + } else { + /* Save some space */ + migrate_send_rp_req_pages(mis, NULL, + rb_offset, hostpagesize); + } + } + trace_postcopy_ram_fault_thread_exit(); + return NULL; +} + +int postcopy_ram_enable_notify(MigrationIncomingState *mis) +{ + /* Open the fd for the kernel to give us userfaults */ + mis->userfault_fd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK); + if (mis->userfault_fd == -1) { + error_report("%s: Failed to open userfault fd: %s", __func__, + strerror(errno)); + return -1; + } + + /* + * Although the host check already tested the API, we need to + * do the check again as an ABI handshake on the new fd. + */ + if (!ufd_version_check(mis->userfault_fd)) { + return -1; + } + + /* Now an eventfd we use to tell the fault-thread to quit */ + mis->userfault_quit_fd = eventfd(0, EFD_CLOEXEC); + if (mis->userfault_quit_fd == -1) { + error_report("%s: Opening userfault_quit_fd: %s", __func__, + strerror(errno)); + close(mis->userfault_fd); + return -1; + } + + qemu_sem_init(&mis->fault_thread_sem, 0); + qemu_thread_create(&mis->fault_thread, "postcopy/fault", + postcopy_ram_fault_thread, mis, QEMU_THREAD_JOINABLE); + qemu_sem_wait(&mis->fault_thread_sem); + qemu_sem_destroy(&mis->fault_thread_sem); + mis->have_fault_thread = true; + + /* Mark so that we get notified of accesses to unwritten areas */ + if (qemu_ram_foreach_block(ram_block_enable_notify, mis)) { + return -1; + } + + /* + * Ballooning can mark pages as absent while we're postcopying + * that would cause false userfaults. + */ + qemu_balloon_inhibit(true); + + trace_postcopy_ram_enable_notify(); + + return 0; +} + +/* + * Place a host page (from) at (host) atomically + * returns 0 on success + */ +int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from) +{ + struct uffdio_copy copy_struct; + + copy_struct.dst = (uint64_t)(uintptr_t)host; + copy_struct.src = (uint64_t)(uintptr_t)from; + copy_struct.len = getpagesize(); + copy_struct.mode = 0; + + /* copy also acks to the kernel waking the stalled thread up + * TODO: We can inhibit that ack and only do it if it was requested + * which would be slightly cheaper, but we'd have to be careful + * of the order of updating our page state. + */ + if (ioctl(mis->userfault_fd, UFFDIO_COPY, ©_struct)) { + int e = errno; + error_report("%s: %s copy host: %p from: %p", + __func__, strerror(e), host, from); + + return -e; + } + + trace_postcopy_place_page(host); + return 0; +} + +/* + * Place a zero page at (host) atomically + * returns 0 on success + */ +int postcopy_place_page_zero(MigrationIncomingState *mis, void *host) +{ + struct uffdio_zeropage zero_struct; + + zero_struct.range.start = (uint64_t)(uintptr_t)host; + zero_struct.range.len = getpagesize(); + zero_struct.mode = 0; + + if (ioctl(mis->userfault_fd, UFFDIO_ZEROPAGE, &zero_struct)) { + int e = errno; + error_report("%s: %s zero host: %p", + __func__, strerror(e), host); + + return -e; + } + + trace_postcopy_place_page_zero(host); + return 0; +} + +/* + * Returns a target page of memory that can be mapped at a later point in time + * using postcopy_place_page + * The same address is used repeatedly, postcopy_place_page just takes the + * backing page away. + * Returns: Pointer to allocated page + * + */ +void *postcopy_get_tmp_page(MigrationIncomingState *mis) +{ + if (!mis->postcopy_tmp_page) { + mis->postcopy_tmp_page = mmap(NULL, getpagesize(), + PROT_READ | PROT_WRITE, MAP_PRIVATE | + MAP_ANONYMOUS, -1, 0); + if (!mis->postcopy_tmp_page) { + error_report("%s: %s", __func__, strerror(errno)); + return NULL; + } + } + + return mis->postcopy_tmp_page; +} + +#else +/* No target OS support, stubs just fail */ +bool postcopy_ram_supported_by_host(void) +{ + error_report("%s: No OS support", __func__); + return false; +} + +int postcopy_ram_incoming_init(MigrationIncomingState *mis, size_t ram_pages) +{ + error_report("postcopy_ram_incoming_init: No OS support"); + return -1; +} + +int postcopy_ram_incoming_cleanup(MigrationIncomingState *mis) +{ + assert(0); + return -1; +} + +int postcopy_ram_discard_range(MigrationIncomingState *mis, uint8_t *start, + size_t length) +{ + assert(0); + return -1; +} + +int postcopy_ram_prepare_discard(MigrationIncomingState *mis) +{ + assert(0); + return -1; +} + +int postcopy_ram_enable_notify(MigrationIncomingState *mis) +{ + assert(0); + return -1; +} + +int postcopy_place_page(MigrationIncomingState *mis, void *host, void *from) +{ + assert(0); + return -1; +} + +int postcopy_place_page_zero(MigrationIncomingState *mis, void *host) +{ + assert(0); + return -1; +} + +void *postcopy_get_tmp_page(MigrationIncomingState *mis) +{ + assert(0); + return NULL; +} + +#endif + +/* ------------------------------------------------------------------------- */ + +/** + * postcopy_discard_send_init: Called at the start of each RAMBlock before + * asking to discard individual ranges. + * + * @ms: The current migration state. + * @offset: the bitmap offset of the named RAMBlock in the migration + * bitmap. + * @name: RAMBlock that discards will operate on. + * + * returns: a new PDS. + */ +PostcopyDiscardState *postcopy_discard_send_init(MigrationState *ms, + unsigned long offset, + const char *name) +{ + PostcopyDiscardState *res = g_malloc0(sizeof(PostcopyDiscardState)); + + if (res) { + res->ramblock_name = name; + res->offset = offset; + } + + return res; +} + +/** + * postcopy_discard_send_range: Called by the bitmap code for each chunk to + * discard. May send a discard message, may just leave it queued to + * be sent later. + * + * @ms: Current migration state. + * @pds: Structure initialised by postcopy_discard_send_init(). + * @start,@length: a range of pages in the migration bitmap in the + * RAM block passed to postcopy_discard_send_init() (length=1 is one page) + */ +void postcopy_discard_send_range(MigrationState *ms, PostcopyDiscardState *pds, + unsigned long start, unsigned long length) +{ + size_t tp_bits = qemu_target_page_bits(); + /* Convert to byte offsets within the RAM block */ + pds->start_list[pds->cur_entry] = (start - pds->offset) << tp_bits; + pds->length_list[pds->cur_entry] = length << tp_bits; + trace_postcopy_discard_send_range(pds->ramblock_name, start, length); + pds->cur_entry++; + pds->nsentwords++; + + if (pds->cur_entry == MAX_DISCARDS_PER_COMMAND) { + /* Full set, ship it! */ + qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file, + pds->ramblock_name, + pds->cur_entry, + pds->start_list, + pds->length_list); + pds->nsentcmds++; + pds->cur_entry = 0; + } +} + +/** + * postcopy_discard_send_finish: Called at the end of each RAMBlock by the + * bitmap code. Sends any outstanding discard messages, frees the PDS + * + * @ms: Current migration state. + * @pds: Structure initialised by postcopy_discard_send_init(). + */ +void postcopy_discard_send_finish(MigrationState *ms, PostcopyDiscardState *pds) +{ + /* Anything unsent? */ + if (pds->cur_entry) { + qemu_savevm_send_postcopy_ram_discard(ms->to_dst_file, + pds->ramblock_name, + pds->cur_entry, + pds->start_list, + pds->length_list); + pds->nsentcmds++; + } + + trace_postcopy_discard_send_finish(pds->ramblock_name, pds->nsentwords, + pds->nsentcmds); + + g_free(pds); +} diff --git a/qemu/migration/qemu-file-buf.c b/qemu/migration/qemu-file-buf.c index 2de9330ca..7b8e78e99 100644 --- a/qemu/migration/qemu-file-buf.c +++ b/qemu/migration/qemu-file-buf.c @@ -25,11 +25,12 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ +#include "qemu/osdep.h" #include "qemu-common.h" #include "qemu/error-report.h" #include "qemu/iov.h" #include "qemu/sockets.h" -#include "block/coroutine.h" +#include "qemu/coroutine.h" #include "migration/migration.h" #include "migration/qemu-file.h" #include "migration/qemu-file-internal.h" @@ -372,7 +373,8 @@ typedef struct QEMUBuffer { bool qsb_allocated; } QEMUBuffer; -static int buf_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size) +static ssize_t buf_get_buffer(void *opaque, uint8_t *buf, int64_t pos, + size_t size) { QEMUBuffer *s = opaque; ssize_t len = qsb_get_length(s->qsb) - pos; @@ -387,8 +389,8 @@ static int buf_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size) return qsb_get_buffer(s->qsb, pos, len, buf); } -static int buf_put_buffer(void *opaque, const uint8_t *buf, - int64_t pos, int size) +static ssize_t buf_put_buffer(void *opaque, const uint8_t *buf, + int64_t pos, size_t size) { QEMUBuffer *s = opaque; @@ -439,7 +441,7 @@ QEMUFile *qemu_bufopen(const char *mode, QEMUSizedBuffer *input) return NULL; } - s = g_malloc0(sizeof(QEMUBuffer)); + s = g_new0(QEMUBuffer, 1); s->qsb = input; if (s->qsb == NULL) { diff --git a/qemu/migration/qemu-file-stdio.c b/qemu/migration/qemu-file-stdio.c index 285068b30..f402e8f70 100644 --- a/qemu/migration/qemu-file-stdio.c +++ b/qemu/migration/qemu-file-stdio.c @@ -21,8 +21,9 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ +#include "qemu/osdep.h" #include "qemu-common.h" -#include "block/coroutine.h" +#include "qemu/coroutine.h" #include "migration/qemu-file.h" typedef struct QEMUFileStdio { @@ -37,11 +38,11 @@ static int stdio_get_fd(void *opaque) return fileno(s->stdio_file); } -static int stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos, - int size) +static ssize_t stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos, + size_t size) { QEMUFileStdio *s = opaque; - int res; + size_t res; res = fwrite(buf, 1, size, s->stdio_file); @@ -51,11 +52,12 @@ static int stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos, return res; } -static int stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size) +static ssize_t stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos, + size_t size) { QEMUFileStdio *s = opaque; FILE *fp = s->stdio_file; - int bytes; + ssize_t bytes; for (;;) { clearerr(fp); @@ -143,7 +145,7 @@ QEMUFile *qemu_popen_cmd(const char *command, const char *mode) return NULL; } - s = g_malloc0(sizeof(QEMUFileStdio)); + s = g_new0(QEMUFileStdio, 1); s->stdio_file = stdio_file; @@ -175,7 +177,7 @@ QEMUFile *qemu_fopen(const char *filename, const char *mode) return NULL; } - s = g_malloc0(sizeof(QEMUFileStdio)); + s = g_new0(QEMUFileStdio, 1); s->stdio_file = fopen(filename, mode); if (!s->stdio_file) { diff --git a/qemu/migration/qemu-file-unix.c b/qemu/migration/qemu-file-unix.c index bfbc0861a..4474e18ff 100644 --- a/qemu/migration/qemu-file-unix.c +++ b/qemu/migration/qemu-file-unix.c @@ -21,10 +21,12 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ +#include "qemu/osdep.h" #include "qemu-common.h" +#include "qemu/error-report.h" #include "qemu/iov.h" #include "qemu/sockets.h" -#include "block/coroutine.h" +#include "qemu/coroutine.h" #include "migration/qemu-file.h" #include "migration/qemu-file-internal.h" @@ -39,12 +41,42 @@ static ssize_t socket_writev_buffer(void *opaque, struct iovec *iov, int iovcnt, QEMUFileSocket *s = opaque; ssize_t len; ssize_t size = iov_size(iov, iovcnt); + ssize_t offset = 0; + int err; - len = iov_send(s->fd, iov, iovcnt, 0, size); - if (len < size) { - len = -socket_error(); - } - return len; + while (size > 0) { + len = iov_send(s->fd, iov, iovcnt, offset, size); + + if (len > 0) { + size -= len; + offset += len; + } + + if (size > 0) { + if (errno != EAGAIN && errno != EWOULDBLOCK) { + error_report("socket_writev_buffer: Got err=%d for (%zu/%zu)", + errno, (size_t)size, (size_t)len); + /* + * If I've already sent some but only just got the error, I + * could return the amount validly sent so far and wait for the + * next call to report the error, but I'd rather flag the error + * immediately. + */ + return -errno; + } + + /* Emulate blocking */ + GPollFD pfd; + + pfd.fd = s->fd; + pfd.events = G_IO_OUT | G_IO_ERR; + pfd.revents = 0; + TFR(err = g_poll(&pfd, 1, -1 /* no timeout */)); + /* Errors other than EINTR intentionally ignored */ + } + } + + return offset; } static int socket_get_fd(void *opaque) @@ -54,7 +86,8 @@ static int socket_get_fd(void *opaque) return s->fd; } -static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size) +static ssize_t socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, + size_t size) { QEMUFileSocket *s = opaque; ssize_t len; @@ -64,15 +97,15 @@ static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size) if (len != -1) { break; } - if (socket_error() == EAGAIN) { + if (errno == EAGAIN) { yield_until_fd_readable(s->fd); - } else if (socket_error() != EINTR) { + } else if (errno != EINTR) { break; } } if (len == -1) { - len = -socket_error(); + len = -errno; } return len; } @@ -96,6 +129,56 @@ static int socket_shutdown(void *opaque, bool rd, bool wr) } } +static int socket_return_close(void *opaque) +{ + QEMUFileSocket *s = opaque; + /* + * Note: We don't close the socket, that should be done by the forward + * path. + */ + g_free(s); + return 0; +} + +static const QEMUFileOps socket_return_read_ops = { + .get_fd = socket_get_fd, + .get_buffer = socket_get_buffer, + .close = socket_return_close, + .shut_down = socket_shutdown, +}; + +static const QEMUFileOps socket_return_write_ops = { + .get_fd = socket_get_fd, + .writev_buffer = socket_writev_buffer, + .close = socket_return_close, + .shut_down = socket_shutdown, +}; + +/* + * Give a QEMUFile* off the same socket but data in the opposite + * direction. + */ +static QEMUFile *socket_get_return_path(void *opaque) +{ + QEMUFileSocket *forward = opaque; + QEMUFileSocket *reverse; + + if (qemu_file_get_error(forward->file)) { + /* If the forward file is in error, don't try and open a return */ + return NULL; + } + + reverse = g_malloc0(sizeof(QEMUFileSocket)); + reverse->fd = forward->fd; + /* I don't think there's a better way to tell which direction 'this' is */ + if (forward->file->ops->get_buffer != NULL) { + /* being called from the read side, so we need to be able to write */ + return qemu_fopen_ops(reverse, &socket_return_write_ops); + } else { + return qemu_fopen_ops(reverse, &socket_return_read_ops); + } +} + static ssize_t unix_writev_buffer(void *opaque, struct iovec *iov, int iovcnt, int64_t pos) { @@ -138,7 +221,8 @@ static ssize_t unix_writev_buffer(void *opaque, struct iovec *iov, int iovcnt, return total; } -static int unix_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size) +static ssize_t unix_get_buffer(void *opaque, uint8_t *buf, int64_t pos, + size_t size) { QEMUFileSocket *s = opaque; ssize_t len; @@ -192,7 +276,7 @@ QEMUFile *qemu_fdopen(int fd, const char *mode) return NULL; } - s = g_malloc0(sizeof(QEMUFileSocket)); + s = g_new0(QEMUFileSocket, 1); s->fd = fd; if (mode[0] == 'r') { @@ -204,18 +288,19 @@ QEMUFile *qemu_fdopen(int fd, const char *mode) } static const QEMUFileOps socket_read_ops = { - .get_fd = socket_get_fd, - .get_buffer = socket_get_buffer, - .close = socket_close, - .shut_down = socket_shutdown - + .get_fd = socket_get_fd, + .get_buffer = socket_get_buffer, + .close = socket_close, + .shut_down = socket_shutdown, + .get_return_path = socket_get_return_path }; static const QEMUFileOps socket_write_ops = { - .get_fd = socket_get_fd, - .writev_buffer = socket_writev_buffer, - .close = socket_close, - .shut_down = socket_shutdown + .get_fd = socket_get_fd, + .writev_buffer = socket_writev_buffer, + .close = socket_close, + .shut_down = socket_shutdown, + .get_return_path = socket_get_return_path }; QEMUFile *qemu_fopen_socket(int fd, const char *mode) @@ -226,7 +311,7 @@ QEMUFile *qemu_fopen_socket(int fd, const char *mode) return NULL; } - s = g_malloc0(sizeof(QEMUFileSocket)); + s = g_new0(QEMUFileSocket, 1); s->fd = fd; if (mode[0] == 'w') { qemu_set_block(s->fd); diff --git a/qemu/migration/qemu-file.c b/qemu/migration/qemu-file.c index 6bb3dc15c..6f4a1299b 100644 --- a/qemu/migration/qemu-file.c +++ b/qemu/migration/qemu-file.c @@ -21,12 +21,13 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ +#include "qemu/osdep.h" #include <zlib.h> #include "qemu-common.h" #include "qemu/error-report.h" #include "qemu/iov.h" #include "qemu/sockets.h" -#include "block/coroutine.h" +#include "qemu/coroutine.h" #include "migration/migration.h" #include "migration/qemu-file.h" #include "migration/qemu-file-internal.h" @@ -44,6 +45,18 @@ int qemu_file_shutdown(QEMUFile *f) return f->ops->shut_down(f->opaque, true, true); } +/* + * Result: QEMUFile* for a 'return path' for comms in the opposite direction + * NULL if not available + */ +QEMUFile *qemu_file_get_return_path(QEMUFile *f) +{ + if (!f->ops->get_return_path) { + return NULL; + } + return f->ops->get_return_path(f->opaque); +} + bool qemu_file_mode_is_not_valid(const char *mode) { if (mode == NULL || @@ -60,7 +73,7 @@ QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops) { QEMUFile *f; - f = g_malloc0(sizeof(QEMUFile)); + f = g_new0(QEMUFile, 1); f->opaque = opaque; f->ops = ops; @@ -270,7 +283,7 @@ int qemu_fclose(QEMUFile *f) return ret; } -static void add_to_iovec(QEMUFile *f, const uint8_t *buf, int size) +static void add_to_iovec(QEMUFile *f, const uint8_t *buf, size_t size) { /* check for adjacent buffer and coalesce them */ if (f->iovcnt > 0 && buf == f->iov[f->iovcnt - 1].iov_base + @@ -286,7 +299,7 @@ static void add_to_iovec(QEMUFile *f, const uint8_t *buf, int size) } } -void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size) +void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, size_t size) { if (!f->ops->writev_buffer) { qemu_put_buffer(f, buf, size); @@ -301,9 +314,9 @@ void qemu_put_buffer_async(QEMUFile *f, const uint8_t *buf, int size) add_to_iovec(f, buf, size); } -void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size) +void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, size_t size) { - int l; + size_t l; if (f->last_error) { return; @@ -363,10 +376,10 @@ void qemu_file_skip(QEMUFile *f, int size) * return as many as it managed to read (assuming blocking fd's which * all current QEMUFile are) */ -int qemu_peek_buffer(QEMUFile *f, uint8_t **buf, int size, size_t offset) +size_t qemu_peek_buffer(QEMUFile *f, uint8_t **buf, size_t size, size_t offset) { - int pending; - int index; + ssize_t pending; + size_t index; assert(!qemu_file_is_writable(f)); assert(offset < IO_BUF_SIZE); @@ -411,13 +424,13 @@ int qemu_peek_buffer(QEMUFile *f, uint8_t **buf, int size, size_t offset) * return as many as it managed to read (assuming blocking fd's which * all current QEMUFile are) */ -int qemu_get_buffer(QEMUFile *f, uint8_t *buf, int size) +size_t qemu_get_buffer(QEMUFile *f, uint8_t *buf, size_t size) { - int pending = size; - int done = 0; + size_t pending = size; + size_t done = 0; while (pending > 0) { - int res; + size_t res; uint8_t *src; res = qemu_peek_buffer(f, &src, MIN(pending, IO_BUF_SIZE), 0); @@ -434,6 +447,43 @@ int qemu_get_buffer(QEMUFile *f, uint8_t *buf, int size) } /* + * Read 'size' bytes of data from the file. + * 'size' can be larger than the internal buffer. + * + * The data: + * may be held on an internal buffer (in which case *buf is updated + * to point to it) that is valid until the next qemu_file operation. + * OR + * will be copied to the *buf that was passed in. + * + * The code tries to avoid the copy if possible. + * + * It will return size bytes unless there was an error, in which case it will + * return as many as it managed to read (assuming blocking fd's which + * all current QEMUFile are) + * + * Note: Since **buf may get changed, the caller should take care to + * keep a pointer to the original buffer if it needs to deallocate it. + */ +size_t qemu_get_buffer_in_place(QEMUFile *f, uint8_t **buf, size_t size) +{ + if (size < IO_BUF_SIZE) { + size_t res; + uint8_t *src; + + res = qemu_peek_buffer(f, &src, size, 0); + + if (res == size) { + qemu_file_skip(f, res); + *buf = src; + return res; + } + } + + return qemu_get_buffer(f, *buf, size); +} + +/* * Peeks a single byte from the buffer; this isn't guaranteed to work if * offset leaves a gap after the previous read/peeked data. */ @@ -611,3 +661,18 @@ size_t qemu_get_counted_string(QEMUFile *f, char buf[256]) return res == len ? res : 0; } + +/* + * Set the blocking state of the QEMUFile. + * Note: On some transports the OS only keeps a single blocking state for + * both directions, and thus changing the blocking on the main + * QEMUFile can also affect the return path. + */ +void qemu_file_set_blocking(QEMUFile *f, bool block) +{ + if (block) { + qemu_set_block(qemu_get_fd(f)); + } else { + qemu_set_nonblock(qemu_get_fd(f)); + } +} diff --git a/qemu/migration/ram.c b/qemu/migration/ram.c index 6249f6e60..3f057388c 100644 --- a/qemu/migration/ram.c +++ b/qemu/migration/ram.c @@ -25,13 +25,16 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ -#include <stdint.h> +#include "qemu/osdep.h" #include <zlib.h> +#include "qapi-event.h" +#include "qemu/cutils.h" #include "qemu/bitops.h" #include "qemu/bitmap.h" #include "qemu/timer.h" #include "qemu/main-loop.h" #include "migration/migration.h" +#include "migration/postcopy-ram.h" #include "exec/address-spaces.h" #include "migration/page_cache.h" #include "qemu/error-report.h" @@ -47,9 +50,7 @@ do { } while (0) #endif -static bool mig_throttle_on; static int dirty_rate_high_cnt; -static void check_guest_throttling(void); static uint64_t bitmap_sync_count; @@ -221,12 +222,34 @@ static RAMBlock *last_seen_block; /* This is the last block from where we have sent data */ static RAMBlock *last_sent_block; static ram_addr_t last_offset; -static unsigned long *migration_bitmap; static QemuMutex migration_bitmap_mutex; static uint64_t migration_dirty_pages; static uint32_t last_version; static bool ram_bulk_stage; +/* used by the search for pages to send */ +struct PageSearchStatus { + /* Current block being searched */ + RAMBlock *block; + /* Current offset to search from */ + ram_addr_t offset; + /* Set once we wrap around */ + bool complete_round; +}; +typedef struct PageSearchStatus PageSearchStatus; + +static struct BitmapRcu { + struct rcu_head rcu; + /* Main migration bitmap */ + unsigned long *bmap; + /* bitmap of pages that haven't been sent even once + * only maintained and used in postcopy at the moment + * where it's used to send the dirtymap at the start + * of the postcopy phase + */ + unsigned long *unsentmap; +} *migration_bitmap_rcu; + struct CompressParam { bool start; bool done; @@ -243,7 +266,7 @@ struct DecompressParam { QemuMutex mutex; QemuCond cond; void *des; - uint8 *compbuf; + uint8_t *compbuf; int len; }; typedef struct DecompressParam DecompressParam; @@ -264,7 +287,6 @@ static bool quit_comp_thread; static bool quit_decomp_thread; static DecompressParam *decomp_param; static QemuThread *decompress_threads; -static uint8_t *compressed_data_buf; static int do_compress_ram_page(CompressParam *param); @@ -396,6 +418,29 @@ static size_t save_page_header(QEMUFile *f, RAMBlock *block, ram_addr_t offset) return size; } +/* Reduce amount of guest cpu execution to hopefully slow down memory writes. + * If guest dirty memory rate is reduced below the rate at which we can + * transfer pages to the destination then we should be able to complete + * migration. Some workloads dirty memory way too fast and will not effectively + * converge, even with auto-converge. + */ +static void mig_throttle_guest_down(void) +{ + MigrationState *s = migrate_get_current(); + uint64_t pct_initial = + s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INITIAL]; + uint64_t pct_icrement = + s->parameters[MIGRATION_PARAMETER_X_CPU_THROTTLE_INCREMENT]; + + /* We have not started throttling yet. Let's start it. */ + if (!cpu_throttle_active()) { + cpu_throttle_set(pct_initial); + } else { + /* Throttling already on, just increase the rate */ + cpu_throttle_set(cpu_throttle_get_percentage() + pct_icrement); + } +} + /* Update the xbzrle cache to reflect a page that's been sent as all 0. * The important thing is that a stale (not-yet-0'd) page be replaced * by the new data. @@ -495,43 +540,60 @@ static int save_xbzrle_page(QEMUFile *f, uint8_t **current_data, return 1; } -/* Called with rcu_read_lock() to protect migration_bitmap */ +/* Called with rcu_read_lock() to protect migration_bitmap + * rb: The RAMBlock to search for dirty pages in + * start: Start address (typically so we can continue from previous page) + * ram_addr_abs: Pointer into which to store the address of the dirty page + * within the global ram_addr space + * + * Returns: byte offset within memory region of the start of a dirty page + */ static inline -ram_addr_t migration_bitmap_find_and_reset_dirty(MemoryRegion *mr, - ram_addr_t start) +ram_addr_t migration_bitmap_find_dirty(RAMBlock *rb, + ram_addr_t start, + ram_addr_t *ram_addr_abs) { - unsigned long base = mr->ram_addr >> TARGET_PAGE_BITS; + unsigned long base = rb->offset >> TARGET_PAGE_BITS; unsigned long nr = base + (start >> TARGET_PAGE_BITS); - uint64_t mr_size = TARGET_PAGE_ALIGN(memory_region_size(mr)); - unsigned long size = base + (mr_size >> TARGET_PAGE_BITS); + uint64_t rb_size = rb->used_length; + unsigned long size = base + (rb_size >> TARGET_PAGE_BITS); unsigned long *bitmap; unsigned long next; - bitmap = atomic_rcu_read(&migration_bitmap); + bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap; if (ram_bulk_stage && nr > base) { next = nr + 1; } else { next = find_next_bit(bitmap, size, nr); } - if (next < size) { - clear_bit(next, bitmap); + *ram_addr_abs = next << TARGET_PAGE_BITS; + return (next - base) << TARGET_PAGE_BITS; +} + +static inline bool migration_bitmap_clear_dirty(ram_addr_t addr) +{ + bool ret; + int nr = addr >> TARGET_PAGE_BITS; + unsigned long *bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap; + + ret = test_and_clear_bit(nr, bitmap); + + if (ret) { migration_dirty_pages--; } - return (next - base) << TARGET_PAGE_BITS; + return ret; } -/* Called with rcu_read_lock() to protect migration_bitmap */ static void migration_bitmap_sync_range(ram_addr_t start, ram_addr_t length) { unsigned long *bitmap; - bitmap = atomic_rcu_read(&migration_bitmap); + bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap; migration_dirty_pages += cpu_physical_memory_sync_dirty_bitmap(bitmap, start, length); } - /* Fix me: there are too many global variables used in migration process. */ static int64_t start_time; static int64_t bytes_xfer_prev; @@ -548,7 +610,6 @@ static void migration_bitmap_sync_init(void) iterations_prev = 0; } -/* Called with iothread lock held, to protect ram_list.dirty_memory[] */ static void migration_bitmap_sync(void) { RAMBlock *block; @@ -573,7 +634,7 @@ static void migration_bitmap_sync(void) qemu_mutex_lock(&migration_bitmap_mutex); rcu_read_lock(); QLIST_FOREACH_RCU(block, &ram_list.blocks, next) { - migration_bitmap_sync_range(block->mr->ram_addr, block->used_length); + migration_bitmap_sync_range(block->offset, block->used_length); } rcu_read_unlock(); qemu_mutex_unlock(&migration_bitmap_mutex); @@ -589,21 +650,21 @@ static void migration_bitmap_sync(void) /* The following detection logic can be refined later. For now: Check to see if the dirtied bytes is 50% more than the approx. amount of bytes that just got transferred since the last time we - were in this routine. If that happens >N times (for now N==4) - we turn on the throttle down logic */ + were in this routine. If that happens twice, start or increase + throttling */ bytes_xfer_now = ram_bytes_transferred(); + if (s->dirty_pages_rate && (num_dirty_pages_period * TARGET_PAGE_SIZE > (bytes_xfer_now - bytes_xfer_prev)/2) && - (dirty_rate_high_cnt++ > 4)) { + (dirty_rate_high_cnt++ >= 2)) { trace_migration_throttle(); - mig_throttle_on = true; dirty_rate_high_cnt = 0; + mig_throttle_guest_down(); } bytes_xfer_prev = bytes_xfer_now; - } else { - mig_throttle_on = false; } + if (migrate_use_xbzrle()) { if (iterations_prev != acct_info.iterations) { acct_info.xbzrle_cache_miss_rate = @@ -621,6 +682,9 @@ static void migration_bitmap_sync(void) num_dirty_pages_period = 0; } s->dirty_sync_count = bitmap_sync_count; + if (migrate_use_events()) { + qapi_event_send_migration_pass(bitmap_sync_count, NULL); + } } /** @@ -655,6 +719,9 @@ static int save_zero_page(QEMUFile *f, RAMBlock *block, ram_addr_t offset, * ram_save_page: Send the given page to the stream * * Returns: Number of pages written. + * < 0 - error + * >=0 - Number of pages written - this might legally be 0 + * if xbzrle noticed the page was the same. * * @f: QEMUFile where to send the data * @block: block that contains the page we want to send @@ -662,18 +729,19 @@ static int save_zero_page(QEMUFile *f, RAMBlock *block, ram_addr_t offset, * @last_stage: if we are at the completion stage * @bytes_transferred: increase it with the number of transferred bytes */ -static int ram_save_page(QEMUFile *f, RAMBlock* block, ram_addr_t offset, +static int ram_save_page(QEMUFile *f, PageSearchStatus *pss, bool last_stage, uint64_t *bytes_transferred) { int pages = -1; uint64_t bytes_xmit; ram_addr_t current_addr; - MemoryRegion *mr = block->mr; uint8_t *p; int ret; bool send_async = true; + RAMBlock *block = pss->block; + ram_addr_t offset = pss->offset; - p = memory_region_get_ram_ptr(mr) + offset; + p = block->host + offset; /* In doubt sent page as normal */ bytes_xmit = 0; @@ -744,7 +812,7 @@ static int do_compress_ram_page(CompressParam *param) RAMBlock *block = param->block; ram_addr_t offset = param->offset; - p = memory_region_get_ram_ptr(block->mr) + (offset & TARGET_PAGE_MASK); + p = block->host + (offset & TARGET_PAGE_MASK); bytes_sent = save_page_header(param->file, block, offset | RAM_SAVE_FLAG_COMPRESS_PAGE); @@ -846,17 +914,18 @@ static int compress_page_with_multi_thread(QEMUFile *f, RAMBlock *block, * @last_stage: if we are at the completion stage * @bytes_transferred: increase it with the number of transferred bytes */ -static int ram_save_compressed_page(QEMUFile *f, RAMBlock *block, - ram_addr_t offset, bool last_stage, +static int ram_save_compressed_page(QEMUFile *f, PageSearchStatus *pss, + bool last_stage, uint64_t *bytes_transferred) { int pages = -1; uint64_t bytes_xmit; - MemoryRegion *mr = block->mr; uint8_t *p; int ret; + RAMBlock *block = pss->block; + ram_addr_t offset = pss->offset; - p = memory_region_get_ram_ptr(mr) + offset; + p = block->host + offset; bytes_xmit = 0; ret = ram_control_save_page(f, block->offset, @@ -909,6 +978,340 @@ static int ram_save_compressed_page(QEMUFile *f, RAMBlock *block, return pages; } +/* + * Find the next dirty page and update any state associated with + * the search process. + * + * Returns: True if a page is found + * + * @f: Current migration stream. + * @pss: Data about the state of the current dirty page scan. + * @*again: Set to false if the search has scanned the whole of RAM + * *ram_addr_abs: Pointer into which to store the address of the dirty page + * within the global ram_addr space + */ +static bool find_dirty_block(QEMUFile *f, PageSearchStatus *pss, + bool *again, ram_addr_t *ram_addr_abs) +{ + pss->offset = migration_bitmap_find_dirty(pss->block, pss->offset, + ram_addr_abs); + if (pss->complete_round && pss->block == last_seen_block && + pss->offset >= last_offset) { + /* + * We've been once around the RAM and haven't found anything. + * Give up. + */ + *again = false; + return false; + } + if (pss->offset >= pss->block->used_length) { + /* Didn't find anything in this RAM Block */ + pss->offset = 0; + pss->block = QLIST_NEXT_RCU(pss->block, next); + if (!pss->block) { + /* Hit the end of the list */ + pss->block = QLIST_FIRST_RCU(&ram_list.blocks); + /* Flag that we've looped */ + pss->complete_round = true; + ram_bulk_stage = false; + if (migrate_use_xbzrle()) { + /* If xbzrle is on, stop using the data compression at this + * point. In theory, xbzrle can do better than compression. + */ + flush_compressed_data(f); + compression_switch = false; + } + } + /* Didn't find anything this time, but try again on the new block */ + *again = true; + return false; + } else { + /* Can go around again, but... */ + *again = true; + /* We've found something so probably don't need to */ + return true; + } +} + +/* + * Helper for 'get_queued_page' - gets a page off the queue + * ms: MigrationState in + * *offset: Used to return the offset within the RAMBlock + * ram_addr_abs: global offset in the dirty/sent bitmaps + * + * Returns: block (or NULL if none available) + */ +static RAMBlock *unqueue_page(MigrationState *ms, ram_addr_t *offset, + ram_addr_t *ram_addr_abs) +{ + RAMBlock *block = NULL; + + qemu_mutex_lock(&ms->src_page_req_mutex); + if (!QSIMPLEQ_EMPTY(&ms->src_page_requests)) { + struct MigrationSrcPageRequest *entry = + QSIMPLEQ_FIRST(&ms->src_page_requests); + block = entry->rb; + *offset = entry->offset; + *ram_addr_abs = (entry->offset + entry->rb->offset) & + TARGET_PAGE_MASK; + + if (entry->len > TARGET_PAGE_SIZE) { + entry->len -= TARGET_PAGE_SIZE; + entry->offset += TARGET_PAGE_SIZE; + } else { + memory_region_unref(block->mr); + QSIMPLEQ_REMOVE_HEAD(&ms->src_page_requests, next_req); + g_free(entry); + } + } + qemu_mutex_unlock(&ms->src_page_req_mutex); + + return block; +} + +/* + * Unqueue a page from the queue fed by postcopy page requests; skips pages + * that are already sent (!dirty) + * + * ms: MigrationState in + * pss: PageSearchStatus structure updated with found block/offset + * ram_addr_abs: global offset in the dirty/sent bitmaps + * + * Returns: true if a queued page is found + */ +static bool get_queued_page(MigrationState *ms, PageSearchStatus *pss, + ram_addr_t *ram_addr_abs) +{ + RAMBlock *block; + ram_addr_t offset; + bool dirty; + + do { + block = unqueue_page(ms, &offset, ram_addr_abs); + /* + * We're sending this page, and since it's postcopy nothing else + * will dirty it, and we must make sure it doesn't get sent again + * even if this queue request was received after the background + * search already sent it. + */ + if (block) { + unsigned long *bitmap; + bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap; + dirty = test_bit(*ram_addr_abs >> TARGET_PAGE_BITS, bitmap); + if (!dirty) { + trace_get_queued_page_not_dirty( + block->idstr, (uint64_t)offset, + (uint64_t)*ram_addr_abs, + test_bit(*ram_addr_abs >> TARGET_PAGE_BITS, + atomic_rcu_read(&migration_bitmap_rcu)->unsentmap)); + } else { + trace_get_queued_page(block->idstr, + (uint64_t)offset, + (uint64_t)*ram_addr_abs); + } + } + + } while (block && !dirty); + + if (block) { + /* + * As soon as we start servicing pages out of order, then we have + * to kill the bulk stage, since the bulk stage assumes + * in (migration_bitmap_find_and_reset_dirty) that every page is + * dirty, that's no longer true. + */ + ram_bulk_stage = false; + + /* + * We want the background search to continue from the queued page + * since the guest is likely to want other pages near to the page + * it just requested. + */ + pss->block = block; + pss->offset = offset; + } + + return !!block; +} + +/** + * flush_page_queue: Flush any remaining pages in the ram request queue + * it should be empty at the end anyway, but in error cases there may be + * some left. + * + * ms: MigrationState + */ +void flush_page_queue(MigrationState *ms) +{ + struct MigrationSrcPageRequest *mspr, *next_mspr; + /* This queue generally should be empty - but in the case of a failed + * migration might have some droppings in. + */ + rcu_read_lock(); + QSIMPLEQ_FOREACH_SAFE(mspr, &ms->src_page_requests, next_req, next_mspr) { + memory_region_unref(mspr->rb->mr); + QSIMPLEQ_REMOVE_HEAD(&ms->src_page_requests, next_req); + g_free(mspr); + } + rcu_read_unlock(); +} + +/** + * Queue the pages for transmission, e.g. a request from postcopy destination + * ms: MigrationStatus in which the queue is held + * rbname: The RAMBlock the request is for - may be NULL (to mean reuse last) + * start: Offset from the start of the RAMBlock + * len: Length (in bytes) to send + * Return: 0 on success + */ +int ram_save_queue_pages(MigrationState *ms, const char *rbname, + ram_addr_t start, ram_addr_t len) +{ + RAMBlock *ramblock; + + rcu_read_lock(); + if (!rbname) { + /* Reuse last RAMBlock */ + ramblock = ms->last_req_rb; + + if (!ramblock) { + /* + * Shouldn't happen, we can't reuse the last RAMBlock if + * it's the 1st request. + */ + error_report("ram_save_queue_pages no previous block"); + goto err; + } + } else { + ramblock = qemu_ram_block_by_name(rbname); + + if (!ramblock) { + /* We shouldn't be asked for a non-existent RAMBlock */ + error_report("ram_save_queue_pages no block '%s'", rbname); + goto err; + } + ms->last_req_rb = ramblock; + } + trace_ram_save_queue_pages(ramblock->idstr, start, len); + if (start+len > ramblock->used_length) { + error_report("%s request overrun start=" RAM_ADDR_FMT " len=" + RAM_ADDR_FMT " blocklen=" RAM_ADDR_FMT, + __func__, start, len, ramblock->used_length); + goto err; + } + + struct MigrationSrcPageRequest *new_entry = + g_malloc0(sizeof(struct MigrationSrcPageRequest)); + new_entry->rb = ramblock; + new_entry->offset = start; + new_entry->len = len; + + memory_region_ref(ramblock->mr); + qemu_mutex_lock(&ms->src_page_req_mutex); + QSIMPLEQ_INSERT_TAIL(&ms->src_page_requests, new_entry, next_req); + qemu_mutex_unlock(&ms->src_page_req_mutex); + rcu_read_unlock(); + + return 0; + +err: + rcu_read_unlock(); + return -1; +} + +/** + * ram_save_target_page: Save one target page + * + * + * @f: QEMUFile where to send the data + * @block: pointer to block that contains the page we want to send + * @offset: offset inside the block for the page; + * @last_stage: if we are at the completion stage + * @bytes_transferred: increase it with the number of transferred bytes + * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space + * + * Returns: Number of pages written. + */ +static int ram_save_target_page(MigrationState *ms, QEMUFile *f, + PageSearchStatus *pss, + bool last_stage, + uint64_t *bytes_transferred, + ram_addr_t dirty_ram_abs) +{ + int res = 0; + + /* Check the pages is dirty and if it is send it */ + if (migration_bitmap_clear_dirty(dirty_ram_abs)) { + unsigned long *unsentmap; + if (compression_switch && migrate_use_compression()) { + res = ram_save_compressed_page(f, pss, + last_stage, + bytes_transferred); + } else { + res = ram_save_page(f, pss, last_stage, + bytes_transferred); + } + + if (res < 0) { + return res; + } + unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap; + if (unsentmap) { + clear_bit(dirty_ram_abs >> TARGET_PAGE_BITS, unsentmap); + } + /* Only update last_sent_block if a block was actually sent; xbzrle + * might have decided the page was identical so didn't bother writing + * to the stream. + */ + if (res > 0) { + last_sent_block = pss->block; + } + } + + return res; +} + +/** + * ram_save_host_page: Starting at *offset send pages upto the end + * of the current host page. It's valid for the initial + * offset to point into the middle of a host page + * in which case the remainder of the hostpage is sent. + * Only dirty target pages are sent. + * + * Returns: Number of pages written. + * + * @f: QEMUFile where to send the data + * @block: pointer to block that contains the page we want to send + * @offset: offset inside the block for the page; updated to last target page + * sent + * @last_stage: if we are at the completion stage + * @bytes_transferred: increase it with the number of transferred bytes + * @dirty_ram_abs: Address of the start of the dirty page in ram_addr_t space + */ +static int ram_save_host_page(MigrationState *ms, QEMUFile *f, + PageSearchStatus *pss, + bool last_stage, + uint64_t *bytes_transferred, + ram_addr_t dirty_ram_abs) +{ + int tmppages, pages = 0; + do { + tmppages = ram_save_target_page(ms, f, pss, last_stage, + bytes_transferred, dirty_ram_abs); + if (tmppages < 0) { + return tmppages; + } + + pages += tmppages; + pss->offset += TARGET_PAGE_SIZE; + dirty_ram_abs += TARGET_PAGE_SIZE; + } while (pss->offset & (qemu_host_page_size - 1)); + + /* The offset we leave with is the last one we looked at */ + pss->offset -= TARGET_PAGE_SIZE; + return pages; +} + /** * ram_find_and_save_block: Finds a dirty page and sends it to f * @@ -920,61 +1323,47 @@ static int ram_save_compressed_page(QEMUFile *f, RAMBlock *block, * @f: QEMUFile where to send the data * @last_stage: if we are at the completion stage * @bytes_transferred: increase it with the number of transferred bytes + * + * On systems where host-page-size > target-page-size it will send all the + * pages in a host page that are dirty. */ static int ram_find_and_save_block(QEMUFile *f, bool last_stage, uint64_t *bytes_transferred) { - RAMBlock *block = last_seen_block; - ram_addr_t offset = last_offset; - bool complete_round = false; + PageSearchStatus pss; + MigrationState *ms = migrate_get_current(); int pages = 0; - MemoryRegion *mr; + bool again, found; + ram_addr_t dirty_ram_abs; /* Address of the start of the dirty page in + ram_addr_t space */ - if (!block) - block = QLIST_FIRST_RCU(&ram_list.blocks); + pss.block = last_seen_block; + pss.offset = last_offset; + pss.complete_round = false; - while (true) { - mr = block->mr; - offset = migration_bitmap_find_and_reset_dirty(mr, offset); - if (complete_round && block == last_seen_block && - offset >= last_offset) { - break; + if (!pss.block) { + pss.block = QLIST_FIRST_RCU(&ram_list.blocks); + } + + do { + again = true; + found = get_queued_page(ms, &pss, &dirty_ram_abs); + + if (!found) { + /* priority queue empty, so just search for something dirty */ + found = find_dirty_block(f, &pss, &again, &dirty_ram_abs); } - if (offset >= block->used_length) { - offset = 0; - block = QLIST_NEXT_RCU(block, next); - if (!block) { - block = QLIST_FIRST_RCU(&ram_list.blocks); - complete_round = true; - ram_bulk_stage = false; - if (migrate_use_xbzrle()) { - /* If xbzrle is on, stop using the data compression at this - * point. In theory, xbzrle can do better than compression. - */ - flush_compressed_data(f); - compression_switch = false; - } - } - } else { - if (compression_switch && migrate_use_compression()) { - pages = ram_save_compressed_page(f, block, offset, last_stage, - bytes_transferred); - } else { - pages = ram_save_page(f, block, offset, last_stage, - bytes_transferred); - } - /* if page is unmodified, continue to the next */ - if (pages > 0) { - last_sent_block = block; - break; - } + if (found) { + pages = ram_save_host_page(ms, f, &pss, + last_stage, bytes_transferred, + dirty_ram_abs); } - } + } while (!pages && again); - last_seen_block = block; - last_offset = offset; + last_seen_block = pss.block; + last_offset = pss.offset; return pages; } @@ -1024,17 +1413,23 @@ void free_xbzrle_decoded_buf(void) xbzrle_decoded_buf = NULL; } -static void migration_end(void) +static void migration_bitmap_free(struct BitmapRcu *bmap) +{ + g_free(bmap->bmap); + g_free(bmap->unsentmap); + g_free(bmap); +} + +static void ram_migration_cleanup(void *opaque) { /* caller have hold iothread lock or is in a bh, so there is * no writing race against this migration_bitmap */ - unsigned long *bitmap = migration_bitmap; - atomic_rcu_set(&migration_bitmap, NULL); + struct BitmapRcu *bitmap = migration_bitmap_rcu; + atomic_rcu_set(&migration_bitmap_rcu, NULL); if (bitmap) { memory_global_dirty_log_stop(); - synchronize_rcu(); - g_free(bitmap); + call_rcu(bitmap, migration_bitmap_free, rcu); } XBZRLE_cache_lock(); @@ -1049,11 +1444,6 @@ static void migration_end(void) XBZRLE_cache_unlock(); } -static void ram_migration_cancel(void *opaque) -{ - migration_end(); -} - static void reset_ram_globals(void) { last_seen_block = NULL; @@ -1070,9 +1460,10 @@ void migration_bitmap_extend(ram_addr_t old, ram_addr_t new) /* called in qemu main thread, so there is * no writing race against this migration_bitmap */ - if (migration_bitmap) { - unsigned long *old_bitmap = migration_bitmap, *bitmap; - bitmap = bitmap_new(new); + if (migration_bitmap_rcu) { + struct BitmapRcu *old_bitmap = migration_bitmap_rcu, *bitmap; + bitmap = g_new(struct BitmapRcu, 1); + bitmap->bmap = bitmap_new(new); /* prevent migration_bitmap content from being set bit * by migration_bitmap_sync_range() at the same time. @@ -1080,16 +1471,410 @@ void migration_bitmap_extend(ram_addr_t old, ram_addr_t new) * at the same time. */ qemu_mutex_lock(&migration_bitmap_mutex); - bitmap_copy(bitmap, old_bitmap, old); - bitmap_set(bitmap, old, new - old); - atomic_rcu_set(&migration_bitmap, bitmap); + bitmap_copy(bitmap->bmap, old_bitmap->bmap, old); + bitmap_set(bitmap->bmap, old, new - old); + + /* We don't have a way to safely extend the sentmap + * with RCU; so mark it as missing, entry to postcopy + * will fail. + */ + bitmap->unsentmap = NULL; + + atomic_rcu_set(&migration_bitmap_rcu, bitmap); qemu_mutex_unlock(&migration_bitmap_mutex); migration_dirty_pages += new - old; - synchronize_rcu(); - g_free(old_bitmap); + call_rcu(old_bitmap, migration_bitmap_free, rcu); + } +} + +/* + * 'expected' is the value you expect the bitmap mostly to be full + * of; it won't bother printing lines that are all this value. + * If 'todump' is null the migration bitmap is dumped. + */ +void ram_debug_dump_bitmap(unsigned long *todump, bool expected) +{ + int64_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS; + + int64_t cur; + int64_t linelen = 128; + char linebuf[129]; + + if (!todump) { + todump = atomic_rcu_read(&migration_bitmap_rcu)->bmap; + } + + for (cur = 0; cur < ram_pages; cur += linelen) { + int64_t curb; + bool found = false; + /* + * Last line; catch the case where the line length + * is longer than remaining ram + */ + if (cur + linelen > ram_pages) { + linelen = ram_pages - cur; + } + for (curb = 0; curb < linelen; curb++) { + bool thisbit = test_bit(cur + curb, todump); + linebuf[curb] = thisbit ? '1' : '.'; + found = found || (thisbit != expected); + } + if (found) { + linebuf[curb] = '\0'; + fprintf(stderr, "0x%08" PRIx64 " : %s\n", cur, linebuf); + } + } +} + +/* **** functions for postcopy ***** */ + +/* + * Callback from postcopy_each_ram_send_discard for each RAMBlock + * Note: At this point the 'unsentmap' is the processed bitmap combined + * with the dirtymap; so a '1' means it's either dirty or unsent. + * start,length: Indexes into the bitmap for the first bit + * representing the named block and length in target-pages + */ +static int postcopy_send_discard_bm_ram(MigrationState *ms, + PostcopyDiscardState *pds, + unsigned long start, + unsigned long length) +{ + unsigned long end = start + length; /* one after the end */ + unsigned long current; + unsigned long *unsentmap; + + unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap; + for (current = start; current < end; ) { + unsigned long one = find_next_bit(unsentmap, end, current); + + if (one <= end) { + unsigned long zero = find_next_zero_bit(unsentmap, end, one + 1); + unsigned long discard_length; + + if (zero >= end) { + discard_length = end - one; + } else { + discard_length = zero - one; + } + postcopy_discard_send_range(ms, pds, one, discard_length); + current = one + discard_length; + } else { + current = one; + } + } + + return 0; +} + +/* + * Utility for the outgoing postcopy code. + * Calls postcopy_send_discard_bm_ram for each RAMBlock + * passing it bitmap indexes and name. + * Returns: 0 on success + * (qemu_ram_foreach_block ends up passing unscaled lengths + * which would mean postcopy code would have to deal with target page) + */ +static int postcopy_each_ram_send_discard(MigrationState *ms) +{ + struct RAMBlock *block; + int ret; + + QLIST_FOREACH_RCU(block, &ram_list.blocks, next) { + unsigned long first = block->offset >> TARGET_PAGE_BITS; + PostcopyDiscardState *pds = postcopy_discard_send_init(ms, + first, + block->idstr); + + /* + * Postcopy sends chunks of bitmap over the wire, but it + * just needs indexes at this point, avoids it having + * target page specific code. + */ + ret = postcopy_send_discard_bm_ram(ms, pds, first, + block->used_length >> TARGET_PAGE_BITS); + postcopy_discard_send_finish(ms, pds); + if (ret) { + return ret; + } + } + + return 0; +} + +/* + * Helper for postcopy_chunk_hostpages; it's called twice to cleanup + * the two bitmaps, that are similar, but one is inverted. + * + * We search for runs of target-pages that don't start or end on a + * host page boundary; + * unsent_pass=true: Cleans up partially unsent host pages by searching + * the unsentmap + * unsent_pass=false: Cleans up partially dirty host pages by searching + * the main migration bitmap + * + */ +static void postcopy_chunk_hostpages_pass(MigrationState *ms, bool unsent_pass, + RAMBlock *block, + PostcopyDiscardState *pds) +{ + unsigned long *bitmap; + unsigned long *unsentmap; + unsigned int host_ratio = qemu_host_page_size / TARGET_PAGE_SIZE; + unsigned long first = block->offset >> TARGET_PAGE_BITS; + unsigned long len = block->used_length >> TARGET_PAGE_BITS; + unsigned long last = first + (len - 1); + unsigned long run_start; + + bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap; + unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap; + + if (unsent_pass) { + /* Find a sent page */ + run_start = find_next_zero_bit(unsentmap, last + 1, first); + } else { + /* Find a dirty page */ + run_start = find_next_bit(bitmap, last + 1, first); + } + + while (run_start <= last) { + bool do_fixup = false; + unsigned long fixup_start_addr; + unsigned long host_offset; + + /* + * If the start of this run of pages is in the middle of a host + * page, then we need to fixup this host page. + */ + host_offset = run_start % host_ratio; + if (host_offset) { + do_fixup = true; + run_start -= host_offset; + fixup_start_addr = run_start; + /* For the next pass */ + run_start = run_start + host_ratio; + } else { + /* Find the end of this run */ + unsigned long run_end; + if (unsent_pass) { + run_end = find_next_bit(unsentmap, last + 1, run_start + 1); + } else { + run_end = find_next_zero_bit(bitmap, last + 1, run_start + 1); + } + /* + * If the end isn't at the start of a host page, then the + * run doesn't finish at the end of a host page + * and we need to discard. + */ + host_offset = run_end % host_ratio; + if (host_offset) { + do_fixup = true; + fixup_start_addr = run_end - host_offset; + /* + * This host page has gone, the next loop iteration starts + * from after the fixup + */ + run_start = fixup_start_addr + host_ratio; + } else { + /* + * No discards on this iteration, next loop starts from + * next sent/dirty page + */ + run_start = run_end + 1; + } + } + + if (do_fixup) { + unsigned long page; + + /* Tell the destination to discard this page */ + if (unsent_pass || !test_bit(fixup_start_addr, unsentmap)) { + /* For the unsent_pass we: + * discard partially sent pages + * For the !unsent_pass (dirty) we: + * discard partially dirty pages that were sent + * (any partially sent pages were already discarded + * by the previous unsent_pass) + */ + postcopy_discard_send_range(ms, pds, fixup_start_addr, + host_ratio); + } + + /* Clean up the bitmap */ + for (page = fixup_start_addr; + page < fixup_start_addr + host_ratio; page++) { + /* All pages in this host page are now not sent */ + set_bit(page, unsentmap); + + /* + * Remark them as dirty, updating the count for any pages + * that weren't previously dirty. + */ + migration_dirty_pages += !test_and_set_bit(page, bitmap); + } + } + + if (unsent_pass) { + /* Find the next sent page for the next iteration */ + run_start = find_next_zero_bit(unsentmap, last + 1, + run_start); + } else { + /* Find the next dirty page for the next iteration */ + run_start = find_next_bit(bitmap, last + 1, run_start); + } + } +} + +/* + * Utility for the outgoing postcopy code. + * + * Discard any partially sent host-page size chunks, mark any partially + * dirty host-page size chunks as all dirty. + * + * Returns: 0 on success + */ +static int postcopy_chunk_hostpages(MigrationState *ms) +{ + struct RAMBlock *block; + + if (qemu_host_page_size == TARGET_PAGE_SIZE) { + /* Easy case - TPS==HPS - nothing to be done */ + return 0; + } + + /* Easiest way to make sure we don't resume in the middle of a host-page */ + last_seen_block = NULL; + last_sent_block = NULL; + last_offset = 0; + + QLIST_FOREACH_RCU(block, &ram_list.blocks, next) { + unsigned long first = block->offset >> TARGET_PAGE_BITS; + + PostcopyDiscardState *pds = + postcopy_discard_send_init(ms, first, block->idstr); + + /* First pass: Discard all partially sent host pages */ + postcopy_chunk_hostpages_pass(ms, true, block, pds); + /* + * Second pass: Ensure that all partially dirty host pages are made + * fully dirty. + */ + postcopy_chunk_hostpages_pass(ms, false, block, pds); + + postcopy_discard_send_finish(ms, pds); + } /* ram_list loop */ + + return 0; +} + +/* + * Transmit the set of pages to be discarded after precopy to the target + * these are pages that: + * a) Have been previously transmitted but are now dirty again + * b) Pages that have never been transmitted, this ensures that + * any pages on the destination that have been mapped by background + * tasks get discarded (transparent huge pages is the specific concern) + * Hopefully this is pretty sparse + */ +int ram_postcopy_send_discard_bitmap(MigrationState *ms) +{ + int ret; + unsigned long *bitmap, *unsentmap; + + rcu_read_lock(); + + /* This should be our last sync, the src is now paused */ + migration_bitmap_sync(); + + unsentmap = atomic_rcu_read(&migration_bitmap_rcu)->unsentmap; + if (!unsentmap) { + /* We don't have a safe way to resize the sentmap, so + * if the bitmap was resized it will be NULL at this + * point. + */ + error_report("migration ram resized during precopy phase"); + rcu_read_unlock(); + return -EINVAL; + } + + /* Deal with TPS != HPS */ + ret = postcopy_chunk_hostpages(ms); + if (ret) { + rcu_read_unlock(); + return ret; + } + + /* + * Update the unsentmap to be unsentmap = unsentmap | dirty + */ + bitmap = atomic_rcu_read(&migration_bitmap_rcu)->bmap; + bitmap_or(unsentmap, unsentmap, bitmap, + last_ram_offset() >> TARGET_PAGE_BITS); + + + trace_ram_postcopy_send_discard_bitmap(); +#ifdef DEBUG_POSTCOPY + ram_debug_dump_bitmap(unsentmap, true); +#endif + + ret = postcopy_each_ram_send_discard(ms); + rcu_read_unlock(); + + return ret; +} + +/* + * At the start of the postcopy phase of migration, any now-dirty + * precopied pages are discarded. + * + * start, length describe a byte address range within the RAMBlock + * + * Returns 0 on success. + */ +int ram_discard_range(MigrationIncomingState *mis, + const char *block_name, + uint64_t start, size_t length) +{ + int ret = -1; + + rcu_read_lock(); + RAMBlock *rb = qemu_ram_block_by_name(block_name); + + if (!rb) { + error_report("ram_discard_range: Failed to find block '%s'", + block_name); + goto err; } + + uint8_t *host_startaddr = rb->host + start; + + if ((uintptr_t)host_startaddr & (qemu_host_page_size - 1)) { + error_report("ram_discard_range: Unaligned start address: %p", + host_startaddr); + goto err; + } + + if ((start + length) <= rb->used_length) { + uint8_t *host_endaddr = host_startaddr + length; + if ((uintptr_t)host_endaddr & (qemu_host_page_size - 1)) { + error_report("ram_discard_range: Unaligned end address: %p", + host_endaddr); + goto err; + } + ret = postcopy_ram_discard_range(mis, host_startaddr, length); + } else { + error_report("ram_discard_range: Overrun block '%s' (%" PRIu64 + "/%zx/" RAM_ADDR_FMT")", + block_name, start, length, rb->used_length); + } + +err: + rcu_read_unlock(); + + return ret; } + /* Each of ram_save_setup, ram_save_iterate and ram_save_complete has * long-running RCU critical section. When rcu-reclaims in the code * start to become numerous it will be necessary to reduce the @@ -1101,7 +1886,6 @@ static int ram_save_setup(QEMUFile *f, void *opaque) RAMBlock *block; int64_t ram_bitmap_pages; /* Size of bitmap in pages, including gaps */ - mig_throttle_on = false; dirty_rate_high_cnt = 0; bitmap_sync_count = 0; migration_bitmap_sync_init(); @@ -1137,16 +1921,23 @@ static int ram_save_setup(QEMUFile *f, void *opaque) acct_clear(); } - /* iothread lock needed for ram_list.dirty_memory[] */ + /* For memory_global_dirty_log_start below. */ qemu_mutex_lock_iothread(); + qemu_mutex_lock_ramlist(); rcu_read_lock(); bytes_transferred = 0; reset_ram_globals(); ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS; - migration_bitmap = bitmap_new(ram_bitmap_pages); - bitmap_set(migration_bitmap, 0, ram_bitmap_pages); + migration_bitmap_rcu = g_new0(struct BitmapRcu, 1); + migration_bitmap_rcu->bmap = bitmap_new(ram_bitmap_pages); + bitmap_set(migration_bitmap_rcu->bmap, 0, ram_bitmap_pages); + + if (migrate_postcopy_ram()) { + migration_bitmap_rcu->unsentmap = bitmap_new(ram_bitmap_pages); + bitmap_set(migration_bitmap_rcu->unsentmap, 0, ram_bitmap_pages); + } /* * Count the total number of pages used by ram blocks not including any @@ -1206,7 +1997,7 @@ static int ram_save_iterate(QEMUFile *f, void *opaque) } pages_sent += pages; acct_info.iterations++; - check_guest_throttling(); + /* we want to check in the 1st loop, just in case it was the 1st time and we had to sync the dirty bitmap. qemu_get_clock_ns() is a bit expensive, so we only check each some @@ -1247,7 +2038,9 @@ static int ram_save_complete(QEMUFile *f, void *opaque) { rcu_read_lock(); - migration_bitmap_sync(); + if (!migration_in_postcopy(migrate_get_current())) { + migration_bitmap_sync(); + } ram_control_before_iterate(f, RAM_CONTROL_FINISH); @@ -1274,13 +2067,16 @@ static int ram_save_complete(QEMUFile *f, void *opaque) return 0; } -static uint64_t ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size) +static void ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size, + uint64_t *non_postcopiable_pending, + uint64_t *postcopiable_pending) { uint64_t remaining_size; remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE; - if (remaining_size < max_size) { + if (!migration_in_postcopy(migrate_get_current()) && + remaining_size < max_size) { qemu_mutex_lock_iothread(); rcu_read_lock(); migration_bitmap_sync(); @@ -1288,17 +2084,21 @@ static uint64_t ram_save_pending(QEMUFile *f, void *opaque, uint64_t max_size) qemu_mutex_unlock_iothread(); remaining_size = ram_save_remaining() * TARGET_PAGE_SIZE; } - return remaining_size; + + /* We can do postcopy, and all the data is postcopiable */ + *postcopiable_pending += remaining_size; } static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host) { unsigned int xh_len; int xh_flags; + uint8_t *loaded_data; if (!xbzrle_decoded_buf) { xbzrle_decoded_buf = g_malloc(TARGET_PAGE_SIZE); } + loaded_data = xbzrle_decoded_buf; /* extract RLE header */ xh_flags = qemu_get_byte(f); @@ -1314,10 +2114,10 @@ static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host) return -1; } /* load data and decode */ - qemu_get_buffer(f, xbzrle_decoded_buf, xh_len); + qemu_get_buffer_in_place(f, &loaded_data, xh_len); /* decode RLE */ - if (xbzrle_decode_buffer(xbzrle_decoded_buf, xh_len, host, + if (xbzrle_decode_buffer(loaded_data, xh_len, host, TARGET_PAGE_SIZE) == -1) { error_report("Failed to load XBZRLE page - decode error!"); return -1; @@ -1329,36 +2129,48 @@ static int load_xbzrle(QEMUFile *f, ram_addr_t addr, void *host) /* Must be called from within a rcu critical section. * Returns a pointer from within the RCU-protected ram_list. */ -static inline void *host_from_stream_offset(QEMUFile *f, - ram_addr_t offset, - int flags) +/* + * Read a RAMBlock ID from the stream f. + * + * f: Stream to read from + * flags: Page flags (mostly to see if it's a continuation of previous block) + */ +static inline RAMBlock *ram_block_from_stream(QEMUFile *f, + int flags) { static RAMBlock *block = NULL; char id[256]; uint8_t len; if (flags & RAM_SAVE_FLAG_CONTINUE) { - if (!block || block->max_length <= offset) { + if (!block) { error_report("Ack, bad migration stream!"); return NULL; } - - return memory_region_get_ram_ptr(block->mr) + offset; + return block; } len = qemu_get_byte(f); qemu_get_buffer(f, (uint8_t *)id, len); id[len] = 0; - QLIST_FOREACH_RCU(block, &ram_list.blocks, next) { - if (!strncmp(id, block->idstr, sizeof(id)) && - block->max_length > offset) { - return memory_region_get_ram_ptr(block->mr) + offset; - } + block = qemu_ram_block_by_name(id); + if (!block) { + error_report("Can't find block %s", id); + return NULL; } - error_report("Can't find block %s!", id); - return NULL; + return block; +} + +static inline void *host_from_ram_block_offset(RAMBlock *block, + ram_addr_t offset) +{ + if (!offset_in_ramblock(block, offset)) { + return NULL; + } + + return block->host + offset; } /* @@ -1406,7 +2218,6 @@ void migrate_decompress_threads_create(void) thread_count = migrate_decompress_threads(); decompress_threads = g_new0(QemuThread, thread_count); decomp_param = g_new0(DecompressParam, thread_count); - compressed_data_buf = g_malloc0(compressBound(TARGET_PAGE_SIZE)); quit_decomp_thread = false; for (i = 0; i < thread_count; i++) { qemu_mutex_init(&decomp_param[i].mutex); @@ -1437,13 +2248,11 @@ void migrate_decompress_threads_join(void) } g_free(decompress_threads); g_free(decomp_param); - g_free(compressed_data_buf); decompress_threads = NULL; decomp_param = NULL; - compressed_data_buf = NULL; } -static void decompress_data_with_multi_threads(uint8_t *compbuf, +static void decompress_data_with_multi_threads(QEMUFile *f, void *host, int len) { int idx, thread_count; @@ -1452,7 +2261,7 @@ static void decompress_data_with_multi_threads(uint8_t *compbuf, while (true) { for (idx = 0; idx < thread_count; idx++) { if (!decomp_param[idx].start) { - memcpy(decomp_param[idx].compbuf, compbuf, len); + qemu_get_buffer(f, decomp_param[idx].compbuf, len); decomp_param[idx].des = host; decomp_param[idx].len = len; start_decompression(&decomp_param[idx]); @@ -1465,11 +2274,150 @@ static void decompress_data_with_multi_threads(uint8_t *compbuf, } } +/* + * Allocate data structures etc needed by incoming migration with postcopy-ram + * postcopy-ram's similarly names postcopy_ram_incoming_init does the work + */ +int ram_postcopy_incoming_init(MigrationIncomingState *mis) +{ + size_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS; + + return postcopy_ram_incoming_init(mis, ram_pages); +} + +/* + * Called in postcopy mode by ram_load(). + * rcu_read_lock is taken prior to this being called. + */ +static int ram_load_postcopy(QEMUFile *f) +{ + int flags = 0, ret = 0; + bool place_needed = false; + bool matching_page_sizes = qemu_host_page_size == TARGET_PAGE_SIZE; + MigrationIncomingState *mis = migration_incoming_get_current(); + /* Temporary page that is later 'placed' */ + void *postcopy_host_page = postcopy_get_tmp_page(mis); + void *last_host = NULL; + bool all_zero = false; + + while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) { + ram_addr_t addr; + void *host = NULL; + void *page_buffer = NULL; + void *place_source = NULL; + uint8_t ch; + + addr = qemu_get_be64(f); + flags = addr & ~TARGET_PAGE_MASK; + addr &= TARGET_PAGE_MASK; + + trace_ram_load_postcopy_loop((uint64_t)addr, flags); + place_needed = false; + if (flags & (RAM_SAVE_FLAG_COMPRESS | RAM_SAVE_FLAG_PAGE)) { + RAMBlock *block = ram_block_from_stream(f, flags); + + host = host_from_ram_block_offset(block, addr); + if (!host) { + error_report("Illegal RAM offset " RAM_ADDR_FMT, addr); + ret = -EINVAL; + break; + } + page_buffer = host; + /* + * Postcopy requires that we place whole host pages atomically. + * To make it atomic, the data is read into a temporary page + * that's moved into place later. + * The migration protocol uses, possibly smaller, target-pages + * however the source ensures it always sends all the components + * of a host page in order. + */ + page_buffer = postcopy_host_page + + ((uintptr_t)host & ~qemu_host_page_mask); + /* If all TP are zero then we can optimise the place */ + if (!((uintptr_t)host & ~qemu_host_page_mask)) { + all_zero = true; + } else { + /* not the 1st TP within the HP */ + if (host != (last_host + TARGET_PAGE_SIZE)) { + error_report("Non-sequential target page %p/%p", + host, last_host); + ret = -EINVAL; + break; + } + } + + + /* + * If it's the last part of a host page then we place the host + * page + */ + place_needed = (((uintptr_t)host + TARGET_PAGE_SIZE) & + ~qemu_host_page_mask) == 0; + place_source = postcopy_host_page; + } + last_host = host; + + switch (flags & ~RAM_SAVE_FLAG_CONTINUE) { + case RAM_SAVE_FLAG_COMPRESS: + ch = qemu_get_byte(f); + memset(page_buffer, ch, TARGET_PAGE_SIZE); + if (ch) { + all_zero = false; + } + break; + + case RAM_SAVE_FLAG_PAGE: + all_zero = false; + if (!place_needed || !matching_page_sizes) { + qemu_get_buffer(f, page_buffer, TARGET_PAGE_SIZE); + } else { + /* Avoids the qemu_file copy during postcopy, which is + * going to do a copy later; can only do it when we + * do this read in one go (matching page sizes) + */ + qemu_get_buffer_in_place(f, (uint8_t **)&place_source, + TARGET_PAGE_SIZE); + } + break; + case RAM_SAVE_FLAG_EOS: + /* normal exit */ + break; + default: + error_report("Unknown combination of migration flags: %#x" + " (postcopy mode)", flags); + ret = -EINVAL; + } + + if (place_needed) { + /* This gets called at the last target page in the host page */ + if (all_zero) { + ret = postcopy_place_page_zero(mis, + host + TARGET_PAGE_SIZE - + qemu_host_page_size); + } else { + ret = postcopy_place_page(mis, host + TARGET_PAGE_SIZE - + qemu_host_page_size, + place_source); + } + } + if (!ret) { + ret = qemu_file_get_error(f); + } + } + + return ret; +} + static int ram_load(QEMUFile *f, void *opaque, int version_id) { int flags = 0, ret = 0; static uint64_t seq_iter; int len = 0; + /* + * If system is running in postcopy mode, page inserts to host memory must + * be atomic + */ + bool postcopy_running = postcopy_state_get() >= POSTCOPY_INCOMING_LISTENING; seq_iter++; @@ -1483,15 +2431,32 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id) * critical section. */ rcu_read_lock(); - while (!ret && !(flags & RAM_SAVE_FLAG_EOS)) { + + if (postcopy_running) { + ret = ram_load_postcopy(f); + } + + while (!postcopy_running && !ret && !(flags & RAM_SAVE_FLAG_EOS)) { ram_addr_t addr, total_ram_bytes; - void *host; + void *host = NULL; uint8_t ch; addr = qemu_get_be64(f); flags = addr & ~TARGET_PAGE_MASK; addr &= TARGET_PAGE_MASK; + if (flags & (RAM_SAVE_FLAG_COMPRESS | RAM_SAVE_FLAG_PAGE | + RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) { + RAMBlock *block = ram_block_from_stream(f, flags); + + host = host_from_ram_block_offset(block, addr); + if (!host) { + error_report("Illegal RAM offset " RAM_ADDR_FMT, addr); + ret = -EINVAL; + break; + } + } + switch (flags & ~RAM_SAVE_FLAG_CONTINUE) { case RAM_SAVE_FLAG_MEM_SIZE: /* Synchronize RAM block list */ @@ -1506,23 +2471,20 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id) id[len] = 0; length = qemu_get_be64(f); - QLIST_FOREACH_RCU(block, &ram_list.blocks, next) { - if (!strncmp(id, block->idstr, sizeof(id))) { - if (length != block->used_length) { - Error *local_err = NULL; + block = qemu_ram_block_by_name(id); + if (block) { + if (length != block->used_length) { + Error *local_err = NULL; - ret = qemu_ram_resize(block->offset, length, &local_err); - if (local_err) { - error_report_err(local_err); - } + ret = qemu_ram_resize(block->offset, length, + &local_err); + if (local_err) { + error_report_err(local_err); } - ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG, - block->idstr); - break; } - } - - if (!block) { + ram_control_load_hook(f, RAM_CONTROL_BLOCK_REG, + block->idstr); + } else { error_report("Unknown ramblock \"%s\", cannot " "accept migration", id); ret = -EINVAL; @@ -1531,49 +2493,27 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id) total_ram_bytes -= length; } break; + case RAM_SAVE_FLAG_COMPRESS: - host = host_from_stream_offset(f, addr, flags); - if (!host) { - error_report("Illegal RAM offset " RAM_ADDR_FMT, addr); - ret = -EINVAL; - break; - } ch = qemu_get_byte(f); ram_handle_compressed(host, ch, TARGET_PAGE_SIZE); break; + case RAM_SAVE_FLAG_PAGE: - host = host_from_stream_offset(f, addr, flags); - if (!host) { - error_report("Illegal RAM offset " RAM_ADDR_FMT, addr); - ret = -EINVAL; - break; - } qemu_get_buffer(f, host, TARGET_PAGE_SIZE); break; - case RAM_SAVE_FLAG_COMPRESS_PAGE: - host = host_from_stream_offset(f, addr, flags); - if (!host) { - error_report("Invalid RAM offset " RAM_ADDR_FMT, addr); - ret = -EINVAL; - break; - } + case RAM_SAVE_FLAG_COMPRESS_PAGE: len = qemu_get_be32(f); if (len < 0 || len > compressBound(TARGET_PAGE_SIZE)) { error_report("Invalid compressed data length: %d", len); ret = -EINVAL; break; } - qemu_get_buffer(f, compressed_data_buf, len); - decompress_data_with_multi_threads(compressed_data_buf, host, len); + decompress_data_with_multi_threads(f, host, len); break; + case RAM_SAVE_FLAG_XBZRLE: - host = host_from_stream_offset(f, addr, flags); - if (!host) { - error_report("Illegal RAM offset " RAM_ADDR_FMT, addr); - ret = -EINVAL; - break; - } if (load_xbzrle(f, addr, host) < 0) { error_report("Failed to decompress XBZRLE page at " RAM_ADDR_FMT, addr); @@ -1607,10 +2547,11 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id) static SaveVMHandlers savevm_ram_handlers = { .save_live_setup = ram_save_setup, .save_live_iterate = ram_save_iterate, - .save_live_complete = ram_save_complete, + .save_live_complete_postcopy = ram_save_complete, + .save_live_complete_precopy = ram_save_complete, .save_live_pending = ram_save_pending, .load_state = ram_load, - .cancel = ram_migration_cancel, + .cleanup = ram_migration_cleanup, }; void ram_mig_init(void) @@ -1618,52 +2559,3 @@ void ram_mig_init(void) qemu_mutex_init(&XBZRLE.lock); register_savevm_live(NULL, "ram", 0, 4, &savevm_ram_handlers, NULL); } -/* Stub function that's gets run on the vcpu when its brought out of the - VM to run inside qemu via async_run_on_cpu()*/ - -static void mig_sleep_cpu(void *opq) -{ - qemu_mutex_unlock_iothread(); - g_usleep(30*1000); - qemu_mutex_lock_iothread(); -} - -/* To reduce the dirty rate explicitly disallow the VCPUs from spending - much time in the VM. The migration thread will try to catchup. - Workload will experience a performance drop. -*/ -static void mig_throttle_guest_down(void) -{ - CPUState *cpu; - - qemu_mutex_lock_iothread(); - CPU_FOREACH(cpu) { - async_run_on_cpu(cpu, mig_sleep_cpu, NULL); - } - qemu_mutex_unlock_iothread(); -} - -static void check_guest_throttling(void) -{ - static int64_t t0; - int64_t t1; - - if (!mig_throttle_on) { - return; - } - - if (!t0) { - t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); - return; - } - - t1 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME); - - /* If it has been more than 40 ms since the last time the guest - * was throttled then do it again. - */ - if (40 < (t1-t0)/1000000) { - mig_throttle_guest_down(); - t0 = t1; - } -} diff --git a/qemu/migration/rdma.c b/qemu/migration/rdma.c index 74876fd7a..f6a9992b3 100644 --- a/qemu/migration/rdma.c +++ b/qemu/migration/rdma.c @@ -11,7 +11,10 @@ * later. See the COPYING file in the top-level directory. * */ +#include "qemu/osdep.h" +#include "qapi/error.h" #include "qemu-common.h" +#include "qemu/cutils.h" #include "migration/migration.h" #include "migration/qemu-file.h" #include "exec/cpu-common.h" @@ -19,13 +22,10 @@ #include "qemu/main-loop.h" #include "qemu/sockets.h" #include "qemu/bitmap.h" -#include "block/coroutine.h" -#include <stdio.h> -#include <sys/types.h> +#include "qemu/coroutine.h" #include <sys/socket.h> #include <netdb.h> #include <arpa/inet.h> -#include <string.h> #include <rdma/rdma_cma.h> #include "trace.h" @@ -541,7 +541,7 @@ static int rdma_add_block(RDMAContext *rdma, const char *block_name, RDMALocalBlock *block; RDMALocalBlock *old = local->block; - local->block = g_malloc0(sizeof(RDMALocalBlock) * (local->nb_blocks + 1)); + local->block = g_new0(RDMALocalBlock, local->nb_blocks + 1); if (local->nb_blocks) { int x; @@ -572,12 +572,12 @@ static int rdma_add_block(RDMAContext *rdma, const char *block_name, bitmap_clear(block->transit_bitmap, 0, block->nb_chunks); block->unregister_bitmap = bitmap_new(block->nb_chunks); bitmap_clear(block->unregister_bitmap, 0, block->nb_chunks); - block->remote_keys = g_malloc0(block->nb_chunks * sizeof(uint32_t)); + block->remote_keys = g_new0(uint32_t, block->nb_chunks); block->is_ram_block = local->init ? false : true; if (rdma->blockmap) { - g_hash_table_insert(rdma->blockmap, (void *) block_offset, block); + g_hash_table_insert(rdma->blockmap, (void *)(uintptr_t)block_offset, block); } trace_rdma_add_block(block_name, local->nb_blocks, @@ -617,8 +617,8 @@ static int qemu_rdma_init_ram_blocks(RDMAContext *rdma) memset(local, 0, sizeof *local); qemu_ram_foreach_block(qemu_rdma_init_one_block, rdma); trace_qemu_rdma_init_ram_blocks(local->nb_blocks); - rdma->dest_blocks = (RDMADestBlock *) g_malloc0(sizeof(RDMADestBlock) * - rdma->local_ram_blocks.nb_blocks); + rdma->dest_blocks = g_new0(RDMADestBlock, + rdma->local_ram_blocks.nb_blocks); local->init = true; return 0; } @@ -677,8 +677,7 @@ static int rdma_delete_block(RDMAContext *rdma, RDMALocalBlock *block) if (local->nb_blocks > 1) { - local->block = g_malloc0(sizeof(RDMALocalBlock) * - (local->nb_blocks - 1)); + local->block = g_new0(RDMALocalBlock, local->nb_blocks - 1); if (block->index) { memcpy(local->block, old, sizeof(RDMALocalBlock) * block->index); @@ -778,7 +777,7 @@ static void qemu_rdma_dump_gid(const char *who, struct rdma_cm_id *id) * * If the source VM connects with an IPv4 address without knowing that the * destination has bound to '[::]' the migration will unconditionally fail - * unless the management software is explicitly listening on the the IPv4 + * unless the management software is explicitly listening on the IPv4 * address while using a RoCE-based device. * * If the source VM connects with an IPv6 address, then we're OK because we can @@ -1164,7 +1163,7 @@ static int qemu_rdma_register_and_get_keys(RDMAContext *rdma, /* allocate memory to store chunk MRs */ if (!block->pmr) { - block->pmr = g_malloc0(block->nb_chunks * sizeof(struct ibv_mr *)); + block->pmr = g_new0(struct ibv_mr *, block->nb_chunks); } /* @@ -2494,7 +2493,7 @@ static void *qemu_rdma_data_init(const char *host_port, Error **errp) InetSocketAddress *addr; if (host_port) { - rdma = g_malloc0(sizeof(RDMAContext)); + rdma = g_new0(RDMAContext, 1); rdma->current_index = -1; rdma->current_chunk = -1; @@ -2519,8 +2518,8 @@ static void *qemu_rdma_data_init(const char *host_port, Error **errp) * SEND messages for control only. * VM's ram is handled with regular RDMA messages. */ -static int qemu_rdma_put_buffer(void *opaque, const uint8_t *buf, - int64_t pos, int size) +static ssize_t qemu_rdma_put_buffer(void *opaque, const uint8_t *buf, + int64_t pos, size_t size) { QEMUFileRDMA *r = opaque; QEMUFile *f = r->file; @@ -2547,7 +2546,8 @@ static int qemu_rdma_put_buffer(void *opaque, const uint8_t *buf, r->len = MIN(remaining, RDMA_SEND_INCREMENT); remaining -= r->len; - head.len = r->len; + /* Guaranteed to fit due to RDMA_SEND_INCREMENT MIN above */ + head.len = (uint32_t)r->len; head.type = RDMA_CONTROL_QEMU_FILE; ret = qemu_rdma_exchange_send(rdma, &head, data, NULL, NULL, NULL); @@ -2564,7 +2564,7 @@ static int qemu_rdma_put_buffer(void *opaque, const uint8_t *buf, } static size_t qemu_rdma_fill(RDMAContext *rdma, uint8_t *buf, - int size, int idx) + size_t size, int idx) { size_t len = 0; @@ -2585,8 +2585,8 @@ static size_t qemu_rdma_fill(RDMAContext *rdma, uint8_t *buf, * RDMA links don't use bytestreams, so we have to * return bytes to QEMUFile opportunistically. */ -static int qemu_rdma_get_buffer(void *opaque, uint8_t *buf, - int64_t pos, int size) +static ssize_t qemu_rdma_get_buffer(void *opaque, uint8_t *buf, + int64_t pos, size_t size) { QEMUFileRDMA *r = opaque; RDMAContext *rdma = r->rdma; @@ -3399,7 +3399,7 @@ static void *qemu_fopen_rdma(RDMAContext *rdma, const char *mode) return NULL; } - r = g_malloc0(sizeof(QEMUFileRDMA)); + r = g_new0(QEMUFileRDMA, 1); r->rdma = rdma; if (mode[0] == 'w') { @@ -3506,7 +3506,7 @@ void rdma_start_outgoing_migration(void *opaque, trace_rdma_start_outgoing_migration_after_rdma_connect(); - s->file = qemu_fopen_rdma(rdma, "wb"); + s->to_dst_file = qemu_fopen_rdma(rdma, "wb"); migrate_fd_connect(s); return; err: diff --git a/qemu/migration/savevm.c b/qemu/migration/savevm.c index a42874b10..16ba44379 100644 --- a/qemu/migration/savevm.c +++ b/qemu/migration/savevm.c @@ -26,8 +26,7 @@ * THE SOFTWARE. */ -#include "config-host.h" -#include "qemu-common.h" +#include "qemu/osdep.h" #include "hw/boards.h" #include "hw/hw.h" #include "hw/qdev.h" @@ -37,6 +36,7 @@ #include "qemu/timer.h" #include "audio/audio.h" #include "migration/migration.h" +#include "migration/postcopy-ram.h" #include "qapi/qmp/qerror.h" #include "qemu/error-report.h" #include "qemu/sockets.h" @@ -45,10 +45,11 @@ #include "exec/memory.h" #include "qmp-commands.h" #include "trace.h" +#include "qemu/bitops.h" #include "qemu/iov.h" #include "block/snapshot.h" #include "block/qapi.h" - +#include "qemu/cutils.h" #ifndef ETH_P_RARP #define ETH_P_RARP 0x8035 @@ -57,8 +58,26 @@ #define ARP_PTYPE_IP 0x0800 #define ARP_OP_REQUEST_REV 0x3 +const unsigned int postcopy_ram_discard_version = 0; + static bool skip_section_footers; +static struct mig_cmd_args { + ssize_t len; /* -1 = variable */ + const char *name; +} mig_cmd_args[] = { + [MIG_CMD_INVALID] = { .len = -1, .name = "INVALID" }, + [MIG_CMD_OPEN_RETURN_PATH] = { .len = 0, .name = "OPEN_RETURN_PATH" }, + [MIG_CMD_PING] = { .len = sizeof(uint32_t), .name = "PING" }, + [MIG_CMD_POSTCOPY_ADVISE] = { .len = 16, .name = "POSTCOPY_ADVISE" }, + [MIG_CMD_POSTCOPY_LISTEN] = { .len = 0, .name = "POSTCOPY_LISTEN" }, + [MIG_CMD_POSTCOPY_RUN] = { .len = 0, .name = "POSTCOPY_RUN" }, + [MIG_CMD_POSTCOPY_RAM_DISCARD] = { + .len = -1, .name = "POSTCOPY_RAM_DISCARD" }, + [MIG_CMD_PACKAGED] = { .len = 4, .name = "PACKAGED" }, + [MIG_CMD_MAX] = { .len = -1, .name = "MAX" }, +}; + static int announce_self_create(uint8_t *buf, uint8_t *mac_addr) { @@ -138,14 +157,15 @@ static ssize_t block_writev_buffer(void *opaque, struct iovec *iov, int iovcnt, return qiov.size; } -static int block_put_buffer(void *opaque, const uint8_t *buf, - int64_t pos, int size) +static ssize_t block_put_buffer(void *opaque, const uint8_t *buf, + int64_t pos, size_t size) { bdrv_save_vmstate(opaque, buf, pos, size); return size; } -static int block_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size) +static ssize_t block_get_buffer(void *opaque, uint8_t *buf, int64_t pos, + size_t size) { return bdrv_load_vmstate(opaque, buf, pos, size); } @@ -278,8 +298,8 @@ static int configuration_post_load(void *opaque, int version_id) const char *current_name = MACHINE_GET_CLASS(current_machine)->name; if (strncmp(state->name, current_name, state->len) != 0) { - error_report("Machine type received is '%s' and local is '%s'", - state->name, current_name); + error_report("Machine type received is '%.*s' and local is '%s'", + (int) state->len, state->name, current_name); return -EINVAL; } return 0; @@ -480,7 +500,7 @@ int register_savevm_live(DeviceState *dev, { SaveStateEntry *se; - se = g_malloc0(sizeof(SaveStateEntry)); + se = g_new0(SaveStateEntry, 1); se->version_id = version_id; se->section_id = savevm_state.global_section_id++; se->ops = ops; @@ -498,7 +518,7 @@ int register_savevm_live(DeviceState *dev, pstrcat(se->idstr, sizeof(se->idstr), "/"); g_free(id); - se->compat = g_malloc0(sizeof(CompatEntry)); + se->compat = g_new0(CompatEntry, 1); pstrcpy(se->compat->idstr, sizeof(se->compat->idstr), idstr); se->compat->instance_id = instance_id == -1 ? calculate_compat_instance_id(idstr) : instance_id; @@ -526,7 +546,7 @@ int register_savevm(DeviceState *dev, LoadStateHandler *load_state, void *opaque) { - SaveVMHandlers *ops = g_malloc0(sizeof(SaveVMHandlers)); + SaveVMHandlers *ops = g_new0(SaveVMHandlers, 1); ops->save_state = save_state; ops->load_state = load_state; return register_savevm_live(dev, idstr, instance_id, version_id, @@ -551,9 +571,7 @@ void unregister_savevm(DeviceState *dev, const char *idstr, void *opaque) QTAILQ_FOREACH_SAFE(se, &savevm_state.handlers, entry, new_se) { if (strcmp(se->idstr, id) == 0 && se->opaque == opaque) { QTAILQ_REMOVE(&savevm_state.handlers, se, entry); - if (se->compat) { - g_free(se->compat); - } + g_free(se->compat); g_free(se->ops); g_free(se); } @@ -570,7 +588,7 @@ int vmstate_register_with_alias_id(DeviceState *dev, int instance_id, /* If this triggers, alias support can be dropped for the vmsd. */ assert(alias_id == -1 || required_for_version >= vmsd->minimum_version_id); - se = g_malloc0(sizeof(SaveStateEntry)); + se = g_new0(SaveStateEntry, 1); se->version_id = vmsd->version_id; se->section_id = savevm_state.global_section_id++; se->opaque = opaque; @@ -584,7 +602,7 @@ int vmstate_register_with_alias_id(DeviceState *dev, int instance_id, pstrcat(se->idstr, sizeof(se->idstr), "/"); g_free(id); - se->compat = g_malloc0(sizeof(CompatEntry)); + se->compat = g_new0(CompatEntry, 1); pstrcpy(se->compat->idstr, sizeof(se->compat->idstr), vmsd->name); se->compat->instance_id = instance_id == -1 ? calculate_compat_instance_id(vmsd->name) : instance_id; @@ -612,9 +630,7 @@ void vmstate_unregister(DeviceState *dev, const VMStateDescription *vmsd, QTAILQ_FOREACH_SAFE(se, &savevm_state.handlers, entry, new_se) { if (se->vmsd == vmsd && se->opaque == opaque) { QTAILQ_REMOVE(&savevm_state.handlers, se, entry); - if (se->compat) { - g_free(se->compat); - } + g_free(se->compat); g_free(se); } } @@ -697,6 +713,156 @@ static void save_section_footer(QEMUFile *f, SaveStateEntry *se) } } +/** + * qemu_savevm_command_send: Send a 'QEMU_VM_COMMAND' type element with the + * command and associated data. + * + * @f: File to send command on + * @command: Command type to send + * @len: Length of associated data + * @data: Data associated with command. + */ +void qemu_savevm_command_send(QEMUFile *f, + enum qemu_vm_cmd command, + uint16_t len, + uint8_t *data) +{ + trace_savevm_command_send(command, len); + qemu_put_byte(f, QEMU_VM_COMMAND); + qemu_put_be16(f, (uint16_t)command); + qemu_put_be16(f, len); + qemu_put_buffer(f, data, len); + qemu_fflush(f); +} + +void qemu_savevm_send_ping(QEMUFile *f, uint32_t value) +{ + uint32_t buf; + + trace_savevm_send_ping(value); + buf = cpu_to_be32(value); + qemu_savevm_command_send(f, MIG_CMD_PING, sizeof(value), (uint8_t *)&buf); +} + +void qemu_savevm_send_open_return_path(QEMUFile *f) +{ + trace_savevm_send_open_return_path(); + qemu_savevm_command_send(f, MIG_CMD_OPEN_RETURN_PATH, 0, NULL); +} + +/* We have a buffer of data to send; we don't want that all to be loaded + * by the command itself, so the command contains just the length of the + * extra buffer that we then send straight after it. + * TODO: Must be a better way to organise that + * + * Returns: + * 0 on success + * -ve on error + */ +int qemu_savevm_send_packaged(QEMUFile *f, const QEMUSizedBuffer *qsb) +{ + size_t cur_iov; + size_t len = qsb_get_length(qsb); + uint32_t tmp; + + if (len > MAX_VM_CMD_PACKAGED_SIZE) { + error_report("%s: Unreasonably large packaged state: %zu", + __func__, len); + return -1; + } + + tmp = cpu_to_be32(len); + + trace_qemu_savevm_send_packaged(); + qemu_savevm_command_send(f, MIG_CMD_PACKAGED, 4, (uint8_t *)&tmp); + + /* all the data follows (concatinating the iov's) */ + for (cur_iov = 0; cur_iov < qsb->n_iov; cur_iov++) { + /* The iov entries are partially filled */ + size_t towrite = MIN(qsb->iov[cur_iov].iov_len, len); + len -= towrite; + + if (!towrite) { + break; + } + + qemu_put_buffer(f, qsb->iov[cur_iov].iov_base, towrite); + } + + return 0; +} + +/* Send prior to any postcopy transfer */ +void qemu_savevm_send_postcopy_advise(QEMUFile *f) +{ + uint64_t tmp[2]; + tmp[0] = cpu_to_be64(getpagesize()); + tmp[1] = cpu_to_be64(1ul << qemu_target_page_bits()); + + trace_qemu_savevm_send_postcopy_advise(); + qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_ADVISE, 16, (uint8_t *)tmp); +} + +/* Sent prior to starting the destination running in postcopy, discard pages + * that have already been sent but redirtied on the source. + * CMD_POSTCOPY_RAM_DISCARD consist of: + * byte version (0) + * byte Length of name field (not including 0) + * n x byte RAM block name + * byte 0 terminator (just for safety) + * n x Byte ranges within the named RAMBlock + * be64 Start of the range + * be64 Length + * + * name: RAMBlock name that these entries are part of + * len: Number of page entries + * start_list: 'len' addresses + * length_list: 'len' addresses + * + */ +void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name, + uint16_t len, + uint64_t *start_list, + uint64_t *length_list) +{ + uint8_t *buf; + uint16_t tmplen; + uint16_t t; + size_t name_len = strlen(name); + + trace_qemu_savevm_send_postcopy_ram_discard(name, len); + assert(name_len < 256); + buf = g_malloc0(1 + 1 + name_len + 1 + (8 + 8) * len); + buf[0] = postcopy_ram_discard_version; + buf[1] = name_len; + memcpy(buf + 2, name, name_len); + tmplen = 2 + name_len; + buf[tmplen++] = '\0'; + + for (t = 0; t < len; t++) { + cpu_to_be64w((uint64_t *)(buf + tmplen), start_list[t]); + tmplen += 8; + cpu_to_be64w((uint64_t *)(buf + tmplen), length_list[t]); + tmplen += 8; + } + qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_RAM_DISCARD, tmplen, buf); + g_free(buf); +} + +/* Get the destination into a state where it can receive postcopy data. */ +void qemu_savevm_send_postcopy_listen(QEMUFile *f) +{ + trace_savevm_send_postcopy_listen(); + qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_LISTEN, 0, NULL); +} + +/* Kick the destination into running */ +void qemu_savevm_send_postcopy_run(QEMUFile *f) +{ + trace_savevm_send_postcopy_run(); + qemu_savevm_command_send(f, MIG_CMD_POSTCOPY_RUN, 0, NULL); +} + bool qemu_savevm_state_blocked(Error **errp) { SaveStateEntry *se; @@ -711,11 +877,23 @@ bool qemu_savevm_state_blocked(Error **errp) return false; } +static bool enforce_config_section(void) +{ + MachineState *machine = MACHINE(qdev_get_machine()); + return machine->enforce_config_section; +} + void qemu_savevm_state_header(QEMUFile *f) { trace_savevm_state_header(); qemu_put_be32(f, QEMU_VM_FILE_MAGIC); qemu_put_be32(f, QEMU_VM_FILE_VERSION); + + if (!savevm_state.skip_configuration || enforce_config_section()) { + qemu_put_byte(f, QEMU_VM_CONFIGURATION); + vmstate_save_state(f, &vmstate_configuration, &savevm_state, 0); + } + } void qemu_savevm_state_begin(QEMUFile *f, @@ -732,11 +910,6 @@ void qemu_savevm_state_begin(QEMUFile *f, se->ops->set_params(params, se->opaque); } - if (!savevm_state.skip_configuration) { - qemu_put_byte(f, QEMU_VM_CONFIGURATION); - vmstate_save_state(f, &vmstate_configuration, &savevm_state, 0); - } - QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { if (!se->ops || !se->ops->save_live_setup) { continue; @@ -763,7 +936,7 @@ void qemu_savevm_state_begin(QEMUFile *f, * 0 : We haven't finished, caller have to go again * 1 : We have finished, we can go to complete phase */ -int qemu_savevm_state_iterate(QEMUFile *f) +int qemu_savevm_state_iterate(QEMUFile *f, bool postcopy) { SaveStateEntry *se; int ret = 1; @@ -778,6 +951,15 @@ int qemu_savevm_state_iterate(QEMUFile *f) continue; } } + /* + * In the postcopy phase, any device that doesn't know how to + * do postcopy should have saved it's state in the _complete + * call that's already run, it might get confused if we call + * iterate afterwards. + */ + if (postcopy && !se->ops->save_live_complete_postcopy) { + continue; + } if (qemu_file_rate_limit(f)) { return 0; } @@ -806,24 +988,69 @@ int qemu_savevm_state_iterate(QEMUFile *f) static bool should_send_vmdesc(void) { MachineState *machine = MACHINE(qdev_get_machine()); - return !machine->suppress_vmdesc; + bool in_postcopy = migration_in_postcopy(migrate_get_current()); + return !machine->suppress_vmdesc && !in_postcopy; } -void qemu_savevm_state_complete(QEMUFile *f) +/* + * Calls the save_live_complete_postcopy methods + * causing the last few pages to be sent immediately and doing any associated + * cleanup. + * Note postcopy also calls qemu_savevm_state_complete_precopy to complete + * all the other devices, but that happens at the point we switch to postcopy. + */ +void qemu_savevm_state_complete_postcopy(QEMUFile *f) +{ + SaveStateEntry *se; + int ret; + + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { + if (!se->ops || !se->ops->save_live_complete_postcopy) { + continue; + } + if (se->ops && se->ops->is_active) { + if (!se->ops->is_active(se->opaque)) { + continue; + } + } + trace_savevm_section_start(se->idstr, se->section_id); + /* Section type */ + qemu_put_byte(f, QEMU_VM_SECTION_END); + qemu_put_be32(f, se->section_id); + + ret = se->ops->save_live_complete_postcopy(f, se->opaque); + trace_savevm_section_end(se->idstr, se->section_id, ret); + save_section_footer(f, se); + if (ret < 0) { + qemu_file_set_error(f, ret); + return; + } + } + + qemu_put_byte(f, QEMU_VM_EOF); + qemu_fflush(f); +} + +void qemu_savevm_state_complete_precopy(QEMUFile *f, bool iterable_only) { QJSON *vmdesc; int vmdesc_len; SaveStateEntry *se; int ret; + bool in_postcopy = migration_in_postcopy(migrate_get_current()); - trace_savevm_state_complete(); + trace_savevm_state_complete_precopy(); cpu_synchronize_all_states(); QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { - if (!se->ops || !se->ops->save_live_complete) { + if (!se->ops || + (in_postcopy && se->ops->save_live_complete_postcopy) || + (in_postcopy && !iterable_only) || + !se->ops->save_live_complete_precopy) { continue; } + if (se->ops && se->ops->is_active) { if (!se->ops->is_active(se->opaque)) { continue; @@ -833,7 +1060,7 @@ void qemu_savevm_state_complete(QEMUFile *f) save_section_header(f, se, QEMU_VM_SECTION_END); - ret = se->ops->save_live_complete(f, se->opaque); + ret = se->ops->save_live_complete_precopy(f, se->opaque); trace_savevm_section_end(se->idstr, se->section_id, ret); save_section_footer(f, se); if (ret < 0) { @@ -842,6 +1069,10 @@ void qemu_savevm_state_complete(QEMUFile *f) } } + if (iterable_only) { + return; + } + vmdesc = qjson_new(); json_prop_int(vmdesc, "page_size", TARGET_PAGE_SIZE); json_start_array(vmdesc, "devices"); @@ -862,15 +1093,17 @@ void qemu_savevm_state_complete(QEMUFile *f) json_prop_int(vmdesc, "instance_id", se->instance_id); save_section_header(f, se, QEMU_VM_SECTION_FULL); - vmstate_save(f, se, vmdesc); - - json_end_object(vmdesc); trace_savevm_section_end(se->idstr, se->section_id, 0); save_section_footer(f, se); + + json_end_object(vmdesc); } - qemu_put_byte(f, QEMU_VM_EOF); + if (!in_postcopy) { + /* Postcopy stream will still be going */ + qemu_put_byte(f, QEMU_VM_EOF); + } json_end_array(vmdesc); qjson_finish(vmdesc); @@ -886,10 +1119,19 @@ void qemu_savevm_state_complete(QEMUFile *f) qemu_fflush(f); } -uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size) +/* Give an estimate of the amount left to be transferred, + * the result is split into the amount for units that can and + * for units that can't do postcopy. + */ +void qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size, + uint64_t *res_non_postcopiable, + uint64_t *res_postcopiable) { SaveStateEntry *se; - uint64_t ret = 0; + + *res_non_postcopiable = 0; + *res_postcopiable = 0; + QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { if (!se->ops || !se->ops->save_live_pending) { @@ -900,19 +1142,19 @@ uint64_t qemu_savevm_state_pending(QEMUFile *f, uint64_t max_size) continue; } } - ret += se->ops->save_live_pending(f, se->opaque, max_size); + se->ops->save_live_pending(f, se->opaque, max_size, + res_non_postcopiable, res_postcopiable); } - return ret; } -void qemu_savevm_state_cancel(void) +void qemu_savevm_state_cleanup(void) { SaveStateEntry *se; - trace_savevm_state_cancel(); + trace_savevm_state_cleanup(); QTAILQ_FOREACH(se, &savevm_state.handlers, entry) { - if (se->ops && se->ops->cancel) { - se->ops->cancel(se->opaque); + if (se->ops && se->ops->cleanup) { + se->ops->cleanup(se->opaque); } } } @@ -924,6 +1166,8 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp) .blk = 0, .shared = 0 }; + MigrationState *ms = migrate_init(¶ms); + ms->to_dst_file = f; if (qemu_savevm_state_blocked(errp)) { return -EINVAL; @@ -935,17 +1179,17 @@ static int qemu_savevm_state(QEMUFile *f, Error **errp) qemu_mutex_lock_iothread(); while (qemu_file_get_error(f) == 0) { - if (qemu_savevm_state_iterate(f) > 0) { + if (qemu_savevm_state_iterate(f, false) > 0) { break; } } ret = qemu_file_get_error(f); if (ret == 0) { - qemu_savevm_state_complete(f); + qemu_savevm_state_complete_precopy(f, false); ret = qemu_file_get_error(f); } - qemu_savevm_state_cancel(); + qemu_savevm_state_cleanup(); if (ret != 0) { error_setg_errno(errp, -ret, "Error while writing VM state"); } @@ -1004,6 +1248,445 @@ static SaveStateEntry *find_se(const char *idstr, int instance_id) return NULL; } +enum LoadVMExitCodes { + /* Allow a command to quit all layers of nested loadvm loops */ + LOADVM_QUIT = 1, +}; + +static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis); + +/* ------ incoming postcopy messages ------ */ +/* 'advise' arrives before any transfers just to tell us that a postcopy + * *might* happen - it might be skipped if precopy transferred everything + * quickly. + */ +static int loadvm_postcopy_handle_advise(MigrationIncomingState *mis) +{ + PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_ADVISE); + uint64_t remote_hps, remote_tps; + + trace_loadvm_postcopy_handle_advise(); + if (ps != POSTCOPY_INCOMING_NONE) { + error_report("CMD_POSTCOPY_ADVISE in wrong postcopy state (%d)", ps); + return -1; + } + + if (!postcopy_ram_supported_by_host()) { + return -1; + } + + remote_hps = qemu_get_be64(mis->from_src_file); + if (remote_hps != getpagesize()) { + /* + * Some combinations of mismatch are probably possible but it gets + * a bit more complicated. In particular we need to place whole + * host pages on the dest at once, and we need to ensure that we + * handle dirtying to make sure we never end up sending part of + * a hostpage on it's own. + */ + error_report("Postcopy needs matching host page sizes (s=%d d=%d)", + (int)remote_hps, getpagesize()); + return -1; + } + + remote_tps = qemu_get_be64(mis->from_src_file); + if (remote_tps != (1ul << qemu_target_page_bits())) { + /* + * Again, some differences could be dealt with, but for now keep it + * simple. + */ + error_report("Postcopy needs matching target page sizes (s=%d d=%d)", + (int)remote_tps, 1 << qemu_target_page_bits()); + return -1; + } + + if (ram_postcopy_incoming_init(mis)) { + return -1; + } + + postcopy_state_set(POSTCOPY_INCOMING_ADVISE); + + return 0; +} + +/* After postcopy we will be told to throw some pages away since they're + * dirty and will have to be demand fetched. Must happen before CPU is + * started. + * There can be 0..many of these messages, each encoding multiple pages. + */ +static int loadvm_postcopy_ram_handle_discard(MigrationIncomingState *mis, + uint16_t len) +{ + int tmp; + char ramid[256]; + PostcopyState ps = postcopy_state_get(); + + trace_loadvm_postcopy_ram_handle_discard(); + + switch (ps) { + case POSTCOPY_INCOMING_ADVISE: + /* 1st discard */ + tmp = postcopy_ram_prepare_discard(mis); + if (tmp) { + return tmp; + } + break; + + case POSTCOPY_INCOMING_DISCARD: + /* Expected state */ + break; + + default: + error_report("CMD_POSTCOPY_RAM_DISCARD in wrong postcopy state (%d)", + ps); + return -1; + } + /* We're expecting a + * Version (0) + * a RAM ID string (length byte, name, 0 term) + * then at least 1 16 byte chunk + */ + if (len < (1 + 1 + 1 + 1 + 2 * 8)) { + error_report("CMD_POSTCOPY_RAM_DISCARD invalid length (%d)", len); + return -1; + } + + tmp = qemu_get_byte(mis->from_src_file); + if (tmp != postcopy_ram_discard_version) { + error_report("CMD_POSTCOPY_RAM_DISCARD invalid version (%d)", tmp); + return -1; + } + + if (!qemu_get_counted_string(mis->from_src_file, ramid)) { + error_report("CMD_POSTCOPY_RAM_DISCARD Failed to read RAMBlock ID"); + return -1; + } + tmp = qemu_get_byte(mis->from_src_file); + if (tmp != 0) { + error_report("CMD_POSTCOPY_RAM_DISCARD missing nil (%d)", tmp); + return -1; + } + + len -= 3 + strlen(ramid); + if (len % 16) { + error_report("CMD_POSTCOPY_RAM_DISCARD invalid length (%d)", len); + return -1; + } + trace_loadvm_postcopy_ram_handle_discard_header(ramid, len); + while (len) { + uint64_t start_addr, block_length; + start_addr = qemu_get_be64(mis->from_src_file); + block_length = qemu_get_be64(mis->from_src_file); + + len -= 16; + int ret = ram_discard_range(mis, ramid, start_addr, + block_length); + if (ret) { + return ret; + } + } + trace_loadvm_postcopy_ram_handle_discard_end(); + + return 0; +} + +/* + * Triggered by a postcopy_listen command; this thread takes over reading + * the input stream, leaving the main thread free to carry on loading the rest + * of the device state (from RAM). + * (TODO:This could do with being in a postcopy file - but there again it's + * just another input loop, not that postcopy specific) + */ +static void *postcopy_ram_listen_thread(void *opaque) +{ + QEMUFile *f = opaque; + MigrationIncomingState *mis = migration_incoming_get_current(); + int load_res; + + migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE, + MIGRATION_STATUS_POSTCOPY_ACTIVE); + qemu_sem_post(&mis->listen_thread_sem); + trace_postcopy_ram_listen_thread_start(); + + /* + * Because we're a thread and not a coroutine we can't yield + * in qemu_file, and thus we must be blocking now. + */ + qemu_file_set_blocking(f, true); + load_res = qemu_loadvm_state_main(f, mis); + /* And non-blocking again so we don't block in any cleanup */ + qemu_file_set_blocking(f, false); + + trace_postcopy_ram_listen_thread_exit(); + if (load_res < 0) { + error_report("%s: loadvm failed: %d", __func__, load_res); + qemu_file_set_error(f, load_res); + migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE, + MIGRATION_STATUS_FAILED); + } else { + /* + * This looks good, but it's possible that the device loading in the + * main thread hasn't finished yet, and so we might not be in 'RUN' + * state yet; wait for the end of the main thread. + */ + qemu_event_wait(&mis->main_thread_load_event); + } + postcopy_ram_incoming_cleanup(mis); + + if (load_res < 0) { + /* + * If something went wrong then we have a bad state so exit; + * depending how far we got it might be possible at this point + * to leave the guest running and fire MCEs for pages that never + * arrived as a desperate recovery step. + */ + exit(EXIT_FAILURE); + } + + migrate_set_state(&mis->state, MIGRATION_STATUS_POSTCOPY_ACTIVE, + MIGRATION_STATUS_COMPLETED); + /* + * If everything has worked fine, then the main thread has waited + * for us to start, and we're the last use of the mis. + * (If something broke then qemu will have to exit anyway since it's + * got a bad migration state). + */ + migration_incoming_state_destroy(); + + + return NULL; +} + +/* After this message we must be able to immediately receive postcopy data */ +static int loadvm_postcopy_handle_listen(MigrationIncomingState *mis) +{ + PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_LISTENING); + trace_loadvm_postcopy_handle_listen(); + if (ps != POSTCOPY_INCOMING_ADVISE && ps != POSTCOPY_INCOMING_DISCARD) { + error_report("CMD_POSTCOPY_LISTEN in wrong postcopy state (%d)", ps); + return -1; + } + if (ps == POSTCOPY_INCOMING_ADVISE) { + /* + * A rare case, we entered listen without having to do any discards, + * so do the setup that's normally done at the time of the 1st discard. + */ + postcopy_ram_prepare_discard(mis); + } + + /* + * Sensitise RAM - can now generate requests for blocks that don't exist + * However, at this point the CPU shouldn't be running, and the IO + * shouldn't be doing anything yet so don't actually expect requests + */ + if (postcopy_ram_enable_notify(mis)) { + return -1; + } + + if (mis->have_listen_thread) { + error_report("CMD_POSTCOPY_RAM_LISTEN already has a listen thread"); + return -1; + } + + mis->have_listen_thread = true; + /* Start up the listening thread and wait for it to signal ready */ + qemu_sem_init(&mis->listen_thread_sem, 0); + qemu_thread_create(&mis->listen_thread, "postcopy/listen", + postcopy_ram_listen_thread, mis->from_src_file, + QEMU_THREAD_DETACHED); + qemu_sem_wait(&mis->listen_thread_sem); + qemu_sem_destroy(&mis->listen_thread_sem); + + return 0; +} + + +typedef struct { + QEMUBH *bh; +} HandleRunBhData; + +static void loadvm_postcopy_handle_run_bh(void *opaque) +{ + Error *local_err = NULL; + HandleRunBhData *data = opaque; + + /* TODO we should move all of this lot into postcopy_ram.c or a shared code + * in migration.c + */ + cpu_synchronize_all_post_init(); + + qemu_announce_self(); + + /* Make sure all file formats flush their mutable metadata */ + bdrv_invalidate_cache_all(&local_err); + if (local_err) { + error_report_err(local_err); + } + + trace_loadvm_postcopy_handle_run_cpu_sync(); + cpu_synchronize_all_post_init(); + + trace_loadvm_postcopy_handle_run_vmstart(); + + if (autostart) { + /* Hold onto your hats, starting the CPU */ + vm_start(); + } else { + /* leave it paused and let management decide when to start the CPU */ + runstate_set(RUN_STATE_PAUSED); + } + + qemu_bh_delete(data->bh); + g_free(data); +} + +/* After all discards we can start running and asking for pages */ +static int loadvm_postcopy_handle_run(MigrationIncomingState *mis) +{ + PostcopyState ps = postcopy_state_set(POSTCOPY_INCOMING_RUNNING); + HandleRunBhData *data; + + trace_loadvm_postcopy_handle_run(); + if (ps != POSTCOPY_INCOMING_LISTENING) { + error_report("CMD_POSTCOPY_RUN in wrong postcopy state (%d)", ps); + return -1; + } + + data = g_new(HandleRunBhData, 1); + data->bh = qemu_bh_new(loadvm_postcopy_handle_run_bh, data); + qemu_bh_schedule(data->bh); + + /* We need to finish reading the stream from the package + * and also stop reading anything more from the stream that loaded the + * package (since it's now being read by the listener thread). + * LOADVM_QUIT will quit all the layers of nested loadvm loops. + */ + return LOADVM_QUIT; +} + +/** + * Immediately following this command is a blob of data containing an embedded + * chunk of migration stream; read it and load it. + * + * @mis: Incoming state + * @length: Length of packaged data to read + * + * Returns: Negative values on error + * + */ +static int loadvm_handle_cmd_packaged(MigrationIncomingState *mis) +{ + int ret; + uint8_t *buffer; + uint32_t length; + QEMUSizedBuffer *qsb; + + length = qemu_get_be32(mis->from_src_file); + trace_loadvm_handle_cmd_packaged(length); + + if (length > MAX_VM_CMD_PACKAGED_SIZE) { + error_report("Unreasonably large packaged state: %u", length); + return -1; + } + buffer = g_malloc0(length); + ret = qemu_get_buffer(mis->from_src_file, buffer, (int)length); + if (ret != length) { + g_free(buffer); + error_report("CMD_PACKAGED: Buffer receive fail ret=%d length=%d", + ret, length); + return (ret < 0) ? ret : -EAGAIN; + } + trace_loadvm_handle_cmd_packaged_received(ret); + + /* Setup a dummy QEMUFile that actually reads from the buffer */ + qsb = qsb_create(buffer, length); + g_free(buffer); /* Because qsb_create copies */ + if (!qsb) { + error_report("Unable to create qsb"); + } + QEMUFile *packf = qemu_bufopen("r", qsb); + + ret = qemu_loadvm_state_main(packf, mis); + trace_loadvm_handle_cmd_packaged_main(ret); + qemu_fclose(packf); + qsb_free(qsb); + + return ret; +} + +/* + * Process an incoming 'QEMU_VM_COMMAND' + * 0 just a normal return + * LOADVM_QUIT All good, but exit the loop + * <0 Error + */ +static int loadvm_process_command(QEMUFile *f) +{ + MigrationIncomingState *mis = migration_incoming_get_current(); + uint16_t cmd; + uint16_t len; + uint32_t tmp32; + + cmd = qemu_get_be16(f); + len = qemu_get_be16(f); + + trace_loadvm_process_command(cmd, len); + if (cmd >= MIG_CMD_MAX || cmd == MIG_CMD_INVALID) { + error_report("MIG_CMD 0x%x unknown (len 0x%x)", cmd, len); + return -EINVAL; + } + + if (mig_cmd_args[cmd].len != -1 && mig_cmd_args[cmd].len != len) { + error_report("%s received with bad length - expecting %zu, got %d", + mig_cmd_args[cmd].name, + (size_t)mig_cmd_args[cmd].len, len); + return -ERANGE; + } + + switch (cmd) { + case MIG_CMD_OPEN_RETURN_PATH: + if (mis->to_src_file) { + error_report("CMD_OPEN_RETURN_PATH called when RP already open"); + /* Not really a problem, so don't give up */ + return 0; + } + mis->to_src_file = qemu_file_get_return_path(f); + if (!mis->to_src_file) { + error_report("CMD_OPEN_RETURN_PATH failed"); + return -1; + } + break; + + case MIG_CMD_PING: + tmp32 = qemu_get_be32(f); + trace_loadvm_process_command_ping(tmp32); + if (!mis->to_src_file) { + error_report("CMD_PING (0x%x) received with no return path", + tmp32); + return -1; + } + migrate_send_rp_pong(mis, tmp32); + break; + + case MIG_CMD_PACKAGED: + return loadvm_handle_cmd_packaged(mis); + + case MIG_CMD_POSTCOPY_ADVISE: + return loadvm_postcopy_handle_advise(mis); + + case MIG_CMD_POSTCOPY_LISTEN: + return loadvm_postcopy_handle_listen(mis); + + case MIG_CMD_POSTCOPY_RUN: + return loadvm_postcopy_handle_run(mis); + + case MIG_CMD_POSTCOPY_RAM_DISCARD: + return loadvm_postcopy_ram_handle_discard(mis, len); + } + + return 0; +} + struct LoadStateEntry { QLIST_ENTRY(LoadStateEntry) entry; SaveStateEntry *se; @@ -1056,14 +1739,141 @@ void loadvm_free_handlers(MigrationIncomingState *mis) } } +static int +qemu_loadvm_section_start_full(QEMUFile *f, MigrationIncomingState *mis) +{ + uint32_t instance_id, version_id, section_id; + SaveStateEntry *se; + LoadStateEntry *le; + char idstr[256]; + int ret; + + /* Read section start */ + section_id = qemu_get_be32(f); + if (!qemu_get_counted_string(f, idstr)) { + error_report("Unable to read ID string for section %u", + section_id); + return -EINVAL; + } + instance_id = qemu_get_be32(f); + version_id = qemu_get_be32(f); + + trace_qemu_loadvm_state_section_startfull(section_id, idstr, + instance_id, version_id); + /* Find savevm section */ + se = find_se(idstr, instance_id); + if (se == NULL) { + error_report("Unknown savevm section or instance '%s' %d", + idstr, instance_id); + return -EINVAL; + } + + /* Validate version */ + if (version_id > se->version_id) { + error_report("savevm: unsupported version %d for '%s' v%d", + version_id, idstr, se->version_id); + return -EINVAL; + } + + /* Add entry */ + le = g_malloc0(sizeof(*le)); + + le->se = se; + le->section_id = section_id; + le->version_id = version_id; + QLIST_INSERT_HEAD(&mis->loadvm_handlers, le, entry); + + ret = vmstate_load(f, le->se, le->version_id); + if (ret < 0) { + error_report("error while loading state for instance 0x%x of" + " device '%s'", instance_id, idstr); + return ret; + } + if (!check_section_footer(f, le)) { + return -EINVAL; + } + + return 0; +} + +static int +qemu_loadvm_section_part_end(QEMUFile *f, MigrationIncomingState *mis) +{ + uint32_t section_id; + LoadStateEntry *le; + int ret; + + section_id = qemu_get_be32(f); + + trace_qemu_loadvm_state_section_partend(section_id); + QLIST_FOREACH(le, &mis->loadvm_handlers, entry) { + if (le->section_id == section_id) { + break; + } + } + if (le == NULL) { + error_report("Unknown savevm section %d", section_id); + return -EINVAL; + } + + ret = vmstate_load(f, le->se, le->version_id); + if (ret < 0) { + error_report("error while loading state section id %d(%s)", + section_id, le->se->idstr); + return ret; + } + if (!check_section_footer(f, le)) { + return -EINVAL; + } + + return 0; +} + +static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis) +{ + uint8_t section_type; + int ret; + + while ((section_type = qemu_get_byte(f)) != QEMU_VM_EOF) { + + trace_qemu_loadvm_state_section(section_type); + switch (section_type) { + case QEMU_VM_SECTION_START: + case QEMU_VM_SECTION_FULL: + ret = qemu_loadvm_section_start_full(f, mis); + if (ret < 0) { + return ret; + } + break; + case QEMU_VM_SECTION_PART: + case QEMU_VM_SECTION_END: + ret = qemu_loadvm_section_part_end(f, mis); + if (ret < 0) { + return ret; + } + break; + case QEMU_VM_COMMAND: + ret = loadvm_process_command(f); + trace_qemu_loadvm_state_section_command(ret); + if ((ret < 0) || (ret & LOADVM_QUIT)) { + return ret; + } + break; + default: + error_report("Unknown savevm section type %d", section_type); + return -EINVAL; + } + } + + return 0; +} + int qemu_loadvm_state(QEMUFile *f) { MigrationIncomingState *mis = migration_incoming_get_current(); Error *local_err = NULL; - uint8_t section_type; unsigned int v; int ret; - int file_error_after_eof = -1; if (qemu_savevm_state_blocked(&local_err)) { error_report_err(local_err); @@ -1086,7 +1896,7 @@ int qemu_loadvm_state(QEMUFile *f) return -ENOTSUP; } - if (!savevm_state.skip_configuration) { + if (!savevm_state.skip_configuration || enforce_config_section()) { if (qemu_get_byte(f) != QEMU_VM_CONFIGURATION) { error_report("Configuration section missing"); return -EINVAL; @@ -1098,99 +1908,19 @@ int qemu_loadvm_state(QEMUFile *f) } } - while ((section_type = qemu_get_byte(f)) != QEMU_VM_EOF) { - uint32_t instance_id, version_id, section_id; - SaveStateEntry *se; - LoadStateEntry *le; - char idstr[256]; - - trace_qemu_loadvm_state_section(section_type); - switch (section_type) { - case QEMU_VM_SECTION_START: - case QEMU_VM_SECTION_FULL: - /* Read section start */ - section_id = qemu_get_be32(f); - if (!qemu_get_counted_string(f, idstr)) { - error_report("Unable to read ID string for section %u", - section_id); - return -EINVAL; - } - instance_id = qemu_get_be32(f); - version_id = qemu_get_be32(f); - - trace_qemu_loadvm_state_section_startfull(section_id, idstr, - instance_id, version_id); - /* Find savevm section */ - se = find_se(idstr, instance_id); - if (se == NULL) { - error_report("Unknown savevm section or instance '%s' %d", - idstr, instance_id); - ret = -EINVAL; - goto out; - } - - /* Validate version */ - if (version_id > se->version_id) { - error_report("savevm: unsupported version %d for '%s' v%d", - version_id, idstr, se->version_id); - ret = -EINVAL; - goto out; - } - - /* Add entry */ - le = g_malloc0(sizeof(*le)); - - le->se = se; - le->section_id = section_id; - le->version_id = version_id; - QLIST_INSERT_HEAD(&mis->loadvm_handlers, le, entry); - - ret = vmstate_load(f, le->se, le->version_id); - if (ret < 0) { - error_report("error while loading state for instance 0x%x of" - " device '%s'", instance_id, idstr); - goto out; - } - if (!check_section_footer(f, le)) { - ret = -EINVAL; - goto out; - } - break; - case QEMU_VM_SECTION_PART: - case QEMU_VM_SECTION_END: - section_id = qemu_get_be32(f); + ret = qemu_loadvm_state_main(f, mis); + qemu_event_set(&mis->main_thread_load_event); - trace_qemu_loadvm_state_section_partend(section_id); - QLIST_FOREACH(le, &mis->loadvm_handlers, entry) { - if (le->section_id == section_id) { - break; - } - } - if (le == NULL) { - error_report("Unknown savevm section %d", section_id); - ret = -EINVAL; - goto out; - } + trace_qemu_loadvm_state_post_main(ret); - ret = vmstate_load(f, le->se, le->version_id); - if (ret < 0) { - error_report("error while loading state section id %d(%s)", - section_id, le->se->idstr); - goto out; - } - if (!check_section_footer(f, le)) { - ret = -EINVAL; - goto out; - } - break; - default: - error_report("Unknown savevm section type %d", section_type); - ret = -EINVAL; - goto out; - } + if (mis->have_listen_thread) { + /* Listen thread still going, can't clean up yet */ + return ret; } - file_error_after_eof = qemu_file_get_error(f); + if (ret == 0) { + ret = qemu_file_get_error(f); + } /* * Try to read in the VMDESC section as well, so that dumping tools that @@ -1202,10 +1932,10 @@ int qemu_loadvm_state(QEMUFile *f) * We also mustn't read data that isn't there; some transports (RDMA) * will stall waiting for that data when the source has already closed. */ - if (should_send_vmdesc()) { + if (ret == 0 && should_send_vmdesc()) { uint8_t *buf; uint32_t size; - section_type = qemu_get_byte(f); + uint8_t section_type = qemu_get_byte(f); if (section_type != QEMU_VM_VMDESCRIPTION) { error_report("Expected vmdescription section, but got %d", @@ -1229,57 +1959,9 @@ int qemu_loadvm_state(QEMUFile *f) cpu_synchronize_all_post_init(); - ret = 0; - -out: - if (ret == 0) { - /* We may not have a VMDESC section, so ignore relative errors */ - ret = file_error_after_eof; - } - return ret; } -static BlockDriverState *find_vmstate_bs(void) -{ - BlockDriverState *bs = NULL; - while ((bs = bdrv_next(bs))) { - if (bdrv_can_snapshot(bs)) { - return bs; - } - } - return NULL; -} - -/* - * Deletes snapshots of a given name in all opened images. - */ -static int del_existing_snapshots(Monitor *mon, const char *name) -{ - BlockDriverState *bs; - QEMUSnapshotInfo sn1, *snapshot = &sn1; - Error *err = NULL; - - bs = NULL; - while ((bs = bdrv_next(bs))) { - if (bdrv_can_snapshot(bs) && - bdrv_snapshot_find(bs, snapshot, name) >= 0) { - bdrv_snapshot_delete_by_id_or_name(bs, name, &err); - if (err) { - monitor_printf(mon, - "Error while deleting snapshot on device '%s':" - " %s\n", - bdrv_get_device_name(bs), - error_get_pretty(err)); - error_free(err); - return -1; - } - } - } - - return 0; -} - void hmp_savevm(Monitor *mon, const QDict *qdict) { BlockDriverState *bs, *bs1; @@ -1292,27 +1974,28 @@ void hmp_savevm(Monitor *mon, const QDict *qdict) struct tm tm; const char *name = qdict_get_try_str(qdict, "name"); Error *local_err = NULL; + AioContext *aio_context; - /* Verify if there is a device that doesn't support snapshots and is writable */ - bs = NULL; - while ((bs = bdrv_next(bs))) { - - if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) { - continue; - } + if (!bdrv_all_can_snapshot(&bs)) { + monitor_printf(mon, "Device '%s' is writable but does not " + "support snapshots.\n", bdrv_get_device_name(bs)); + return; + } - if (!bdrv_can_snapshot(bs)) { - monitor_printf(mon, "Device '%s' is writable but does not support snapshots.\n", - bdrv_get_device_name(bs)); - return; - } + /* Delete old snapshots of the same name */ + if (name && bdrv_all_delete_snapshot(name, &bs1, &local_err) < 0) { + error_reportf_err(local_err, + "Error while deleting snapshot on device '%s': ", + bdrv_get_device_name(bs1)); + return; } - bs = find_vmstate_bs(); - if (!bs) { + bs = bdrv_all_find_vmstate_bs(); + if (bs == NULL) { monitor_printf(mon, "No block device can accept snapshots\n"); return; } + aio_context = bdrv_get_aio_context(bs); saved_vm_running = runstate_is_running(); @@ -1323,6 +2006,8 @@ void hmp_savevm(Monitor *mon, const QDict *qdict) } vm_stop(RUN_STATE_SAVE_VM); + aio_context_acquire(aio_context); + memset(sn, 0, sizeof(*sn)); /* fill auxiliary fields */ @@ -1345,11 +2030,6 @@ void hmp_savevm(Monitor *mon, const QDict *qdict) strftime(sn->name, sizeof(sn->name), "vm-%Y%m%d%H%M%S", &tm); } - /* Delete old snapshots of the same name */ - if (name && del_existing_snapshots(mon, name) < 0) { - goto the_end; - } - /* save the VM state */ f = qemu_fopen_bdrv(bs, 1); if (!f) { @@ -1360,27 +2040,18 @@ void hmp_savevm(Monitor *mon, const QDict *qdict) vm_state_size = qemu_ftell(f); qemu_fclose(f); if (ret < 0) { - monitor_printf(mon, "%s\n", error_get_pretty(local_err)); - error_free(local_err); + error_report_err(local_err); goto the_end; } - /* create the snapshots */ - - bs1 = NULL; - while ((bs1 = bdrv_next(bs1))) { - if (bdrv_can_snapshot(bs1)) { - /* Write VM state size only to the image that contains the state */ - sn->vm_state_size = (bs == bs1 ? vm_state_size : 0); - ret = bdrv_snapshot_create(bs1, sn); - if (ret < 0) { - monitor_printf(mon, "Error while creating snapshot on '%s'\n", - bdrv_get_device_name(bs1)); - } - } + ret = bdrv_all_create_snapshot(sn, bs, vm_state_size, &bs); + if (ret < 0) { + monitor_printf(mon, "Error while creating snapshot on '%s'\n", + bdrv_get_device_name(bs)); } the_end: + aio_context_release(aio_context); if (saved_vm_running) { vm_start(); } @@ -1419,15 +2090,31 @@ int load_vmstate(const char *name) QEMUSnapshotInfo sn; QEMUFile *f; int ret; + AioContext *aio_context; + + if (!bdrv_all_can_snapshot(&bs)) { + error_report("Device '%s' is writable but does not support snapshots.", + bdrv_get_device_name(bs)); + return -ENOTSUP; + } + ret = bdrv_all_find_snapshot(name, &bs); + if (ret < 0) { + error_report("Device '%s' does not have the requested snapshot '%s'", + bdrv_get_device_name(bs), name); + return ret; + } - bs_vm_state = find_vmstate_bs(); + bs_vm_state = bdrv_all_find_vmstate_bs(); if (!bs_vm_state) { error_report("No block device supports snapshots"); return -ENOTSUP; } + aio_context = bdrv_get_aio_context(bs_vm_state); /* Don't even try to load empty VM states */ + aio_context_acquire(aio_context); ret = bdrv_snapshot_find(bs_vm_state, &sn, name); + aio_context_release(aio_context); if (ret < 0) { return ret; } else if (sn.vm_state_size == 0) { @@ -1436,42 +2123,14 @@ int load_vmstate(const char *name) return -EINVAL; } - /* Verify if there is any device that doesn't support snapshots and is - writable and check if the requested snapshot is available too. */ - bs = NULL; - while ((bs = bdrv_next(bs))) { - - if (!bdrv_is_inserted(bs) || bdrv_is_read_only(bs)) { - continue; - } - - if (!bdrv_can_snapshot(bs)) { - error_report("Device '%s' is writable but does not support snapshots.", - bdrv_get_device_name(bs)); - return -ENOTSUP; - } - - ret = bdrv_snapshot_find(bs, &sn, name); - if (ret < 0) { - error_report("Device '%s' does not have the requested snapshot '%s'", - bdrv_get_device_name(bs), name); - return ret; - } - } - /* Flush all IO requests so they don't interfere with the new state. */ bdrv_drain_all(); - bs = NULL; - while ((bs = bdrv_next(bs))) { - if (bdrv_can_snapshot(bs)) { - ret = bdrv_snapshot_goto(bs, name); - if (ret < 0) { - error_report("Error %d while activating snapshot '%s' on '%s'", - ret, name, bdrv_get_device_name(bs)); - return ret; - } - } + ret = bdrv_all_goto_snapshot(name, &bs); + if (ret < 0) { + error_report("Error %d while activating snapshot '%s' on '%s'", + ret, name, bdrv_get_device_name(bs)); + return ret; } /* restore the VM state */ @@ -1483,9 +2142,12 @@ int load_vmstate(const char *name) qemu_system_reset(VMRESET_SILENT); migration_incoming_state_new(f); - ret = qemu_loadvm_state(f); + aio_context_acquire(aio_context); + ret = qemu_loadvm_state(f); qemu_fclose(f); + aio_context_release(aio_context); + migration_incoming_state_destroy(); if (ret < 0) { error_report("Error %d while loading VM state", ret); @@ -1501,43 +2163,33 @@ void hmp_delvm(Monitor *mon, const QDict *qdict) Error *err; const char *name = qdict_get_str(qdict, "name"); - if (!find_vmstate_bs()) { - monitor_printf(mon, "No block device supports snapshots\n"); - return; - } - - bs = NULL; - while ((bs = bdrv_next(bs))) { - if (bdrv_can_snapshot(bs)) { - err = NULL; - bdrv_snapshot_delete_by_id_or_name(bs, name, &err); - if (err) { - monitor_printf(mon, - "Error while deleting snapshot on device '%s':" - " %s\n", - bdrv_get_device_name(bs), - error_get_pretty(err)); - error_free(err); - } - } + if (bdrv_all_delete_snapshot(name, &bs, &err) < 0) { + error_reportf_err(err, + "Error while deleting snapshot on device '%s': ", + bdrv_get_device_name(bs)); } } void hmp_info_snapshots(Monitor *mon, const QDict *qdict) { BlockDriverState *bs, *bs1; - QEMUSnapshotInfo *sn_tab, *sn, s, *sn_info = &s; - int nb_sns, i, ret, available; + QEMUSnapshotInfo *sn_tab, *sn; + int nb_sns, i; int total; int *available_snapshots; + AioContext *aio_context; - bs = find_vmstate_bs(); + bs = bdrv_all_find_vmstate_bs(); if (!bs) { monitor_printf(mon, "No available block device supports snapshots\n"); return; } + aio_context = bdrv_get_aio_context(bs); + aio_context_acquire(aio_context); nb_sns = bdrv_snapshot_list(bs, &sn_tab); + aio_context_release(aio_context); + if (nb_sns < 0) { monitor_printf(mon, "bdrv_snapshot_list: error %d\n", nb_sns); return; @@ -1548,24 +2200,10 @@ void hmp_info_snapshots(Monitor *mon, const QDict *qdict) return; } - available_snapshots = g_malloc0(sizeof(int) * nb_sns); + available_snapshots = g_new0(int, nb_sns); total = 0; for (i = 0; i < nb_sns; i++) { - sn = &sn_tab[i]; - available = 1; - bs1 = NULL; - - while ((bs1 = bdrv_next(bs1))) { - if (bdrv_can_snapshot(bs1) && bs1 != bs) { - ret = bdrv_snapshot_find(bs1, sn_info, sn->id_str); - if (ret < 0) { - available = 0; - break; - } - } - } - - if (available) { + if (bdrv_all_find_snapshot(sn_tab[i].id_str, &bs1) == 0) { available_snapshots[total] = i; total++; } diff --git a/qemu/migration/tcp.c b/qemu/migration/tcp.c index ae891728e..e1fa7f8f1 100644 --- a/qemu/migration/tcp.c +++ b/qemu/migration/tcp.c @@ -13,7 +13,7 @@ * GNU GPL, version 2 or (at your option) any later version. */ -#include <string.h> +#include "qemu/osdep.h" #include "qemu-common.h" #include "qemu/error-report.h" @@ -39,11 +39,11 @@ static void tcp_wait_for_connect(int fd, Error *err, void *opaque) if (fd < 0) { DPRINTF("migrate connect error: %s\n", error_get_pretty(err)); - s->file = NULL; + s->to_dst_file = NULL; migrate_fd_error(s); } else { DPRINTF("migrate connect success\n"); - s->file = qemu_fopen_socket(fd, "wb"); + s->to_dst_file = qemu_fopen_socket(fd, "wb"); migrate_fd_connect(s); } } @@ -59,12 +59,11 @@ static void tcp_accept_incoming_migration(void *opaque) socklen_t addrlen = sizeof(addr); int s = (intptr_t)opaque; QEMUFile *f; - int c, err; + int c; do { c = qemu_accept(s, (struct sockaddr *)&addr, &addrlen); - err = socket_error(); - } while (c < 0 && err == EINTR); + } while (c < 0 && errno == EINTR); qemu_set_fd_handler(s, NULL, NULL, NULL); closesocket(s); @@ -72,7 +71,7 @@ static void tcp_accept_incoming_migration(void *opaque) if (c < 0) { error_report("could not accept migration connection (%s)", - strerror(err)); + strerror(errno)); return; } diff --git a/qemu/migration/unix.c b/qemu/migration/unix.c index b591813eb..d9aac36b9 100644 --- a/qemu/migration/unix.c +++ b/qemu/migration/unix.c @@ -13,7 +13,7 @@ * GNU GPL, version 2 or (at your option) any later version. */ -#include <string.h> +#include "qemu/osdep.h" #include "qemu-common.h" #include "qemu/error-report.h" @@ -39,11 +39,11 @@ static void unix_wait_for_connect(int fd, Error *err, void *opaque) if (fd < 0) { DPRINTF("migrate connect error: %s\n", error_get_pretty(err)); - s->file = NULL; + s->to_dst_file = NULL; migrate_fd_error(s); } else { DPRINTF("migrate connect success\n"); - s->file = qemu_fopen_socket(fd, "wb"); + s->to_dst_file = qemu_fopen_socket(fd, "wb"); migrate_fd_connect(s); } } diff --git a/qemu/migration/vmstate.c b/qemu/migration/vmstate.c index e8ccf22f6..bf3d5db30 100644 --- a/qemu/migration/vmstate.c +++ b/qemu/migration/vmstate.c @@ -1,3 +1,4 @@ +#include "qemu/osdep.h" #include "qemu-common.h" #include "migration/migration.h" #include "migration/qemu-file.h" @@ -28,6 +29,10 @@ static int vmstate_n_elems(void *opaque, VMStateField *field) n_elems = *(uint8_t *)(opaque+field->num_offset); } + if (field->flags & VMS_MULTIPLY_ELEMENTS) { + n_elems *= field->num; + } + return n_elems; } @@ -794,6 +799,29 @@ const VMStateInfo vmstate_info_float64 = { .put = put_float64, }; +/* CPU_DoubleU type */ + +static int get_cpudouble(QEMUFile *f, void *pv, size_t size) +{ + CPU_DoubleU *v = pv; + qemu_get_be32s(f, &v->l.upper); + qemu_get_be32s(f, &v->l.lower); + return 0; +} + +static void put_cpudouble(QEMUFile *f, void *pv, size_t size) +{ + CPU_DoubleU *v = pv; + qemu_put_be32s(f, &v->l.upper); + qemu_put_be32s(f, &v->l.lower); +} + +const VMStateInfo vmstate_info_cpudouble = { + .name = "CPU_Double_U", + .get = get_cpudouble, + .put = put_cpudouble, +}; + /* uint8_t buffers */ static int get_buffer(QEMUFile *f, void *pv, size_t size) diff --git a/qemu/migration/xbzrle.c b/qemu/migration/xbzrle.c index 8e220bf25..c85833925 100644 --- a/qemu/migration/xbzrle.c +++ b/qemu/migration/xbzrle.c @@ -10,7 +10,8 @@ * See the COPYING file in the top-level directory. * */ -#include "qemu-common.h" +#include "qemu/osdep.h" +#include "qemu/cutils.h" #include "include/migration/migration.h" /* |