libvirt savevm 命令存储内存文件分析

叁歲伎倆 2022-09-25 07:20 91阅读 0赞

在libvirt 使用save命令对虚拟机进行存储后，将libvirt的信息，虚拟机的内存，cpu以及其他设备的信息都存储了起来。

先把文件格式大致的画出来。

libvirt 存储文件信息
            +-------------+-----+-----+-----+-----+
            |Libvirt-Magic|版本 | xml |运行 |是否 |
            |             |     |长度 |状态 |压缩 |
            |8字节        |4字节|4字节|4字节|4字节|
            +-------------+-----+-----+-----+-----+
            |  保留字段（60字节）                 |
            +-------------------------------------+
            | xml 详细信息（占用xml长度个字节)    |
            +-------------------------------------+
    qemu文件开头存储信息。
            +-----+-----+--------+
            |QEMV |版本 |设备信息|
            |     |     |        |
            |4字节|4字节|        |
            +-----+-----+--------+
    
    qemu中savevm_handlers中的设备信息，可能有多个设备。
    
            设备循环，一个设备信息如下。
            +-------+-------+------+------+------+------+--------+------+
            |section|section|设备名|设备名|实例id|版本id|子设备信|循环  |
            |start  | id    | 长度 |      |      |      | 息     |设备  |
            |1字节  | 4字节 |1字节 |n字节 |4字节 |4字节 |        |      |
            +-------+-------+------+------+------+------+--------+------+
    
            子设备信息存储 这里以ram设备 为例
                    +--------------------------------------+
                    |共占用内存字节数 8字节                |
                    +------+------+------+--------+--------+
                    |设备名|设备名|设备内|循环前面|ram存储 |
                    | 长度 |      |存长度|信息    |结束标记|
                    |1字节 |n字节 |8字节 |        |8字节   |
                    +------+------+------+--------+--------+
    
    
            开始存储设备详细信息
            +-------+-------+------+------+
            |section|section|设备详|循环  |
            |part   | id    |细信息|设备  |
            |1字节  |4字节  |n字节 |      |
            +-------+-------+------+------+
    
            处理存储结束
            +-------+-------+------+------+
            |section|section|设备结|循环  |
            |end   | id     |束信息|设备  |
            |1字节  |4字节  |n字节 |      |
            +-------+-------+------+------+
            存储设备状态
            +-------+-------+------+------+
            |section|section|设备状|循环  |
            |full   | id    |态信息|设备  |
            |1字节  |4字节  |n字节 |      |
            +-------+-------+------+------+
    
            +-----+
            |结束 |
            |标记 |
            |1字节|
            +-----+

这里就对这一文件进行分析。

首先将存储文件的2进制信息发出来。

00000000  4c 69 62 76 69 72 74 51  65 6d 75 64 53 61 76 65  |LibvirtQemudSave|
    00000010  02 00 00 00 07 12 00 00  01 00 00 00 00 00 00 00  |................|
    00000020  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
    00000030  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
    00000040  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
    00000050  00 00 00 00 00 00 00 00  00 00 00 00 3c 64 6f 6d  |............<dom|

这是存储文件中的一部分信息，结合代码来看一下这个文件究竟是什么

#define QEMU_SAVE_MAGIC   "LibvirtQemudSave"
    
    typedef enum {
        QEMU_SAVE_FORMAT_RAW = 0,
        QEMU_SAVE_FORMAT_GZIP = 1,
        QEMU_SAVE_FORMAT_BZIP2 = 2,
        /*
         * Deprecated by xz and never used as part of a release
         * QEMU_SAVE_FORMAT_LZMA
         */
        QEMU_SAVE_FORMAT_XZ = 3,
        QEMU_SAVE_FORMAT_LZOP = 4,
        /* Note: add new members only at the end.
           These values are used in the on-disk format.
           Do not change or re-use numbers. */
    
        QEMU_SAVE_FORMAT_LAST
    } virQEMUSaveFormat;
    
    struct _virQEMUSaveHeader {
        char magic[sizeof(QEMU_SAVE_MAGIC)-1]; //libvirt 存储内存文件的魔数LibvirtQemudSave 16个字节
        uint32_t version; //版本信息 上述例子是02
        uint32_t xml_len; //存储的xml配置信息长度0x1207
        uint32_t was_running;//01 表示恢复时运行，00表示恢复时处于暂停状态
        uint32_t compressed;//表示压缩方式，0表示raw，其他格式根据virQEMUSaveFormat定义。
        uint32_t unused[15];
    };

首先，文件最开始存储了\_virQEMUSaveHeader 文件的头信息。  
头信息总共占用了0x5b个字节。

忽略libvirt存储配置的一部分，然后直接到达qemu开始存储文件的位置。  
根据上面头文件信息，可以得知，qemu存储文件的起始位置为0x1207+0x5b=0x1262

可以看到从0x1263开始就是qemu存储的文件了。接下来就分析qemu存储文件格式。

这是存储qemu虚拟机状态的函数，这里可以看按到大致流程

static int qemu_savevm_state(QEMUFile *f)
    {
        int ret;
        MigrationParams params = {
            .blk = 0,
            .shared = 0
        };
    
        if (qemu_savevm_state_blocked(NULL)) {
            return -EINVAL;
        }
    
        qemu_mutex_unlock_iothread();
        qemu_savevm_state_begin(f, &params);//存储虚拟机状态主要是有数据的设备存储。
        qemu_mutex_lock_iothread();
    
        while (qemu_file_get_error(f) == 0) {
            if (qemu_savevm_state_iterate(f) > 0) { //存储具体信息。
                break;
            }
        }
    
        ret = qemu_file_get_error(f);
        if (ret == 0) {
            qemu_savevm_state_complete(f);//进行有是护具的存储设备结尾，同时对有状态的设备进行存储。
            ret = qemu_file_get_error(f);
        }
        if (ret != 0) {
            qemu_savevm_state_cancel();
        }
        return ret;
    }

#define QEMU_VM_FILE_MAGIC           0x5145564d
    #define QEMU_VM_FILE_VERSION         0x00000003
    
    #define QEMU_VM_SECTION_START        0x01
    
    void qemu_savevm_state_begin(QEMUFile *f,
                                 const MigrationParams *params)
    {
        SaveStateEntry *se;
        int ret;
    
        trace_savevm_state_begin();
        QTAILQ_FOREACH(se, &savevm_handlers, entry) {
            if (!se->ops || !se->ops->set_params) {
                continue;
            }
            se->ops->set_params(params, se->opaque);
        }
    
        qemu_put_be32(f, QEMU_VM_FILE_MAGIC);//qemu文件起始QEVM 4字节
        qemu_put_be32(f, QEMU_VM_FILE_VERSION);//qemu文件的版本 4字节
    
        QTAILQ_FOREACH(se, &savevm_handlers, entry) {
            int len;
    
            if (!se->ops || !se->ops->save_live_setup) {
                continue;
            }
            if (se->ops && se->ops->is_active) {
                if (!se->ops->is_active(se->opaque)) {
                    continue;
                }
            }
            /* Section type */
            qemu_put_byte(f, QEMU_VM_SECTION_START);//设备起始·1字节
            qemu_put_be32(f, se->section_id);//设备的section_id 4字节
    
            /* ID string */
            len = strlen(se->idstr);
            qemu_put_byte(f, len);//设备字符串长度1字节
            qemu_put_buffer(f, (uint8_t *)se->idstr, len);//设备字符
    
            qemu_put_be32(f, se->instance_id);//实例id 4字节
            qemu_put_be32(f, se->version_id);//版本id 4字节
    
            ret = se->ops->save_live_setup(f, se->opaque);//进入具体的设备存储信息，这里首先是ram
            if (ret < 0) {
                qemu_file_set_error(f, ret);
                break;
            }
        }
    }

首先是qemu中的魔数（4字节），qemu文件版本（4字节），  
然后就是需要save的设备的信息，首先是section start的标记（1字节）  
然后是section\_id。（4字节）设备名字长度1字节，设备名字，然后实例id（4字节）（版本id）4字节

#define RAM_SAVE_FLAG_EOS      0x10
    
    static int ram_save_setup(QEMUFile *f, void *opaque)
    {
        RAMBlock *block;
        int64_t ram_bitmap_pages; /* Size of bitmap in pages, including gaps */
    
        mig_throttle_on = false;
        dirty_rate_high_cnt = 0;
        bitmap_sync_count = 0;
        migration_bitmap_sync_init();
    
        if (migrate_use_xbzrle()) {
            XBZRLE_cache_lock();
            XBZRLE.cache = cache_init(migrate_xbzrle_cache_size() /
                                      TARGET_PAGE_SIZE,
                                      TARGET_PAGE_SIZE);
            if (!XBZRLE.cache) {
                XBZRLE_cache_unlock();
                error_report("Error creating cache");
                return -1;
            }
            XBZRLE_cache_unlock();
    
            /* We prefer not to abort if there is no memory */
            XBZRLE.encoded_buf = g_try_malloc0(TARGET_PAGE_SIZE);
            if (!XBZRLE.encoded_buf) {
                error_report("Error allocating encoded_buf");
                return -1;
            }
    
            XBZRLE.current_buf = g_try_malloc(TARGET_PAGE_SIZE);
            if (!XBZRLE.current_buf) {
                error_report("Error allocating current_buf");
                g_free(XBZRLE.encoded_buf);
                XBZRLE.encoded_buf = NULL;
                return -1;
            }
    
            acct_clear();
        }
    
        qemu_mutex_lock_iothread();
        qemu_mutex_lock_ramlist();
        bytes_transferred = 0;
        reset_ram_globals();
    
        ram_bitmap_pages = last_ram_offset() >> TARGET_PAGE_BITS;
        migration_bitmap = bitmap_new(ram_bitmap_pages);
        bitmap_set(migration_bitmap, 0, ram_bitmap_pages);
    
        /*
         * Count the total number of pages used by ram blocks not including any
         * gaps due to alignment or unplugs.
         */
        migration_dirty_pages = 0;
        QTAILQ_FOREACH(block, &ram_list.blocks, next) {
            uint64_t block_pages;
    
            block_pages = block->length >> TARGET_PAGE_BITS;
            migration_dirty_pages += block_pages;
        }
    
        memory_global_dirty_log_start();
        migration_bitmap_sync();
        qemu_mutex_unlock_iothread();
    
        qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);//存了8位用来表示内存站了多少个字节 
    
        QTAILQ_FOREACH(block, &ram_list.blocks, next) {
            qemu_put_byte(f, strlen(block->idstr));//1个字节长度
            qemu_put_buffer(f, (uint8_t *)block->idstr, strlen(block->idstr));//设备名字
            qemu_put_be64(f, block->length);//设备占用长度。//8字节
        }
    
        qemu_mutex_unlock_ramlist();
    
        ram_control_before_iterate(f, RAM_CONTROL_SETUP);
        ram_control_after_iterate(f, RAM_CONTROL_SETUP);
    
        qemu_put_be64(f, RAM_SAVE_FLAG_EOS); //存储结尾8字节
    
        return 0;
    }

0x127b开始，就是ram的save函数进行记录的信息。  
首先是存储了8字节 ram总共占用了多少字节 这里是0x808d1004。这里除了guest的内存还有其他设备的寄存器也在这里保存。  
接下来是pc.ram 内存，我们的虚拟机使用了2G内存，使用了0x80000000。  
vga.vram 显卡的显存 8M 使用了0x800000  
pc.bios bios寄存器 256k 0x40000  
virtio-net-pci寄存器 256k 0x40000  
acpi表 128k 0x20000  
cirrus\_vga显存 64k 0x010000  
acpi的table-loader 4k 0x1000  
存储结尾标记 RAM\_SAVE\_FLAG\_EOS  
这里到达0x1351结束，接下来就是要存储的设备进行一一存储了。

#define QEMU_VM_SECTION_PART         0x02
    int qemu_savevm_state_iterate(QEMUFile *f)
    {
        SaveStateEntry *se;
        int ret = 1;
    
        trace_savevm_state_iterate();
        QTAILQ_FOREACH(se, &savevm_handlers, entry) {
            if (!se->ops || !se->ops->save_live_iterate) {
                continue;
            }
            if (se->ops && se->ops->is_active) {
                if (!se->ops->is_active(se->opaque)) {
                    continue;
                }
            }
            if (qemu_file_rate_limit(f)) {
                return 0;
            }
            trace_savevm_section_start(se->idstr, se->section_id);
            /* Section type */
            qemu_put_byte(f, QEMU_VM_SECTION_PART);// 开始标记1字节
            qemu_put_be32(f, se->section_id);//section id
    
            ret = se->ops->save_live_iterate(f, se->opaque);
            trace_savevm_section_end(se->idstr, se->section_id);
    
            if (ret < 0) {
                qemu_file_set_error(f, ret);
            }
            if (ret <= 0) {
                /* Do not proceed to the next vmstate before this one reported
                   completion of the current stage. This serializes the migration
                   and reduces the probability that a faster changing state is
                   synchronized over and over again. */
                break;
            }
        }
        return ret;
    }

记录了设备的其实信息后，就会调用设备自身来进行数据存储到达0x1356

static int ram_save_iterate(QEMUFile *f, void *opaque)
    {
        int ret;
        int i;
        int64_t t0;
        int total_sent = 0;
    
        qemu_mutex_lock_ramlist();
    
        if (ram_list.version != last_version) {
            reset_ram_globals();
        }
    
        ram_control_before_iterate(f, RAM_CONTROL_ROUND);
    
        t0 = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
        i = 0;
        while ((ret = qemu_file_rate_limit(f)) == 0) {
            int bytes_sent;
    
            bytes_sent = ram_find_and_save_block(f, false);//开始存储
            /* no more blocks to sent */
            if (bytes_sent == 0) {
                break;
            }
            total_sent += bytes_sent;
            acct_info.iterations++;
            check_guest_throttling();
            /* we want to check in the 1st loop, just in case it was the 1st time
               and we had to sync the dirty bitmap.
               qemu_get_clock_ns() is a bit expensive, so we only check each some
               iterations
            */
            if ((i & 63) == 0) {
                uint64_t t1 = (qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - t0) / 1000000;
                if (t1 > MAX_WAIT) {
                    DPRINTF("big wait: %" PRIu64 " milliseconds, %d iterations\n",
                            t1, i);
                    break;
                }
            }
            i++;
        }
    
        qemu_mutex_unlock_ramlist();
    
        /*
         * Must occur before EOS (or any QEMUFile operation)
         * because of RDMA protocol.
         */
        ram_control_after_iterate(f, RAM_CONTROL_ROUND);
    
        bytes_transferred += total_sent;
    
        /*
         * Do not count these 8 bytes into total_sent, so that we can
         * return 0 if no page had been dirtied.
         */
        qemu_put_be64(f, RAM_SAVE_FLAG_EOS);//8字节的结束标记
        bytes_transferred += 8;
    
        ret = qemu_file_get_error(f);
        if (ret < 0) {
            return ret;
        }
    
        return total_sent;
    }

在ram\_find\_and\_save\_block中 进行了实际的存储。

static int ram_find_and_save_block(QEMUFile *f, bool last_stage)
    {
        RAMBlock *block = last_seen_block;
        ram_addr_t offset = last_offset;
        bool complete_round = false;
        int bytes_sent = 0;
        MemoryRegion *mr;
    
        if (!block)
            block = QTAILQ_FIRST(&ram_list.blocks);
    
        while (true) {
            mr = block->mr;
            offset = migration_bitmap_find_and_reset_dirty(mr, offset);//从dirty页中拿出一页
            if (complete_round && block == last_seen_block &&
                offset >= last_offset) {
                break;
            }
            if (offset >= block->length) {
                offset = 0;
                block = QTAILQ_NEXT(block, next);
                if (!block) {
                    block = QTAILQ_FIRST(&ram_list.blocks);
                    complete_round = true;
                    ram_bulk_stage = false;
                }
            } else {
                bytes_sent = ram_save_page(f, block, offset, last_stage);//存储page
    
                /* if page is unmodified, continue to the next */
                if (bytes_sent > 0) {
                    last_sent_block = block;//标记上一次存储的设备
                    break;
                }
            }
        }
        last_seen_block = block;
        last_offset = offset;
    
        return bytes_sent;
    }

ram\_save\_page 继续执行存储page

#define RAM_SAVE_FLAG_CONTINUE 0x20
    
    static int ram_save_page(QEMUFile *f, RAMBlock* block, ram_addr_t offset,
                             bool last_stage)
    {
        int bytes_sent;
        int cont;
        ram_addr_t current_addr;
        MemoryRegion *mr = block->mr;
        uint8_t *p;
        int ret;
        bool send_async = true;
    
        cont = (block == last_sent_block) ? RAM_SAVE_FLAG_CONTINUE : 0;//这里进行了比较如果新的block，这里标记cont
    
        p = memory_region_get_ram_ptr(mr) + offset;//获取要存储页的地址
    
    
        ...
         } else if (is_zero_range(p, TARGET_PAGE_SIZE)) {//如果页面全0就会采用压缩方式。
            acct_info.dup_pages++;
            bytes_sent = save_block_hdr(f, block, offset, cont,
                                        RAM_SAVE_FLAG_COMPRESS);
            qemu_put_byte(f, 0);//并存入一个字节0
            bytes_sent++;
    
        ...
    
        /* XBZRLE overflow or normal page */
        if (bytes_sent == -1) {
            bytes_sent = save_block_hdr(f, block, offset, cont, RAM_SAVE_FLAG_PAGE);//存储页的偏移
            if (send_async) {
                qemu_put_buffer_async(f, p, TARGET_PAGE_SIZE);//实际存储数据异步
            } else {
                qemu_put_buffer(f, p, TARGET_PAGE_SIZE);//实际存储数据同步
            }
            bytes_sent += TARGET_PAGE_SIZE;
            acct_info.norm_pages++;
        }
    
        XBZRLE_cache_unlock();
    
        return bytes_sent;
    }

这里来看一下如何存储页面偏移。

#define RAM_SAVE_FLAG_PAGE     0x08
    
    static size_t save_block_hdr(QEMUFile *f, RAMBlock *block, ram_addr_t offset,
                                 int cont, int flag)
    {
        size_t size;
    
        qemu_put_be64(f, offset | cont | flag); //一个8字节的标记位
        size = 8;
    
        if (!cont) {
       //如果是第一次存储，会存储设备的名字。
            qemu_put_byte(f, strlen(block->idstr));
            qemu_put_buffer(f, (uint8_t *)block->idstr,
                            strlen(block->idstr));
            size += 1 + strlen(block->idstr);
        }
        return size;
    }

这样就可以看到 0x1356后接下来是存储8个字节page的标记位  
0x00000008 就是offset=0 cont=0 flag=RAM\_SAVE\_FLAG\_PAGE,因为是第一次，所以存储了  
设备的名字。  
从0x1366开始就是存储4k的页面，  
在ram\_find\_and\_save\_block中 会循环执行ram\_save\_page函数，直到全部存储完所有内存。  
因此存储模式就是 页偏移+页内容  
所以接下来直接查看0x2366的信息。

00002360  00 00 00 00 00 00 00 00  00 00 00 00 10 22 00 00  |............."..|
    00002370  00 00 00 00 00 20 22 00  00 00 00 00 00 00 30 22  |..... ".......0"|
    00002380  00 00 00 00 00 00 00 40  22 00 00 00 00 00 00 00  |.......@".......|
    00002390  50 22 00 00 00 00 00 00  00 60 28 00 00 00 00 00  |P".......`(.....|
    000023a0  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
    000023b0  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
    000023c0  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
    000023d0  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
    000023e0  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
    000023f0  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|

可以看到第二个页的标记是0x1022，  
由于第二个页面是全0页，所以做了压缩处理。第二个页offset=0x10000，cont=0x20，flag=RAM\_SAVE\_FLAG\_COMPRESS。  
然后存入1字节0  
紧接着0x236e是第三个页面是0x2022 后面的就不过多说明了。  
pc.ram 是2G，当pc.ram存储完后，会接着存储其他设备。

00939ec0  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
    00939ed0  00 00 00 02 08 76 67 61  2e 76 72 61 6d 00 00 00  |.....vga.vram...|
    00939ee0  00 00 00 00 10 22 00 00  00 00 00 00 00 20 22 00  |....."....... ".|
    00939ef0  00 00 00 00 00 00 30 22  00 00 00 00 00 00 00 40  |......0".......@|
    00939f00  22 00 00 00 00 00 00 00  50 22 00 00 00 00 00 00  |".......P"......|

可以看到从0x00939cb开始就是存储vag.ram的信息。由于第一个页就是0页。值是0x00000002.  
所以标记offset=0，cont=0，flag=RAM\_SAVE\_FLAG\_COMPRESS。由于第一次存储该设备，所以存入设备名字。

当这些设备全部存储完毕后，就是存储结尾。

这里从最后一个 设备开始查找，最后一个设备占用4k，所以从0x00a72d6d开始向后4k来查找。

00a73d50  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................| 
    00a73d60  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00  |................|
    00a73d70  00 00 00 00 00 00 10 03  00 00 00 02 00 00 00 00  |................|
    00a73d80  00 00 00 10 04 00 00 00  00 05 74 69 6d 65 72 00  |..........timer.|
    00a73d90  00 00 00 00 00 00 02 00  00 00 05 9f e1 15 85 00  |................|
    00a73da0  00 00 00 00 00 00 00 00  00 00 02 73 7e b6 7e 04  |...........s~.~.|
    00a73db0  00 00 00 03 0a 63 70 75  5f 63 6f 6d 6d 6f 6e 00  |.....cpu_common.|

从0x00a73d6e开始设备信息查找完成，然后存入ram结束信息RAM\_SAVE\_FLAG\_EOS 8字节 0x00000010 。  
从0x00a73d77开始 ，就是进行最后一次存储，此时会存储cpu信息。

#define QEMU_VM_SECTION_END          0x03
    #define QEMU_VM_SECTION_FULL         0x04
    #define QEMU_VM_EOF                  0x00
    
    void qemu_savevm_state_complete(QEMUFile *f)
    {       
        SaveStateEntry *se;
        int ret;
    
        trace_savevm_state_complete();
    
        cpu_synchronize_all_states();//同步此时的cpu状态。
    
        QTAILQ_FOREACH(se, &savevm_handlers, entry) {
            if (!se->ops || !se->ops->save_live_complete) {
                continue;
            }
            if (se->ops && se->ops->is_active) {
                if (!se->ops->is_active(se->opaque)) {
                    continue;
                }
            }
            trace_savevm_section_start(se->idstr, se->section_id);
            /* Section type */
            qemu_put_byte(f, QEMU_VM_SECTION_END);
            qemu_put_be32(f, se->section_id);
    
            ret = se->ops->save_live_complete(f, se->opaque);//调用设备的complete函数
            trace_savevm_section_end(se->idstr, se->section_id);
            if (ret < 0) {
                qemu_file_set_error(f, ret);
                return;
            }
        }
        //开始存储具有状态的设备。
        QTAILQ_FOREACH(se, &savevm_handlers, entry) {
            int len;
    
            if ((!se->ops || !se->ops->save_state) && !se->vmsd) {
                continue;
            }
            trace_savevm_section_start(se->idstr, se->section_id);
            /* Section type */
            qemu_put_byte(f, QEMU_VM_SECTION_FULL); //1字节section full
            qemu_put_be32(f, se->section_id); 
    
            /* ID string */
            len = strlen(se->idstr);
            qemu_put_byte(f, len);
            qemu_put_buffer(f, (uint8_t *)se->idstr, len);
    
            qemu_put_be32(f, se->instance_id);
            qemu_put_be32(f, se->version_id);
    
            vmstate_save(f, se);
            trace_savevm_section_end(se->idstr, se->section_id);
        }
    
        qemu_put_byte(f, QEMU_VM_EOF);
        qemu_fflush(f);
    }

static int ram_save_complete(QEMUFile *f, void *opaque)
    {
        qemu_mutex_lock_ramlist();
        migration_bitmap_sync();
    
        ram_control_before_iterate(f, RAM_CONTROL_FINISH);
    
        /* try transferring iterative blocks of memory */
        ram_control_before_iterate(f, RAM_CONTROL_FINISH);
    
        /* try transferring iterative blocks of memory */
    
        /* flush all remaining blocks regardless of rate limiting */
        while (true) {
            int bytes_sent;
    
            bytes_sent = ram_find_and_save_block(f, true);
            /* no more blocks to sent */
            if (bytes_sent == 0) {
                break;
            }
            bytes_transferred += bytes_sent;
        }
    
        ram_control_after_iterate(f, RAM_CONTROL_FINISH);
        migration_end();
    
        qemu_mutex_unlock_ramlist();
        qemu_put_be64(f, RAM_SAVE_FLAG_EOS);//存入8字节 ram结束标记。
    
        return 0;
    }

0x00a73d78 是03 是标记存储结束。sectionid是 0x00000002。然后存入表示RAM\_SAVE\_FLAG\_EOS 8字节0x00000010

从 0x00a73d83开始 存储QEMU\_VM\_SECTION\_FULL 1字节 0x04 然后sectionid 0x00000000  
然后是设备的名字，设备instance\_id和version\_id。然后使用vmstate\_save来存储信息。

从0x00a73d89开始是timer设备的信息。

static const VMStateDescription vmstate_timers = {
        .name = "timer",
        .version_id = 2,
        .minimum_version_id = 1,
        .fields = (VMStateField[]) {
            VMSTATE_INT64(cpu_ticks_offset, TimersState),
            VMSTATE_INT64(dummy, TimersState),
            VMSTATE_INT64_V(cpu_clock_offset, TimersState, 2),
            VMSTATE_END_OF_LIST()
        },
        .subsections = (VMStateSubsection[]) {
            {
                .vmsd = &icount_vmstate_timers,
                .needed = icount_state_needed,
            }, {
                /* empty */
            }
        }
    };
    
    void vmstate_save_state(QEMUFile *f, const VMStateDescription *vmsd,
                            void *opaque)
    {
        VMStateField *field = vmsd->fields;
    
        if (vmsd->pre_save) {    
            vmsd->pre_save(opaque);
        }   
        while (field->name) {
            if (!field->field_exists || 
                field->field_exists(opaque, vmsd->version_id)) {
                void *base_addr = vmstate_base_addr(opaque, field, false);
                int i, n_elems = vmstate_n_elems(opaque, field);
                int size = vmstate_size(opaque, field);//获取state的size
    
                for (i = 0; i < n_elems; i++) {
                    void *addr = base_addr + size * i;
    
                    if (field->flags & VMS_ARRAY_OF_POINTER) {
                        addr = *(void **)addr;
                    }
                    if (field->flags & VMS_STRUCT) {
                        vmstate_save_state(f, field->vmsd, addr);
                    } else {
                        field->info->put(f, addr, size);//这里对state进行存储
                    }
                }
            } else {
                if (field->flags & VMS_MUST_EXIST) {
                    fprintf(stderr, "Output state validation failed: %s/%s\n",
                            vmsd->name, field->name);
                    assert(!(field->flags & VMS_MUST_EXIST));
                }
            }
            field++;
        }
        vmstate_subsection_save(f, vmsd, opaque);
    }

timer设备的state有3个VMSTATE\_INT64组成，即24字节。  
从00a73d97到00a73dae用存储timer的state信息。

所以对于有状态设备来说，就在这里就进行存储。

后面的设备就不一一分析。

当存储完成后，存入QEMU\_VM\_EOF。