REdis AOF文件结构分析

不念不忘少年蓝@ 2022-03-08 05:04 237阅读 0赞

REdis-4.0之前的AOF文件没有文件头，而从REdis-4.0开始AOF文件带有一个文件头，文件头格式和RDB文件头相同。

REdis-4.0版本，如果开启**aof-use-rdb-preamble**（值为yes，5.0默认为yes，4.0默认为no），则AOF文件内容由三部分组成：

1) 文件头（和RDB文件头格式相同）

2) RDB格式的二进制数据段

3) AOF格式的文本数据段

AOF格式的文本数据段和之前版本的保持相同，如下图所示：

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0FxdWVzdGVy_size_16_color_FFFFFF_t_70][]

从REdis-4.0开始，AOF有文件头，目的是支持同时加载RDB格式和AOF格式数据。AOF文件头和RDB文件头基本相同，但RDB文件头多了三个字段。

先看AOF和RDB通用部分的文件头内容：

1) 头5字节固定为REDIS

2) 第6~9共四字节为RDB版本号

3) 接下来为**redis-ver**和它的值，即redis版本

4) 接着**redis-bits**和它的值，即redis的位数，值为32或64

5) 接着为**ctime**和它的值，值为当前时间戳

6) 接着为**used-mem**和它的值

7) 最后是**aof-preamble**和它的值，值为0或1，1表示RDB有效。

RDB的文件头和AOF的文件基本相同，但RDB文件头在aof-preamble之前多了如下三项：

1) repl-stream-db

2) repl-id

3) repl-offset

如果配置项appendonly值为no，则不会加载AOF文件，而是直接加载RDB文件，因为RDB文件保存了复制信息，所以进程重启可增量复制。而如果置项appendonly值为yes，因为AOF文件头不包含复制信息，所以只能全量复制。

相关代码：

<table style="width:426.1pt;"> 
 <tbody> 
 <tr> 
 <td style="vertical-align:top;width:426.1pt;"> /*&nbsp;Replay&nbsp;the&nbsp;append&nbsp;log&nbsp;file.&nbsp; &nbsp;*&nbsp;On&nbsp;success&nbsp;C_OK&nbsp;is&nbsp;returned.&nbsp;On&nbsp;non&nbsp;fatal &nbsp;*&nbsp;error&nbsp;(the&nbsp;append&nbsp;only&nbsp;file&nbsp;is&nbsp;zero-length) &nbsp;*&nbsp;C_ERR&nbsp;is&nbsp;returned.&nbsp;On &nbsp;*&nbsp;fatal&nbsp;error&nbsp;an&nbsp;error&nbsp;message&nbsp;is&nbsp;logged &nbsp;*&nbsp;and&nbsp;the&nbsp;program&nbsp;exists.&nbsp;*/ int&nbsp;loadAppendOnlyFile(char&nbsp;*filename)&nbsp;//&nbsp;aof.c:673 { &nbsp;&nbsp;/*&nbsp;Check&nbsp;if&nbsp;this&nbsp;AOF&nbsp;file&nbsp;has&nbsp;an&nbsp;RDB&nbsp;preamble.&nbsp;In&nbsp;that&nbsp;case&nbsp;we&nbsp;need&nbsp;to &nbsp;&nbsp;&nbsp;*&nbsp;load&nbsp;the&nbsp;RDB&nbsp;file&nbsp;and&nbsp;later&nbsp;continue&nbsp;loading&nbsp;the&nbsp;AOF&nbsp;tail.&nbsp;*/ &nbsp;&nbsp;char&nbsp;sig[5];&nbsp;/*&nbsp;"REDIS"&nbsp;*/ &nbsp;&nbsp;if&nbsp;(fread(sig,1,5,fp)&nbsp;!=&nbsp;5&nbsp;||&nbsp;memcmp(sig,"REDIS",5)&nbsp;!=&nbsp;0)&nbsp;//&nbsp;aof.c:707 &nbsp;&nbsp;{ &nbsp;&nbsp;&nbsp;&nbsp;/*&nbsp;No&nbsp;RDB&nbsp;preamble,&nbsp;seek&nbsp;back&nbsp;at&nbsp;0&nbsp;offset.&nbsp;*/ &nbsp;&nbsp;&nbsp;&nbsp;if&nbsp;(fseek(fp,0,SEEK_SET)&nbsp;==&nbsp;-1)&nbsp;goto&nbsp;readerr; &nbsp;&nbsp;}&nbsp;else&nbsp;{ &nbsp;&nbsp;&nbsp;&nbsp;/*&nbsp;RDB&nbsp;preamble.&nbsp;Pass&nbsp;loading&nbsp;the&nbsp;RDB&nbsp;functions.&nbsp;*/ &nbsp;&nbsp;&nbsp;&nbsp;if&nbsp;(rdbLoadRio(&amp;rdb,NULL,1)&nbsp;!=&nbsp;C_OK)&nbsp;{&nbsp;//&nbsp;aof.c:717 &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;。。。。。。 &nbsp;&nbsp;&nbsp;&nbsp;} &nbsp;&nbsp;} } &nbsp; /*&nbsp;Load&nbsp;an&nbsp;RDB&nbsp;file&nbsp;from&nbsp;the&nbsp;rio&nbsp;stream&nbsp;'rdb'.&nbsp; &nbsp;*&nbsp;On&nbsp;success&nbsp;C_OK&nbsp;is&nbsp;returned, &nbsp;*&nbsp;otherwise&nbsp;C_ERR&nbsp;is&nbsp;returned&nbsp;and&nbsp;'errno'&nbsp;is&nbsp;set&nbsp;accordingly.&nbsp;*/ int&nbsp;rdbLoadRio(rio&nbsp;*rdb,&nbsp;rdbSaveInfo&nbsp;*rsi,&nbsp;int&nbsp;loading_aof)&nbsp;//&nbsp;rdb.c:1850 { &nbsp;&nbsp;。。。。。。 } &nbsp; /*&nbsp;Save&nbsp;a&nbsp;few&nbsp;default&nbsp;AUX&nbsp;fields&nbsp;with&nbsp;information&nbsp;about&nbsp;the&nbsp;RDB&nbsp;generated.&nbsp;*/ int&nbsp;rdbSaveInfoAuxFields(rio&nbsp;*rdb,&nbsp;int&nbsp;flags,&nbsp;rdbSaveInfo&nbsp;*rsi)&nbsp;//&nbsp;rdb.c:1071 { &nbsp;&nbsp;&nbsp;&nbsp;int&nbsp;redis_bits&nbsp;=&nbsp;(sizeof(void*)&nbsp;==&nbsp;8)&nbsp;?&nbsp;64&nbsp;:&nbsp;32; &nbsp;&nbsp;&nbsp;&nbsp;int&nbsp;aof_preamble&nbsp;=&nbsp;(flags&nbsp;&amp;&nbsp;RDB_SAVE_AOF_PREAMBLE)&nbsp;!=&nbsp;0; &nbsp; &nbsp;&nbsp;&nbsp;&nbsp;/*&nbsp;Add&nbsp;a&nbsp;few&nbsp;fields&nbsp;about&nbsp;the&nbsp;state&nbsp;when&nbsp;the&nbsp;RDB&nbsp;was&nbsp;created.&nbsp;*/ &nbsp;&nbsp;&nbsp;&nbsp;if&nbsp;(rdbSaveAuxFieldStrStr(rdb,"redis-ver",REDIS_VERSION)&nbsp;==&nbsp;-1)&nbsp;return&nbsp;-1; &nbsp;&nbsp;&nbsp;&nbsp;if&nbsp;(rdbSaveAuxFieldStrInt(rdb,"redis-bits",redis_bits)&nbsp;==&nbsp;-1)&nbsp;return&nbsp;-1; &nbsp;&nbsp;&nbsp;&nbsp;if&nbsp;(rdbSaveAuxFieldStrInt(rdb,"ctime",time(NULL))&nbsp;==&nbsp;-1)&nbsp;return&nbsp;-1; &nbsp;&nbsp;&nbsp;&nbsp;if&nbsp;(rdbSaveAuxFieldStrInt(rdb,"used-mem",zmalloc_used_memory())&nbsp;==&nbsp;-1)&nbsp;return&nbsp;-1; &nbsp; &nbsp;&nbsp;&nbsp;&nbsp;/*&nbsp;Handle&nbsp;saving&nbsp;options&nbsp;that&nbsp;generate&nbsp;aux&nbsp;fields.&nbsp;*/ &nbsp;&nbsp;&nbsp;&nbsp;//&nbsp;如果是AOF，传入的rsi为NULL， &nbsp;&nbsp;&nbsp;&nbsp;//&nbsp;因此不会记录以下三项辅助数据 &nbsp;&nbsp;&nbsp;&nbsp;//&nbsp;可参见函数rewriteAppendOnlyFile（aof.c:1344）的实现 &nbsp;&nbsp;&nbsp;&nbsp;if&nbsp;(rsi)&nbsp;{&nbsp;//&nbsp;rdb.c:1082 &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;if&nbsp;(rdbSaveAuxFieldStrInt(rdb,"repl-stream-db",rsi-&gt;repl_stream_db) &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;==&nbsp;-1)&nbsp;return&nbsp;-1; &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;if&nbsp;(rdbSaveAuxFieldStrStr(rdb,"repl-id",server.replid) &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;==&nbsp;-1)&nbsp;return&nbsp;-1; &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;if&nbsp;(rdbSaveAuxFieldStrInt(rdb,"repl-offset",server.master_repl_offset) &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;==&nbsp;-1)&nbsp;return&nbsp;-1; &nbsp;&nbsp;&nbsp;&nbsp;} &nbsp;&nbsp;&nbsp;&nbsp;if&nbsp;(rdbSaveAuxFieldStrInt(rdb,"aof-preamble",aof_preamble)&nbsp;==&nbsp;-1)&nbsp;return&nbsp;-1; &nbsp;&nbsp;&nbsp;&nbsp;return&nbsp;1; } &nbsp; /*&nbsp;Produces&nbsp;a&nbsp;dump&nbsp;of&nbsp;the&nbsp;database&nbsp;in&nbsp;RDB&nbsp;format&nbsp;sending&nbsp;it&nbsp;to&nbsp;the&nbsp;specified &nbsp;*&nbsp;Redis&nbsp;I/O&nbsp;channel.&nbsp;On&nbsp;success&nbsp;C_OK&nbsp;is&nbsp;returned,&nbsp;otherwise&nbsp;C_ERR &nbsp;*&nbsp;is&nbsp;returned&nbsp;and&nbsp;part&nbsp;of&nbsp;the&nbsp;output,&nbsp;or&nbsp;all&nbsp;the&nbsp;output,&nbsp;can&nbsp;be &nbsp;*&nbsp;missing&nbsp;because&nbsp;of&nbsp;I/O&nbsp;errors. &nbsp;* &nbsp;*&nbsp;When&nbsp;the&nbsp;function&nbsp;returns&nbsp;C_ERR&nbsp;and&nbsp;if&nbsp;'error'&nbsp;is&nbsp;not&nbsp;NULL,&nbsp;the &nbsp;*&nbsp;integer&nbsp;pointed&nbsp;by&nbsp;'error'&nbsp;is&nbsp;set&nbsp;to&nbsp;the&nbsp;value&nbsp;of&nbsp;errno&nbsp;just&nbsp;after&nbsp;the&nbsp;I/O &nbsp;*&nbsp;error.&nbsp;*/ int&nbsp;rdbSaveRio(rio&nbsp;*rdb,&nbsp;int&nbsp;*error,&nbsp;int&nbsp;flags,&nbsp;rdbSaveInfo&nbsp;*rsi)&nbsp;//&nbsp;rdb.c:1102 { &nbsp;&nbsp;&nbsp;if&nbsp;(rdbSaveInfoAuxFields(rdb,flags,rsi)&nbsp;==&nbsp;-1)&nbsp;goto&nbsp;werr;&nbsp;//&nbsp;rdb.c:1114 } &nbsp; /*&nbsp;Write&nbsp;a&nbsp;sequence&nbsp;of&nbsp;commands&nbsp;able&nbsp;to&nbsp;fully&nbsp;rebuild&nbsp;the&nbsp;dataset&nbsp;into &nbsp;*&nbsp;"filename".&nbsp;Used&nbsp;both&nbsp;by&nbsp;REWRITEAOF&nbsp;and&nbsp;BGREWRITEAOF. &nbsp;* &nbsp;*&nbsp;In&nbsp;order&nbsp;to&nbsp;minimize&nbsp;the&nbsp;number&nbsp;of&nbsp;commands&nbsp;needed&nbsp;in&nbsp;the&nbsp;rewritten &nbsp;*&nbsp;log&nbsp;Redis&nbsp;uses&nbsp;variadic&nbsp;commands&nbsp;when&nbsp;possible,&nbsp;such&nbsp;as&nbsp;RPUSH,&nbsp;SADD &nbsp;*&nbsp;and&nbsp;ZADD.&nbsp;However&nbsp;at&nbsp;max&nbsp;AOF_REWRITE_ITEMS_PER_CMD&nbsp;items&nbsp;per&nbsp;time &nbsp;*&nbsp;are&nbsp;inserted&nbsp;using&nbsp;a&nbsp;single&nbsp;command.&nbsp;*/ int&nbsp;rewriteAppendOnlyFile(char&nbsp;*filename)&nbsp;//&nbsp;aof.c:1344 { &nbsp;&nbsp;//&nbsp;对于AOF，传入的第三个参数rsi值为NULL， &nbsp;&nbsp;//&nbsp;因此不会记录复制信息，这样进程重启需要全量复制。 &nbsp;&nbsp;if&nbsp;(rdbSaveRio(&amp;aof,&amp;error,RDB_SAVE_AOF_PREAMBLE,NULL)&nbsp;==&nbsp;C_ERR)&nbsp;{&nbsp;//&nbsp;aof.c:1367 } </td> 
 </tr> 
 </tbody> 
</table>

启动时调用顺序：

<table style="width:426.1pt;"> 
 <tbody> 
 <tr> 
 <td style="vertical-align:top;width:426.1pt;"> &nbsp;&nbsp;&nbsp;main(server.c:4003) -&gt;&nbsp;loadDataFromDisk(server.c:3850) -&gt;&nbsp;loadAppendOnlyFile(aof.c:673，当配置appendonly值为yes时) -&gt;&nbsp;rdbLoadRio(rdb.c:1850) </td> 
 </tr> 
 </tbody> 
</table>

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L0FxdWVzdGVy_size_16_color_FFFFFF_t_70]: /images/20220308/ed781d3821644746aca4fd1697f5a04a.png