2023-11-02 monetdb-事务提交-wal日志写入-分析
摘要:
2023-11-02 monetdb-事务提交-日志写入-分析
场景:
- 向mdb中写入数据
- 查看在事务commit时, mdb所作的处理
- 其中最为关键是记录wal日志, 分析具体的处理
- wal日志可以理解为redo log, 在真正做数据写入磁盘前, 先写入redo log
- 目的是为了在数据写磁盘发生错误时, 可以用redo log中的日志做重写, 保证持久性
核心流程:
#0 log_constant (lg=0x7f0fe35e7130, type=1, val=0x7f138c7f497f, id=6680, offset=35, cnt=5) at /root/work/trunk/mysql-server-mysql-8.0.33/storage/monetdb/gdk/gdk_logger.c:2508
#1 0x00007f13d9955e0a in log_segment (tr=0x7f0fe37756c0, s=0x7f0fe38a0950, id=6680) at /root/work/trunk/mysql-server-mysql-8.0.33/storage/monetdb/sql/storage/bat/bat_storage.c:3427
#2 0x00007f13d9955ecd in log_segments (tr=0x7f0fe37756c0, segs=0x7f0fe38e6880, id=6680) at /root/work/trunk/mysql-server-mysql-8.0.33/storage/monetdb/sql/storage/bat/bat_storage.c:3438
#3 0x00007f13d99587de in log_storage (tr=0x7f0fe37756c0, t=0x7f0fe38e60b0, s=0x7f0fe38e6810) at /root/work/trunk/mysql-server-mysql-8.0.33/storage/monetdb/sql/storage/bat/bat_storage.c:4043
#4 0x00007f13d995970f in log_update_del (tr=0x7f0fe37756c0, change=0x7f0fe37523f0) at /root/work/trunk/mysql-server-mysql-8.0.33/storage/monetdb/sql/storage/bat/bat_storage.c:4312
#5 0x00007f13d9910a8f in sql_trans_commit (tr=0x7f0fe37756c0) at /root/work/trunk/mysql-server-mysql-8.0.33/storage/monetdb/sql/storage/store.c:4047
#6 0x00007f13d992350c in sql_trans_end (s=0x7f0fe388ff80, ok=1) at /root/work/trunk/mysql-server-mysql-8.0.33/storage/monetdb/sql/storage/store.c:7201
#7 0x00007f13d99864a3 in mvc_commit (m=0x7f0fe3cc3c40, chain=0, name=0x0, enabling_auto_commit=true) at /root/work/trunk/mysql-server-mysql-8.0.33/storage/monetdb/sql/server/sql_mvc.c:548
#8 0x00007f13dbb5ba05 in monetdbe_set_commit (dbhdl=0x7f130c0112a0, value=1) at /root/work/trunk/mysql-server-mysql-8.0.33/storage/monetdb/tools/monetdbe/monetdbe.c:1047
#9 0x00000000037a155d in Mondetdb_Adaptor::execute_commit (this=0x7f13c27d0300, ok=true) at /root/work/trunk/mysql-server-mysql-8.0.33/sql/monetdb_adaptor/monetdb_adaptor.cc:341
#10 0x00000000037a4490 in Monetdb_Execute::~Monetdb_Execute (this=0x7f138c7f6fa0, __in_chrg=)
at /root/work/trunk/mysql-server-mysql-8.0.33/sql/monetdb_adaptor/monetdb_adaptor.cc:896
#11 0x0000000003d37041 in Sql_cmd_insert_values::execute_inner (this=0x7f0fe3b82ec8, thd=0x7f130c001040) at /root/work/trunk/mysql-server-mysql-8.0.33/sql/sql_insert.cc:801
#12 0x0000000003650c4c in Sql_cmd_dml::execute (this=0x7f0fe3b82ec8, thd=0x7f130c001040) at /root/work/trunk/mysql-server-mysql-8.0.33/sql/sql_select.cc:578
#13 0x00000000035c9b58 in mysql_execute_command (thd=0x7f130c001040, first_level=true) at /root/work/trunk/mysql-server-mysql-8.0.33/sql/sql_parse.cc:3685
#14 0x00000000035cf38f in dispatch_sql_command (thd=0x7f130c001040, parser_state=0x7f138c7f88b0) at /root/work/trunk/mysql-server-mysql-8.0.33/sql/sql_parse.cc:5398
#15 0x00000000035c4eb2 in dispatch_command (thd=0x7f130c001040, com_data=0x7f138c7f99a0, command=COM_QUERY) at /root/work/trunk/mysql-server-mysql-8.0.33/sql/sql_parse.cc:2052
#16 0x00000000035c2dfa in do_command (thd=0x7f130c001040) at /root/work/trunk/mysql-server-mysql-8.0.33/sql/sql_parse.cc:1441
#17 0x00000000038034ee in handle_connection (arg=0xca4b270) at /root/work/trunk/mysql-server-mysql-8.0.33/sql/conn_handler/connection_handler_per_thread.cc:302
#18 0x0000000005813951 in pfs_spawn_thread (arg=0xcacb6b0) at /root/work/trunk/mysql-server-mysql-8.0.33/storage/perfschema/pfs.cc:3042
#19 0x00007f13db8741ca in start_thread () from /lib64/libpthread.so.0
#20 0x00007f13d9e24e73 in clone () from /lib64/libc.so.6
核心函数:
log_constant
函数源码:
gdk_return
log_constant(logger *lg, int type, ptr val, log_id id, lng offset, lng cnt)
{
bte tpe = find_type(lg, type);
gdk_return ok = GDK_SUCCEED;
logformat l;
lng nr;
l.flag = LOG_UPDATE_CONST;
l.id = id;
nr = cnt;
if (LOG_DISABLED(lg) || !nr) {
/* logging is switched off */
if (nr) {
log_lock(lg);
ok = la_bat_update_count(lg, id, offset+cnt, lg->tid);
log_unlock(lg);
}
return ok;
}
gdk_return (*wt) (const void *, stream *, size_t) = BATatoms[type].atomWrite;
if (log_write_format(lg, &l) != GDK_SUCCEED ||
!mnstr_writeLng(lg->current->output_log, nr) ||
mnstr_write(lg->current->output_log, &tpe, 1, 1) != 1 ||
!mnstr_writeLng(lg->current->output_log, offset)) {
(void) ATOMIC_DEC(&lg->current->refcount);
ok = GDK_FAIL;
goto bailout;
}
ok = wt(val, lg->current->output_log, 1);
if (lg->debug & 1)
fprintf(stderr, "#Logged %d " LLFMT " inserts\n", id, nr);
bailout:
if (ok != GDK_SUCCEED) {
const char *err = mnstr_peek_error(lg->current->output_log);
TRC_CRITICAL(GDK, "write failed%s%s\n", err ? ": " : "", err ? err : "");
}
return ok;
}日志数据:
(gdb) p lg->current->output_log[0]
$19 = {
name = 0x7f0fe360b4d0 "/usr/local/share/mysql/data/.#__mdb_dbfarm__/sql_logs/sql/log.5",
inner = 0x0,
swapbytes = false,
readonly = false,
isutf8 = false,
binary = true,
eof = false,
timeout = 0,
timeout_func = 0x0,
timeout_data = 0x0,
stream_data = {
p = 0x7f130c012c90,
i = 201403536,
s = 201403536
},
read = 0x7f13dbaf8b9c ,
write = 0x7f13dbaf8ccc ,
close = 0x7f13dbaf8da6 ,
clrerr = 0x7f13dbaf8ec8 ,
destroy = 0x7f13dbaf8e7e ,
flush = 0x7f13dbaf8f19 ,
fsync = 0x7f13dbaf8fa8 ,
fgetpos = 0x7f13dbaf903b ,
fsetpos = 0x7f13dbaf90b4 ,
update_timeout = 0x0,
isalive = 0x0,
errkind = MNSTR_NO__ERROR,
errmsg = '\000'
}
[root@dev-stonedb-zhangshilong1 sql]# hexdump -C log.5
00000000 d2 04 00 0b 00 00 00 02 18 1a 00 00 05 00 00 00 |................|
00000010 00 00 00 00 01 19 00 00 00 00 00 00 00 00 09 18 |................|
00000020 1a 00 00 02 18 1a 00 00 05 00 00 00 00 00 00 00 |................|
00000030 01 19 00 00 00 00 00 00 00 00 03 16 1a 00 00 05 |................|
00000040 00 00 00 00 00 00 00 06 ff ff ff ff ff ff ff ff |................|
00000050 01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00 |................|
00000060 05 00 00 00 03 17 1a 00 00 05 00 00 00 00 00 00 |................|
00000070 00 81 ff ff ff ff ff ff ff ff 0a 00 00 00 00 00 |................|
00000080 00 00 61 00 62 00 63 00 64 00 65 00 09 e8 e5 ff |..a.b.c.d.e.....|
00000090 ff 01 0b 00 00 00 00 0c 00 00 00 02 18 1a 00 00 |................|
000000a0 05 00 00 00 00 00 00 00 01 1e 00 00 00 00 00 00 |................|
000000b0 00 00 09 18 1a 00 00 02 18 1a 00 00 05 00 00 00 |................|
000000c0 00 00 00 00 01 1e 00 00 00 00 00 00 00 00 03 16 |................|
000000d0 1a 00 00 05 00 00 00 00 00 00 00 06 ff ff ff ff |................|
000000e0 ff ff ff ff 01 00 00 00 02 00 00 00 03 00 00 00 |................|
000000f0 04 00 00 00 05 00 00 00 03 17 1a 00 00 05 00 00 |................|
00000100 00 00 00 00 00 81 ff ff ff ff ff ff ff ff 0a 00 |................|
00000110 00 00 00 00 00 00 61 00 62 00 63 00 64 00 65 00 |......a.b.c.d.e.|
00000120 09 e8 e5 ff ff 01 0c 00 00 00 |..........|
0000012a
sql_trans_commit
函数源码:
int
sql_trans_commit(sql_trans *tr)
{
int ok = LOG_OK;
sqlstore *store = tr->store;
if (!list_empty(tr->changes)) {
bool flush = false;
ulng commit_ts = 0, oldest = 0, log_file_id = 0;
MT_lock_set(&store->commit);
if (!tr->parent && !list_empty(tr->predicates)) {
ok = sql_trans_valid(tr);
if (ok != LOG_OK) {
sql_trans_rollback(tr, true);
MT_lock_unset(&store->commit);
return ok == LOG_CONFLICT ? SQL_CONFLICT : SQL_ERR;
}
}
if (!tr->parent &&
(!list_empty(tr->dependencies) || !list_empty(tr->depchanges))) {
ok = transaction_check_dependencies_and_removals(tr);
if (ok != LOG_OK) {
sql_trans_rollback(tr, true);
MT_lock_unset(&store->commit);
return ok == LOG_CONFLICT ? SQL_CONFLICT : SQL_ERR;
}
}
/* log changes should only be done if there is something to log */
const bool log = !tr->parent && tr->logchanges > 0;
if (log) {
const int min_changes = GDKdebug & FORCEMITOMASK ? 5 : 1000000;
flush = (tr->logchanges > min_changes && list_empty(store->changes));
}
if (flush)
MT_lock_set(&store->flush);
if (log) {
ok = store->logger_api.log_tstart(store, flush, &log_file_id); /* wal start */
/* log */
for(node *n=tr->changes->h; n && ok == LOG_OK; n = n->next) {
sql_change *c = n->data;
if (c->log && ok == LOG_OK)
ok = c->log(tr, c);
}
if (ok == LOG_OK && !list_empty(store->seqchanges)) {
sequences_lock(store);
for(node *n = store->seqchanges->h; n; ) {
node *next = n->next;
log_store_sequence(store, n->data);
list_remove_node(store->seqchanges, NULL, n);
n = next;
}
sequences_unlock(store);
}
if (ok == LOG_OK && store->prev_oid != store->obj_id)
ok = store->logger_api.log_tsequence(store, OBJ_SID, store->obj_id);
store->prev_oid = store->obj_id;
if (ok == LOG_OK)
ok = store->logger_api.log_tend(store); /* wal end */
}
store_lock(store);
if (tr->parent) {
commit_ts = oldest = tr->parent->tid;
tr->parent->logchanges += tr->logchanges;
}
else {
commit_ts = store_timestamp(store);
oldest = store_oldest(store);
}
tr->logchanges = 0;
TRC_DEBUG(SQL_STORE, "Forwarding changes (" ULLFMT ", " ULLFMT ") -> " ULLFMT "\n", tr->tid, tr->ts, commit_ts);
/* apply committed changes */
if (ATOMIC_GET(&store->nr_active) == 1 && !tr->parent)
oldest = commit_ts;
store_pending_changes(store, oldest);
for(node *n=tr->changes->h; n && ok == LOG_OK; n = n->next) {
sql_change *c = n->data;
if (c->commit && ok == LOG_OK)
ok = c->commit(tr, c, commit_ts, oldest);
else
c->obj->new = 0;
c->ts = commit_ts;
}
/* propagate transaction dependencies to the storage only if other transactions are running */
if (ok == LOG_OK && !tr->parent && ATOMIC_GET(&store->nr_active) > 1) {
if (!list_empty(tr->dependencies)) {
for (node *n = tr->dependencies->h; n && ok == LOG_OK; n = n->next) {
sql_dependency_change *lchange = (sql_dependency_change*) n->data;
ok = transaction_add_hash_entry(store->dependencies, lchange->objid, lchange->type, commit_ts);
}
}
if (!list_empty(tr->depchanges)) {
for (node *n = tr->depchanges->h; n && ok == LOG_OK; n = n->next) {
sql_dependency_change *lchange = (sql_dependency_change*) n->data;
ok = transaction_add_hash_entry(store->depchanges, lchange->objid, lchange->type, commit_ts);
}
}
}
/* garbage collect */
for(node *n=tr->changes->h; n && ok == LOG_OK; ) {
node *next = n->next;
sql_change *c = n->data;
if (!c->cleanup || c->cleanup(store, c, oldest)) {
_DELETE(c);
} else if (tr->parent) { /* need to keep everything */
tr->parent->changes = sa_list_append(tr->sa, tr->parent->changes, c);
} else {
store->changes = sa_list_append(tr->sa, store->changes, c);
}
n = next;
}
tr->active = 2; /* small hack enabling to signal that this transaction has committed */
tr->ts = commit_ts; /* fix profiler output */
store_unlock(store);
/* flush the log structure */
if (log) {
if (!flush)
MT_lock_unset(&store->commit); /* release the commit log when flushing to disk */
if (ok == LOG_OK)
ok = store->logger_api.log_tflush(store, log_file_id, commit_ts); /* flush/sync */
if (!flush)
MT_lock_set(&store->commit); /* release the commit log when flushing to disk */
if (flush)
MT_lock_unset(&store->flush);
}
MT_lock_unset(&store->commit);
list_destroy(tr->changes);
tr->changes = NULL;
} else if (ATOMIC_GET(&store->nr_active) == 1) { /* just me cleanup */
MT_lock_set(&store->commit);
store_lock(store);
ulng oldest = store_timestamp(store);
store_pending_changes(store, oldest);
store_unlock(store);
MT_lock_unset(&store->commit);
}
/* drop local temp tables with commit action CA_DROP, after cleanup */
if (cs_size(&tr->localtmps)) {
for(node *n=tr->localtmps.set->h; n; ) {
node *next = n->next;
sql_table *tt = n->data;
if (tt->commit_action == CA_DROP)
(void) sql_trans_drop_table_id(tr, tt->s, tt->base.id, DROP_RESTRICT);
n = next;
}
}
if (tr->localtmps.dset) {
list_destroy2(tr->localtmps.dset, store);
tr->localtmps.dset = NULL;
}
tr->localtmps.nelm = NULL;
if (ok == LOG_OK)
ok = clean_predicates_and_propagate_to_parent(tr);
return (ok==LOG_OK)?SQL_OK:SQL_ERR;
}