From 31e929337fe5f45f694bd3b93ebb9a9765bb4587 Mon Sep 17 00:00:00 2001 From: Sonal Agarwal Date: Fri, 31 Jul 2026 10:34:19 +0530 Subject: [PATCH 1/4] - retry loading data if there is failure in load phase --- src/lua/oltp_common.lua | 66 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 2 deletions(-) diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua index ab0e1db09..ba5853f25 100644 --- a/src/lua/oltp_common.lua +++ b/src/lua/oltp_common.lua @@ -90,7 +90,11 @@ sysbench.cmdline.options = { num_rows_in_insert = {"Number of INSERT per transaction, for multi-insert test", 10}, batch_insert_count = {"Number of rows inserted with one insert", 4000}, - smaller_row_sizes = {"Smaller row sizes, applicable only for oltp_multi_value_insert ", false} + smaller_row_sizes = {"Smaller row sizes, applicable only for oltp_multi_value_insert ", false}, + load_max_retries = + {"Maximum number of retries for data loading on SQL errors", 3}, + load_retry_delay = + {"Delay in seconds between load retries", 10} } -- Prepare the dataset. This command supports parallel execution, i.e. will @@ -113,9 +117,67 @@ end function cmd_load() local drv = sysbench.sql.driver() local con = drv:connect() + local max_retries = sysbench.opt.load_max_retries + local retry_delay = sysbench.opt.load_retry_delay + for i = sysbench.tid % sysbench.opt.threads + 1, sysbench.opt.tables, sysbench.opt.threads do - bulk_load(con, i) + local attempt = 0 + local success = false + + while not success and attempt <= max_retries do + local ok, err = pcall(bulk_load, con, i) + + if ok then + success = true + else + attempt = attempt + 1 + local time = os.date("*t") + print(string.format( + "(%2d:%2d:%2d) ERROR loading table 'sbtest%d': %s", + time.hour, time.min, time.sec, i, tostring(err))) + + if attempt > max_retries then + error(string.format( + "Failed to load table 'sbtest%d' after %d retries. Last error: %s", + i, max_retries, tostring(err))) + end + + print(string.format( + "(%2d:%2d:%2d) Retry %d/%d: dropping and recreating table 'sbtest%d' before reload...", + time.hour, time.min, time.sec, attempt, max_retries, i)) + + -- Reconnect in case the connection is in a bad state + local rok, rerr = pcall(function() con:reconnect() end) + if not rok then + print(string.format( + "(%2d:%2d:%2d) WARNING: reconnect failed (%s), creating new connection...", + time.hour, time.min, time.sec, tostring(rerr))) + con = drv:connect() + end + + -- Drop the partially loaded table and recreate it + local dok, derr = pcall(function() + con:query("DROP TABLE IF EXISTS sbtest" .. i) + end) + if not dok then + print(string.format( + "(%2d:%2d:%2d) WARNING: DROP TABLE failed (%s), reconnecting...", + time.hour, time.min, time.sec, tostring(derr))) + con = drv:connect() + con:query("DROP TABLE IF EXISTS sbtest" .. i) + end + + create_table(drv, con, i) + + if retry_delay > 0 then + print(string.format( + "(%2d:%2d:%2d) Waiting %d seconds before retry...", + time.hour, time.min, time.sec, retry_delay)) + os.execute("sleep " .. retry_delay) + end + end + end end end From 0541114c04a382b378a3fca270db81a96ad62f3e Mon Sep 17 00:00:00 2001 From: Sonal Agarwal Date: Fri, 31 Jul 2026 12:12:52 +0530 Subject: [PATCH 2/4] - refactoring changes --- src/lua/oltp_common.lua | 75 +++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 48 deletions(-) diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua index ba5853f25..b28e11da8 100644 --- a/src/lua/oltp_common.lua +++ b/src/lua/oltp_common.lua @@ -114,6 +114,17 @@ function cmd_create() end end +local function log_time(fmt, ...) + local t = os.date("*t") + print(string.format("(%2d:%2d:%2d) " .. fmt, t.hour, t.min, t.sec, ...)) +end + +local function ensure_connection(drv, con) + local ok = pcall(function() con:reconnect() end) + if ok then return con end + return drv:connect() +end + function cmd_load() local drv = sysbench.sql.driver() local con = drv:connect() @@ -122,60 +133,28 @@ function cmd_load() for i = sysbench.tid % sysbench.opt.threads + 1, sysbench.opt.tables, sysbench.opt.threads do - local attempt = 0 - local success = false - - while not success and attempt <= max_retries do + for attempt = 1, max_retries + 1 do local ok, err = pcall(bulk_load, con, i) + if ok then break end - if ok then - success = true - else - attempt = attempt + 1 - local time = os.date("*t") - print(string.format( - "(%2d:%2d:%2d) ERROR loading table 'sbtest%d': %s", - time.hour, time.min, time.sec, i, tostring(err))) - - if attempt > max_retries then - error(string.format( - "Failed to load table 'sbtest%d' after %d retries. Last error: %s", - i, max_retries, tostring(err))) - end + log_time("ERROR loading 'sbtest%d': %s", i, tostring(err)) - print(string.format( - "(%2d:%2d:%2d) Retry %d/%d: dropping and recreating table 'sbtest%d' before reload...", - time.hour, time.min, time.sec, attempt, max_retries, i)) - - -- Reconnect in case the connection is in a bad state - local rok, rerr = pcall(function() con:reconnect() end) - if not rok then - print(string.format( - "(%2d:%2d:%2d) WARNING: reconnect failed (%s), creating new connection...", - time.hour, time.min, time.sec, tostring(rerr))) - con = drv:connect() - end + if attempt > max_retries then + error(string.format( + "Failed to load 'sbtest%d' after %d retries: %s", + i, max_retries, tostring(err))) + end - -- Drop the partially loaded table and recreate it - local dok, derr = pcall(function() - con:query("DROP TABLE IF EXISTS sbtest" .. i) - end) - if not dok then - print(string.format( - "(%2d:%2d:%2d) WARNING: DROP TABLE failed (%s), reconnecting...", - time.hour, time.min, time.sec, tostring(derr))) - con = drv:connect() - con:query("DROP TABLE IF EXISTS sbtest" .. i) - end + log_time("Retry %d/%d: recreating 'sbtest%d'...", + attempt, max_retries, i) - create_table(drv, con, i) + con = ensure_connection(drv, con) + con:query("DROP TABLE IF EXISTS sbtest" .. i) + create_table(drv, con, i) - if retry_delay > 0 then - print(string.format( - "(%2d:%2d:%2d) Waiting %d seconds before retry...", - time.hour, time.min, time.sec, retry_delay)) - os.execute("sleep " .. retry_delay) - end + if retry_delay > 0 then + log_time("Waiting %d seconds before retry...", retry_delay) + os.execute("sleep " .. retry_delay) end end end From c8e3fa2029088036ee754819de17c4968ef7503a Mon Sep 17 00:00:00 2001 From: Sonal Agarwal Date: Fri, 31 Jul 2026 15:31:28 +0530 Subject: [PATCH 3/4] - errors before retry should be warnings --- src/lua/oltp_common.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua index b28e11da8..c7e8b8c04 100644 --- a/src/lua/oltp_common.lua +++ b/src/lua/oltp_common.lua @@ -137,14 +137,14 @@ function cmd_load() local ok, err = pcall(bulk_load, con, i) if ok then break end - log_time("ERROR loading 'sbtest%d': %s", i, tostring(err)) - if attempt > max_retries then + log_time("ERROR loading 'sbtest%d': %s", i, tostring(err)) error(string.format( "Failed to load 'sbtest%d' after %d retries: %s", i, max_retries, tostring(err))) end + log_time("WARNING loading 'sbtest%d': %s", i, tostring(err)) log_time("Retry %d/%d: recreating 'sbtest%d'...", attempt, max_retries, i) From 85c4b66d3c8b6c228cf97a6f35f7ab2ba078f770 Mon Sep 17 00:00:00 2001 From: Sonal Agarwal Date: Tue, 4 Aug 2026 19:21:27 +0530 Subject: [PATCH 4/4] - chnages to stimulate the max retries --- src/lua/oltp_common.lua | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/lua/oltp_common.lua b/src/lua/oltp_common.lua index c7e8b8c04..9776d917e 100644 --- a/src/lua/oltp_common.lua +++ b/src/lua/oltp_common.lua @@ -94,7 +94,9 @@ sysbench.cmdline.options = { load_max_retries = {"Maximum number of retries for data loading on SQL errors", 3}, load_retry_delay = - {"Delay in seconds between load retries", 10} + {"Delay in seconds between load retries", 10}, + load_simulate_failures = + {"[TEST ONLY] Simulate this many consecutive load failures per table before succeeding. 0 to disable", 0} } -- Prepare the dataset. This command supports parallel execution, i.e. will @@ -125,16 +127,33 @@ local function ensure_connection(drv, con) return drv:connect() end +local simulate_fail_counts = {} + function cmd_load() local drv = sysbench.sql.driver() local con = drv:connect() local max_retries = sysbench.opt.load_max_retries local retry_delay = sysbench.opt.load_retry_delay + local sim_failures = sysbench.opt.load_simulate_failures for i = sysbench.tid % sysbench.opt.threads + 1, sysbench.opt.tables, sysbench.opt.threads do for attempt = 1, max_retries + 1 do - local ok, err = pcall(bulk_load, con, i) + local ok, err + + if sim_failures > 0 then + simulate_fail_counts[i] = (simulate_fail_counts[i] or 0) + 1 + if simulate_fail_counts[i] <= sim_failures then + ok, err = false, string.format( + "Simulated SQL error on 'sbtest%d' (failure %d/%d)", + i, simulate_fail_counts[i], sim_failures) + else + ok, err = pcall(bulk_load, con, i) + end + else + ok, err = pcall(bulk_load, con, i) + end + if ok then break end if attempt > max_retries then