Line data Source code
1 : use std::{cell::Cell, str::FromStr, sync::Arc};
2 :
3 : use crate::walproposer_sim::{safekeeper::run_server, walproposer_api::SimulationApi};
4 : use desim::{
5 : executor::{self, ExternalHandle},
6 : node_os::NodeOs,
7 : options::{Delay, NetworkOptions},
8 : proto::{AnyMessage, NodeEvent},
9 : world::Node,
10 : world::World,
11 : };
12 : use rand::{Rng, SeedableRng};
13 : use tracing::{debug, info_span, warn};
14 : use utils::{id::TenantTimelineId, lsn::Lsn};
15 : use walproposer::walproposer::{Config, Wrapper};
16 :
17 : use super::{
18 : log::SimClock, safekeeper_disk::SafekeeperDisk, walproposer_api,
19 : walproposer_disk::DiskWalProposer,
20 : };
21 :
22 : /// Simulated safekeeper node.
23 : pub struct SafekeeperNode {
24 : pub node: Arc<Node>,
25 : pub id: u32,
26 : pub disk: Arc<SafekeeperDisk>,
27 : pub thread: Cell<ExternalHandle>,
28 : }
29 :
30 : impl SafekeeperNode {
31 : /// Create and start a safekeeper at the specified Node.
32 12048 : pub fn new(node: Arc<Node>) -> Self {
33 12048 : let disk = Arc::new(SafekeeperDisk::new());
34 12048 : let thread = Cell::new(SafekeeperNode::launch(disk.clone(), node.clone()));
35 12048 :
36 12048 : Self {
37 12048 : id: node.id,
38 12048 : node,
39 12048 : disk,
40 12048 : thread,
41 12048 : }
42 12048 : }
43 :
44 77739 : fn launch(disk: Arc<SafekeeperDisk>, node: Arc<Node>) -> ExternalHandle {
45 77739 : // start the server thread
46 77739 : node.launch(move |os| {
47 76407 : run_server(os, disk).expect("server should finish without errors");
48 77739 : })
49 77739 : }
50 :
51 : /// Restart the safekeeper.
52 65691 : pub fn restart(&self) {
53 65691 : let new_thread = SafekeeperNode::launch(self.disk.clone(), self.node.clone());
54 65691 : let old_thread = self.thread.replace(new_thread);
55 65691 : old_thread.crash_stop();
56 65691 : }
57 : }
58 :
59 : /// Simulated walproposer node.
60 : pub struct WalProposer {
61 : thread: ExternalHandle,
62 : node: Arc<Node>,
63 : disk: Arc<DiskWalProposer>,
64 : sync_safekeepers: bool,
65 : }
66 :
67 : impl WalProposer {
68 : /// Generic start function for both modes.
69 72092 : fn start(
70 72092 : os: NodeOs,
71 72092 : disk: Arc<DiskWalProposer>,
72 72092 : ttid: TenantTimelineId,
73 72092 : addrs: Vec<String>,
74 72092 : lsn: Option<Lsn>,
75 72092 : ) {
76 72092 : let sync_safekeepers = lsn.is_none();
77 :
78 72092 : let _enter = if sync_safekeepers {
79 69504 : info_span!("sync", started = executor::now()).entered()
80 : } else {
81 2588 : info_span!("walproposer", started = executor::now()).entered()
82 : };
83 :
84 72092 : os.log_event(format!("started;walproposer;{}", sync_safekeepers as i32));
85 72092 :
86 72092 : let config = Config {
87 72092 : ttid,
88 72092 : safekeepers_list: addrs,
89 72092 : safekeeper_reconnect_timeout: 1000,
90 72092 : safekeeper_connection_timeout: 5000,
91 72092 : sync_safekeepers,
92 72092 : };
93 72092 : let args = walproposer_api::Args {
94 72092 : os,
95 72092 : config: config.clone(),
96 72092 : disk,
97 72092 : redo_start_lsn: lsn,
98 72092 : };
99 72092 : let api = SimulationApi::new(args);
100 72092 : let wp = Wrapper::new(Box::new(api), config);
101 72092 : wp.start();
102 72092 : }
103 :
104 : /// Start walproposer in a sync_safekeepers mode.
105 70919 : pub fn launch_sync(ttid: TenantTimelineId, addrs: Vec<String>, node: Arc<Node>) -> Self {
106 70919 : debug!("sync_safekeepers started at node {}", node.id);
107 70919 : let disk = DiskWalProposer::new();
108 70919 : let disk_wp = disk.clone();
109 70919 :
110 70919 : // start the client thread
111 70919 : let handle = node.launch(move |os| {
112 69504 : WalProposer::start(os, disk_wp, ttid, addrs, None);
113 70919 : });
114 70919 :
115 70919 : Self {
116 70919 : thread: handle,
117 70919 : node,
118 70919 : disk,
119 70919 : sync_safekeepers: true,
120 70919 : }
121 70919 : }
122 :
123 : /// Start walproposer in a normal mode.
124 2588 : pub fn launch_walproposer(
125 2588 : ttid: TenantTimelineId,
126 2588 : addrs: Vec<String>,
127 2588 : node: Arc<Node>,
128 2588 : lsn: Lsn,
129 2588 : ) -> Self {
130 2588 : debug!("walproposer started at node {}", node.id);
131 2588 : let disk = DiskWalProposer::new();
132 2588 : disk.lock().reset_to(lsn);
133 2588 : let disk_wp = disk.clone();
134 2588 :
135 2588 : // start the client thread
136 2588 : let handle = node.launch(move |os| {
137 2588 : WalProposer::start(os, disk_wp, ttid, addrs, Some(lsn));
138 2588 : });
139 2588 :
140 2588 : Self {
141 2588 : thread: handle,
142 2588 : node,
143 2588 : disk,
144 2588 : sync_safekeepers: false,
145 2588 : }
146 2588 : }
147 :
148 2857 : pub fn write_tx(&mut self, cnt: usize) {
149 2857 : let start_lsn = self.disk.lock().flush_rec_ptr();
150 2857 :
151 31916 : for _ in 0..cnt {
152 31916 : self.disk
153 31916 : .lock()
154 31916 : .insert_logical_message("prefix", b"message")
155 31916 : .expect("failed to generate logical message");
156 31916 : }
157 :
158 2857 : let end_lsn = self.disk.lock().flush_rec_ptr();
159 2857 :
160 2857 : // log event
161 2857 : self.node
162 2857 : .log_event(format!("write_wal;{};{};{}", start_lsn.0, end_lsn.0, cnt));
163 2857 :
164 2857 : // now we need to set "Latch" in walproposer
165 2857 : self.node
166 2857 : .node_events()
167 2857 : .send(NodeEvent::Internal(AnyMessage::Just32(0)));
168 2857 : }
169 :
170 66479 : pub fn stop(&self) {
171 66479 : self.thread.crash_stop();
172 66479 : }
173 : }
174 :
175 : /// Holds basic simulation settings, such as network options.
176 : pub struct TestConfig {
177 : pub network: NetworkOptions,
178 : pub timeout: u64,
179 : pub clock: Option<SimClock>,
180 : }
181 :
182 : impl TestConfig {
183 : /// Create a new TestConfig with default settings.
184 18 : pub fn new(clock: Option<SimClock>) -> Self {
185 18 : Self {
186 18 : network: NetworkOptions {
187 18 : keepalive_timeout: Some(2000),
188 18 : connect_delay: Delay {
189 18 : min: 1,
190 18 : max: 5,
191 18 : fail_prob: 0.0,
192 18 : },
193 18 : send_delay: Delay {
194 18 : min: 1,
195 18 : max: 5,
196 18 : fail_prob: 0.0,
197 18 : },
198 18 : },
199 18 : timeout: 1_000 * 10,
200 18 : clock,
201 18 : }
202 18 : }
203 :
204 : /// Start a new simulation with the specified seed.
205 4016 : pub fn start(&self, seed: u64) -> Test {
206 4016 : let world = Arc::new(World::new(seed, Arc::new(self.network.clone())));
207 :
208 4016 : if let Some(clock) = &self.clock {
209 4016 : clock.set_clock(world.clock());
210 4016 : }
211 :
212 4016 : let servers = [
213 4016 : SafekeeperNode::new(world.new_node()),
214 4016 : SafekeeperNode::new(world.new_node()),
215 4016 : SafekeeperNode::new(world.new_node()),
216 4016 : ];
217 4016 :
218 4016 : let server_ids = [servers[0].id, servers[1].id, servers[2].id];
219 12048 : let safekeepers_addrs = server_ids.map(|id| format!("node:{}", id)).to_vec();
220 4016 :
221 4016 : let ttid = TenantTimelineId::generate();
222 4016 :
223 4016 : Test {
224 4016 : world,
225 4016 : servers,
226 4016 : sk_list: safekeepers_addrs,
227 4016 : ttid,
228 4016 : timeout: self.timeout,
229 4016 : }
230 4016 : }
231 : }
232 :
233 : /// Holds simulation state.
234 : pub struct Test {
235 : pub world: Arc<World>,
236 : pub servers: [SafekeeperNode; 3],
237 : pub sk_list: Vec<String>,
238 : pub ttid: TenantTimelineId,
239 : pub timeout: u64,
240 : }
241 :
242 : impl Test {
243 : /// Start a sync_safekeepers thread and wait for it to finish.
244 12 : pub fn sync_safekeepers(&self) -> anyhow::Result<Lsn> {
245 12 : let wp = self.launch_sync_safekeepers();
246 12 :
247 12 : // poll until exit or timeout
248 12 : let time_limit = self.timeout;
249 460 : while self.world.step() && self.world.now() < time_limit && !wp.thread.is_finished() {}
250 :
251 12 : if !wp.thread.is_finished() {
252 0 : anyhow::bail!("timeout or idle stuck");
253 12 : }
254 12 :
255 12 : let res = wp.thread.result();
256 12 : if res.0 != 0 {
257 0 : anyhow::bail!("non-zero exitcode: {:?}", res);
258 12 : }
259 12 : let lsn = Lsn::from_str(&res.1)?;
260 12 : Ok(lsn)
261 12 : }
262 :
263 : /// Spawn a new sync_safekeepers thread.
264 70919 : pub fn launch_sync_safekeepers(&self) -> WalProposer {
265 70919 : WalProposer::launch_sync(self.ttid, self.sk_list.clone(), self.world.new_node())
266 70919 : }
267 :
268 : /// Spawn a new walproposer thread.
269 2588 : pub fn launch_walproposer(&self, lsn: Lsn) -> WalProposer {
270 2588 : let lsn = if lsn.0 == 0 {
271 : // usual LSN after basebackup
272 1329 : Lsn(21623024)
273 : } else {
274 1259 : lsn
275 : };
276 :
277 2588 : WalProposer::launch_walproposer(self.ttid, self.sk_list.clone(), self.world.new_node(), lsn)
278 2588 : }
279 :
280 : /// Execute the simulation for the specified duration.
281 210 : pub fn poll_for_duration(&self, duration: u64) {
282 210 : let time_limit = std::cmp::min(self.world.now() + duration, self.timeout);
283 3552 : while self.world.step() && self.world.now() < time_limit {}
284 210 : }
285 :
286 : /// Execute the simulation together with events defined in some schedule.
287 4008 : pub fn run_schedule(&self, schedule: &Schedule) -> anyhow::Result<()> {
288 4008 : // scheduling empty events so that world will stop in those points
289 4008 : {
290 4008 : let clock = self.world.clock();
291 4008 :
292 4008 : let now = self.world.now();
293 202338 : for (time, _) in schedule {
294 198330 : if *time < now {
295 0 : continue;
296 198330 : }
297 198330 : clock.schedule_fake(*time - now);
298 : }
299 : }
300 :
301 4008 : let mut wp = self.launch_sync_safekeepers();
302 4008 :
303 4008 : let mut skipped_tx = 0;
304 4008 : let mut started_tx = 0;
305 4008 :
306 4008 : let mut schedule_ptr = 0;
307 :
308 205446 : loop {
309 205446 : if wp.sync_safekeepers && wp.thread.is_finished() {
310 3004 : let res = wp.thread.result();
311 3004 : if res.0 != 0 {
312 422 : warn!("sync non-zero exitcode: {:?}", res);
313 422 : debug!("restarting sync_safekeepers");
314 : // restart the sync_safekeepers
315 422 : wp = self.launch_sync_safekeepers();
316 422 : continue;
317 2582 : }
318 2582 : let lsn = Lsn::from_str(&res.1)?;
319 2582 : debug!("sync_safekeepers finished at LSN {}", lsn);
320 2582 : wp = self.launch_walproposer(lsn);
321 2582 : debug!("walproposer started at thread {}", wp.thread.id());
322 202442 : }
323 :
324 205024 : let now = self.world.now();
325 403354 : while schedule_ptr < schedule.len() && schedule[schedule_ptr].0 <= now {
326 198330 : if now != schedule[schedule_ptr].0 {
327 0 : warn!("skipped event {:?} at {}", schedule[schedule_ptr], now);
328 198330 : }
329 :
330 198330 : let action = &schedule[schedule_ptr].1;
331 198330 : match action {
332 66164 : TestAction::WriteTx(size) => {
333 66164 : if !wp.sync_safekeepers && !wp.thread.is_finished() {
334 2653 : started_tx += *size;
335 2653 : wp.write_tx(*size);
336 2653 : debug!("written {} transactions", size);
337 : } else {
338 63511 : skipped_tx += size;
339 63511 : debug!("skipped {} transactions", size);
340 : }
341 : }
342 65689 : TestAction::RestartSafekeeper(id) => {
343 65689 : debug!("restarting safekeeper {}", id);
344 65689 : self.servers[*id].restart();
345 : }
346 : TestAction::RestartWalProposer => {
347 66477 : debug!("restarting sync_safekeepers");
348 66477 : wp.stop();
349 66477 : wp = self.launch_sync_safekeepers();
350 : }
351 : }
352 198330 : schedule_ptr += 1;
353 : }
354 :
355 205024 : if schedule_ptr == schedule.len() {
356 4008 : break;
357 201016 : }
358 201016 : let next_event_time = schedule[schedule_ptr].0;
359 201016 :
360 201016 : // poll until the next event
361 201016 : if wp.thread.is_finished() {
362 2252 : while self.world.step() && self.world.now() < next_event_time {}
363 : } else {
364 3077528 : while self.world.step()
365 3077528 : && self.world.now() < next_event_time
366 2879808 : && !wp.thread.is_finished()
367 2876722 : {}
368 : }
369 : }
370 :
371 4008 : debug!(
372 4 : "finished schedule, total steps: {}",
373 4 : self.world.get_thread_step_count()
374 4 : );
375 4008 : debug!("skipped_tx: {}", skipped_tx);
376 4008 : debug!("started_tx: {}", started_tx);
377 :
378 4008 : Ok(())
379 4008 : }
380 : }
381 :
382 240 : #[derive(Debug, Clone)]
383 : pub enum TestAction {
384 : WriteTx(usize),
385 : RestartSafekeeper(usize),
386 : RestartWalProposer,
387 : }
388 :
389 : pub type Schedule = Vec<(u64, TestAction)>;
390 :
391 4004 : pub fn generate_schedule(seed: u64) -> Schedule {
392 4004 : let mut rng = rand::rngs::StdRng::seed_from_u64(seed);
393 4004 : let mut schedule = Vec::new();
394 4004 : let mut time = 0;
395 4004 :
396 4004 : let cnt = rng.gen_range(1..100);
397 4004 :
398 4004 : for _ in 0..cnt {
399 198100 : time += rng.gen_range(0..500);
400 198100 : let action = match rng.gen_range(0..3) {
401 65950 : 0 => TestAction::WriteTx(rng.gen_range(1..10)),
402 65677 : 1 => TestAction::RestartSafekeeper(rng.gen_range(0..3)),
403 66473 : 2 => TestAction::RestartWalProposer,
404 0 : _ => unreachable!(),
405 : };
406 198100 : schedule.push((time, action));
407 : }
408 :
409 4004 : schedule
410 4004 : }
411 :
412 4004 : pub fn generate_network_opts(seed: u64) -> NetworkOptions {
413 4004 : let mut rng = rand::rngs::StdRng::seed_from_u64(seed);
414 4004 :
415 4004 : let timeout = rng.gen_range(100..2000);
416 4004 : let max_delay = rng.gen_range(1..2 * timeout);
417 4004 : let min_delay = rng.gen_range(1..=max_delay);
418 4004 :
419 4004 : let max_fail_prob = rng.gen_range(0.0..0.9);
420 4004 : let connect_fail_prob = rng.gen_range(0.0..max_fail_prob);
421 4004 : let send_fail_prob = rng.gen_range(0.0..connect_fail_prob);
422 4004 :
423 4004 : NetworkOptions {
424 4004 : keepalive_timeout: Some(timeout),
425 4004 : connect_delay: Delay {
426 4004 : min: min_delay,
427 4004 : max: max_delay,
428 4004 : fail_prob: connect_fail_prob,
429 4004 : },
430 4004 : send_delay: Delay {
431 4004 : min: min_delay,
432 4004 : max: max_delay,
433 4004 : fail_prob: send_fail_prob,
434 4004 : },
435 4004 : }
436 4004 : }
|
