1use alloy::network::ReceiptResponse;
6use chrono::{DateTime, Duration, Utc};
7use eyre::Result;
8use tracing::{debug, error, warn};
9
10use super::super::common::is_active_nonce_status;
11use super::EvmRelayerTransaction;
12use super::{
13 ensure_status, evm_transaction::TX_NONCE_RECONCILE_TRIGGER, get_age_since_status_change,
14 has_enough_confirmations, is_noop, is_too_early_to_resubmit, is_transaction_valid, make_noop,
15 too_many_attempts, too_many_noop_attempts,
16};
17use crate::constants::{
18 get_evm_min_age_for_hash_recovery, get_evm_pending_recovery_trigger_timeout,
19 get_evm_prepare_timeout, get_evm_resend_timeout, ARBITRUM_TIME_TO_RESUBMIT,
20 EVM_MIN_HASHES_FOR_RECOVERY, MAX_GAP_SCAN_RANGE,
21};
22use crate::domain::transaction::common::{
23 get_age_of_sent_at, is_final_state, is_pending_transaction,
24};
25use crate::domain::transaction::util::get_age_since_created;
26use crate::models::{EvmNetwork, NetworkRepoModel, NetworkType};
27use crate::repositories::{NetworkRepository, RelayerRepository};
28use crate::{
29 domain::transaction::evm::price_calculator::PriceCalculatorTrait,
30 jobs::{JobProducerTrait, StatusCheckContext},
31 models::{
32 NetworkTransactionData, RelayerRepoModel, TransactionError, TransactionRepoModel,
33 TransactionStatus, TransactionUpdateRequest,
34 },
35 repositories::{Repository, TransactionCounterTrait, TransactionRepository},
36 services::{provider::EvmProviderTrait, signer::Signer},
37 utils::{get_resubmit_timeout_for_speed, get_resubmit_timeout_with_backoff},
38};
39
40impl<P, RR, NR, TR, J, S, TCR, PC> EvmRelayerTransaction<P, RR, NR, TR, J, S, TCR, PC>
41where
42 P: EvmProviderTrait + Send + Sync,
43 RR: RelayerRepository + Repository<RelayerRepoModel, String> + Send + Sync + 'static,
44 NR: NetworkRepository + Repository<NetworkRepoModel, String> + Send + Sync + 'static,
45 TR: TransactionRepository + Repository<TransactionRepoModel, String> + Send + Sync + 'static,
46 J: JobProducerTrait + Send + Sync + 'static,
47 S: Signer + Send + Sync + 'static,
48 TCR: TransactionCounterTrait + Send + Sync + 'static,
49 PC: PriceCalculatorTrait + Send + Sync,
50{
51 pub(super) async fn check_transaction_status(
52 &self,
53 tx: &TransactionRepoModel,
54 ) -> Result<TransactionStatus, TransactionError> {
55 if is_final_state(&tx.status) {
57 return Ok(tx.status.clone());
58 }
59
60 match tx.status {
63 TransactionStatus::Pending | TransactionStatus::Sent => {
64 return Ok(tx.status.clone());
65 }
66 _ => {}
67 }
68
69 let evm_data = tx.network_data.get_evm_transaction_data()?;
70 let tx_hash = evm_data
71 .hash
72 .as_ref()
73 .ok_or(TransactionError::UnexpectedError(
74 "Transaction hash is missing".to_string(),
75 ))?;
76
77 let receipt_result = self.provider().get_transaction_receipt(tx_hash).await?;
78
79 if let Some(receipt) = receipt_result {
80 if !receipt.inner.status() {
81 return Ok(TransactionStatus::Failed);
82 }
83 let last_block_number = self.provider().get_block_number().await?;
84 let tx_block_number = receipt
85 .block_number
86 .ok_or(TransactionError::UnexpectedError(
87 "Transaction receipt missing block number".to_string(),
88 ))?;
89
90 let network_model = self
91 .network_repository()
92 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
93 .await?
94 .ok_or(TransactionError::UnexpectedError(format!(
95 "Network with chain id {} not found",
96 evm_data.chain_id
97 )))?;
98
99 let network = EvmNetwork::try_from(network_model).map_err(|e| {
100 TransactionError::UnexpectedError(format!(
101 "Error converting network model to EvmNetwork: {e}"
102 ))
103 })?;
104
105 if !has_enough_confirmations(
106 tx_block_number,
107 last_block_number,
108 network.required_confirmations,
109 ) {
110 debug!(
111 tx_id = %tx.id,
112 relayer_id = %tx.relayer_id,
113 tx_hash = %tx_hash,
114 "transaction mined but not confirmed"
115 );
116 return Ok(TransactionStatus::Mined);
117 }
118 if tx.is_canceled == Some(true) && is_noop(&evm_data) {
123 debug!(
124 tx_id = %tx.id,
125 relayer_id = %tx.relayer_id,
126 tx_hash = %tx_hash,
127 "cancellation NOOP confirmed on-chain; marking transaction as Canceled"
128 );
129 return Ok(TransactionStatus::Canceled);
130 }
131 Ok(TransactionStatus::Confirmed)
132 } else {
133 debug!(
134 tx_id = %tx.id,
135 relayer_id = %tx.relayer_id,
136 tx_hash = %tx_hash,
137 "transaction not yet mined"
138 );
139
140 if tx.hashes.len() > 1 && self.should_try_hash_recovery(tx)? {
144 if let Some(recovered_tx) = self
145 .try_recover_with_historical_hashes(tx, &evm_data)
146 .await?
147 {
148 return Ok(recovered_tx.status);
150 }
151 }
152
153 Ok(TransactionStatus::Submitted)
154 }
155 }
156
157 pub(super) async fn should_resubmit(
159 &self,
160 tx: &TransactionRepoModel,
161 ) -> Result<bool, TransactionError> {
162 ensure_status(tx, TransactionStatus::Submitted, Some("should_resubmit"))?;
164
165 let evm_data = tx.network_data.get_evm_transaction_data()?;
166 let age = get_age_of_sent_at(tx)?;
167
168 let network_model = self
170 .network_repository()
171 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
172 .await?
173 .ok_or(TransactionError::UnexpectedError(format!(
174 "Network with chain id {} not found",
175 evm_data.chain_id
176 )))?;
177
178 let network = EvmNetwork::try_from(network_model).map_err(|e| {
179 TransactionError::UnexpectedError(format!(
180 "Error converting network model to EvmNetwork: {e}"
181 ))
182 })?;
183
184 let timeout = match network.is_arbitrum() {
185 true => ARBITRUM_TIME_TO_RESUBMIT,
186 false => get_resubmit_timeout_for_speed(&evm_data.speed),
187 };
188
189 let timeout_with_backoff = match network.is_arbitrum() {
190 true => timeout, false => get_resubmit_timeout_with_backoff(timeout, tx.hashes.len()),
192 };
193
194 if age > Duration::milliseconds(timeout_with_backoff) {
195 debug!(
196 tx_id = %tx.id,
197 relayer_id = %tx.relayer_id,
198 age_ms = %age.num_milliseconds(),
199 "transaction has been pending for too long, resubmitting"
200 );
201 return Ok(true);
202 }
203 Ok(false)
204 }
205
206 pub(super) async fn should_noop(
216 &self,
217 tx: &TransactionRepoModel,
218 ) -> Result<(bool, Option<String>), TransactionError> {
219 if too_many_noop_attempts(tx) {
220 debug!("Transaction has too many NOOP attempts already");
221 return Ok((false, None));
222 }
223
224 let evm_data = tx.network_data.get_evm_transaction_data()?;
225 if is_noop(&evm_data) {
226 return Ok((false, None));
227 }
228
229 let network_model = self
230 .network_repository()
231 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
232 .await?
233 .ok_or(TransactionError::UnexpectedError(format!(
234 "Network with chain id {} not found",
235 evm_data.chain_id
236 )))?;
237
238 let network = EvmNetwork::try_from(network_model).map_err(|e| {
239 TransactionError::UnexpectedError(format!(
240 "Error converting network model to EvmNetwork: {e}"
241 ))
242 })?;
243
244 if network.is_rollup() && too_many_attempts(tx) {
245 let reason =
246 "Rollup transaction has too many attempts. Replacing with NOOP.".to_string();
247 debug!(
248 tx_id = %tx.id,
249 relayer_id = %tx.relayer_id,
250 reason = %reason,
251 "replacing transaction with NOOP"
252 );
253 return Ok((true, Some(reason)));
254 }
255
256 if !is_transaction_valid(&tx.created_at, &tx.valid_until) {
257 let reason = "Transaction is expired. Replacing with NOOP.".to_string();
258 debug!(
259 tx_id = %tx.id,
260 relayer_id = %tx.relayer_id,
261 reason = %reason,
262 "replacing transaction with NOOP"
263 );
264 return Ok((true, Some(reason)));
265 }
266
267 if tx.status == TransactionStatus::Pending {
268 let created_at = &tx.created_at;
269 let created_time = DateTime::parse_from_rfc3339(created_at)
270 .map_err(|e| {
271 TransactionError::UnexpectedError(format!("Invalid created_at timestamp: {e}"))
272 })?
273 .with_timezone(&Utc);
274 let age = Utc::now().signed_duration_since(created_time);
275 if age > get_evm_prepare_timeout() {
276 let reason = format!(
277 "Transaction in Pending state for over {} minutes. Replacing with NOOP.",
278 get_evm_prepare_timeout().num_minutes()
279 );
280 debug!(
281 tx_id = %tx.id,
282 relayer_id = %tx.relayer_id,
283 reason = %reason,
284 "replacing transaction with NOOP"
285 );
286 return Ok((true, Some(reason)));
287 }
288 }
289
290 let latest_block = self.provider().get_block_by_number().await;
291 if let Ok(block) = latest_block {
292 let block_gas_limit = block.header.gas_limit;
293 if let Some(gas_limit) = evm_data.gas_limit {
294 if gas_limit > block_gas_limit {
295 let reason = format!(
296 "Transaction gas limit ({gas_limit}) exceeds block gas limit ({block_gas_limit}). Replacing with NOOP.",
297 );
298 warn!(
299 tx_id = %tx.id,
300 tx_gas_limit = %gas_limit,
301 block_gas_limit = %block_gas_limit,
302 "transaction gas limit exceeds block gas limit, replacing with NOOP"
303 );
304 return Ok((true, Some(reason)));
305 }
306 }
307 }
308
309 Ok((false, None))
310 }
311
312 pub(super) async fn update_transaction_status_if_needed(
314 &self,
315 tx: TransactionRepoModel,
316 new_status: TransactionStatus,
317 status_reason: Option<String>,
318 ) -> Result<TransactionRepoModel, TransactionError> {
319 if tx.status != new_status {
320 return self
321 .update_transaction_status(tx, new_status, status_reason)
322 .await;
323 }
324 Ok(tx)
325 }
326
327 pub(super) async fn prepare_noop_update_request(
329 &self,
330 tx: &TransactionRepoModel,
331 is_cancellation: bool,
332 reason: Option<String>,
333 ) -> Result<TransactionUpdateRequest, TransactionError> {
334 let mut evm_data = tx.network_data.get_evm_transaction_data()?;
335 let network_model = self
336 .network_repository()
337 .get_by_chain_id(NetworkType::Evm, evm_data.chain_id)
338 .await?
339 .ok_or(TransactionError::UnexpectedError(format!(
340 "Network with chain id {} not found",
341 evm_data.chain_id
342 )))?;
343
344 let network = EvmNetwork::try_from(network_model).map_err(|e| {
345 TransactionError::UnexpectedError(format!(
346 "Error converting network model to EvmNetwork: {e}"
347 ))
348 })?;
349
350 make_noop(&mut evm_data, &network, Some(self.provider())).await?;
351
352 let noop_count = tx.noop_count.unwrap_or(0) + 1;
353 let update_request = TransactionUpdateRequest {
354 network_data: Some(NetworkTransactionData::Evm(evm_data)),
355 noop_count: Some(noop_count),
356 status_reason: reason,
357 is_canceled: if is_cancellation {
358 Some(true)
359 } else {
360 tx.is_canceled
361 },
362 ..Default::default()
363 };
364 Ok(update_request)
365 }
366
367 async fn handle_submitted_state(
374 &self,
375 tx: TransactionRepoModel,
376 ) -> Result<TransactionRepoModel, TransactionError> {
377 if self.should_resubmit(&tx).await? {
378 if let Some(nonce_gap_detected) = self.detect_nonce_gap_ahead(&tx).await {
381 if nonce_gap_detected {
382 return self
384 .update_transaction_status_if_needed(tx, TransactionStatus::Submitted, None)
385 .await;
386 }
387 }
388
389 let resubmitted_tx = self.handle_resubmission(tx).await?;
390 return Ok(resubmitted_tx);
391 }
392
393 self.update_transaction_status_if_needed(tx, TransactionStatus::Submitted, None)
394 .await
395 }
396
397 async fn detect_nonce_gap_ahead(&self, tx: &TransactionRepoModel) -> Option<bool> {
411 let evm_data = match tx.network_data.get_evm_transaction_data() {
412 Ok(d) => d,
413 Err(_) => return None,
414 };
415 let tx_nonce = match evm_data.nonce {
416 Some(n) => n,
417 None => return None,
418 };
419
420 let on_chain_nonce = match self
421 .provider()
422 .get_transaction_count(&self.relayer().address)
423 .await
424 {
425 Ok(n) => n,
426 Err(e) => {
427 debug!(
428 tx_id = %tx.id,
429 error = %e,
430 "nonce gap check: failed to get on-chain nonce, skipping"
431 );
432 return None;
433 }
434 };
435
436 if tx_nonce <= on_chain_nonce {
438 return Some(false);
439 }
440
441 let scan_to = std::cmp::min(tx_nonce, on_chain_nonce + MAX_GAP_SCAN_RANGE);
445
446 let occupancy = match self
447 .transaction_repository()
448 .get_nonce_occupancy(&tx.relayer_id, on_chain_nonce, scan_to)
449 .await
450 {
451 Ok(o) => o,
452 Err(e) => {
453 debug!(
454 tx_id = %tx.id,
455 error = %e,
456 "nonce gap check: occupancy lookup failed, skipping"
457 );
458 return None;
459 }
460 };
461
462 let gap_nonces: Vec<u64> = occupancy
463 .into_iter()
464 .filter(|(_, status)| !status.as_ref().is_some_and(is_active_nonce_status))
465 .map(|(nonce, _)| nonce)
466 .collect();
467
468 if gap_nonces.is_empty() {
469 return Some(false);
471 }
472
473 warn!(
474 tx_id = %tx.id,
475 relayer_id = %tx.relayer_id,
476 tx_nonce = tx_nonce,
477 on_chain_nonce = on_chain_nonce,
478 gap_count = gap_nonces.len(),
479 gaps = ?gap_nonces,
480 "nonce gaps confirmed below tx, scheduling nonce health to fill"
481 );
482
483 if let Err(e) = self.schedule_relayer_nonce_health_job(tx).await {
484 warn!(
485 tx_id = %tx.id,
486 error = %e,
487 "failed to schedule nonce health job for nonce gap"
488 );
489 }
490
491 Some(true)
492 }
493
494 async fn handle_resubmission(
496 &self,
497 tx: TransactionRepoModel,
498 ) -> Result<TransactionRepoModel, TransactionError> {
499 debug!(
500 tx_id = %tx.id,
501 relayer_id = %tx.relayer_id,
502 status = ?tx.status,
503 "scheduling resubmit job for transaction"
504 );
505
506 let (should_noop, reason) = self.should_noop(&tx).await?;
508 let tx_to_process = if should_noop {
509 self.process_noop_transaction(&tx, reason).await?
510 } else {
511 tx
512 };
513
514 self.send_transaction_resubmit_job(&tx_to_process).await?;
515 Ok(tx_to_process)
516 }
517
518 async fn process_noop_transaction(
520 &self,
521 tx: &TransactionRepoModel,
522 reason: Option<String>,
523 ) -> Result<TransactionRepoModel, TransactionError> {
524 debug!(
525 tx_id = %tx.id,
526 relayer_id = %tx.relayer_id,
527 status = ?tx.status,
528 "preparing transaction NOOP before resubmission"
529 );
530 let update = self.prepare_noop_update_request(tx, false, reason).await?;
531 let updated_tx = self
532 .transaction_repository()
533 .partial_update(tx.id.clone(), update)
534 .await?;
535
536 let res = self.send_transaction_update_notification(&updated_tx).await;
537 if let Err(e) = res {
538 error!(
539 tx_id = %updated_tx.id,
540 relayer_id = %updated_tx.relayer_id,
541 status = ?updated_tx.status,
542 error = %e,
543 "sending transaction update notification failed for NOOP transaction"
544 );
545 }
546 Ok(updated_tx)
547 }
548
549 async fn handle_pending_state(
551 &self,
552 tx: TransactionRepoModel,
553 ) -> Result<TransactionRepoModel, TransactionError> {
554 let (should_noop, reason) = self.should_noop(&tx).await?;
555 if should_noop {
556 debug!(
559 tx_id = %tx.id,
560 relayer_id = %tx.relayer_id,
561 reason = %reason.as_ref().unwrap_or(&"unknown".to_string()),
562 "marking pending transaction as Failed (nonce not assigned, no NOOP needed)"
563 );
564 let update = TransactionUpdateRequest {
565 status: Some(TransactionStatus::Failed),
566 status_reason: reason,
567 ..Default::default()
568 };
569 let updated_tx = self
570 .transaction_repository()
571 .partial_update(tx.id.clone(), update)
572 .await?;
573
574 let res = self.send_transaction_update_notification(&updated_tx).await;
575 if let Err(e) = res {
576 error!(
577 tx_id = %updated_tx.id,
578 relayer_id = %updated_tx.relayer_id,
579 status = ?updated_tx.status,
580 error = %e,
581 "sending transaction update notification failed for Pending state NOOP"
582 );
583 }
584 return Ok(updated_tx);
585 }
586
587 let age = get_age_since_created(&tx)?;
589 if age > get_evm_pending_recovery_trigger_timeout() {
590 warn!(
591 tx_id = %tx.id,
592 relayer_id = %tx.relayer_id,
593 age_seconds = age.num_seconds(),
594 "transaction stuck in Pending, queuing prepare job"
595 );
596
597 self.send_transaction_request_job(&tx).await?;
599 }
600
601 Ok(tx)
602 }
603
604 async fn handle_mined_state(
606 &self,
607 tx: TransactionRepoModel,
608 ) -> Result<TransactionRepoModel, TransactionError> {
609 self.update_transaction_status_if_needed(tx, TransactionStatus::Mined, None)
610 .await
611 }
612
613 async fn handle_final_state(
615 &self,
616 tx: TransactionRepoModel,
617 status: TransactionStatus,
618 status_reason: Option<String>,
619 ) -> Result<TransactionRepoModel, TransactionError> {
620 self.update_transaction_status_if_needed(tx, status, status_reason)
621 .await
622 }
623
624 async fn mark_as_failed(
626 &self,
627 tx: TransactionRepoModel,
628 reason: String,
629 ) -> Result<TransactionRepoModel, TransactionError> {
630 warn!(
631 tx_id = %tx.id,
632 relayer_id = %tx.relayer_id,
633 reason = %reason,
634 "force-failing transaction due to circuit breaker"
635 );
636
637 let update = TransactionUpdateRequest {
638 status: Some(TransactionStatus::Failed),
639 status_reason: Some(reason),
640 ..Default::default()
641 };
642
643 let updated_tx = self
644 .transaction_repository()
645 .partial_update(tx.id.clone(), update)
646 .await?;
647
648 if let Err(e) = self.send_transaction_update_notification(&updated_tx).await {
650 error!(
651 tx_id = %updated_tx.id,
652 relayer_id = %updated_tx.relayer_id,
653 error = %e,
654 "failed to send notification for force-failed transaction"
655 );
656 }
657
658 Ok(updated_tx)
659 }
660
661 async fn reconcile_tx_nonce_state(
672 &self,
673 tx: &TransactionRepoModel,
674 ) -> Result<Option<TransactionRepoModel>, TransactionError> {
675 let evm_data = tx.network_data.get_evm_transaction_data()?;
676
677 let mut had_rpc_errors = false;
681
682 if let Some(ref hash) = evm_data.hash {
684 match self.provider().get_transaction_receipt(hash).await {
685 Ok(Some(_)) => {
686 debug!(
687 tx_id = %tx.id,
688 hash = %hash,
689 "nonce recovery: receipt found for current hash, deferring to normal flow"
690 );
691 return Ok(None);
692 }
693 Ok(None) => {
694 }
696 Err(e) => {
697 warn!(
698 tx_id = %tx.id,
699 hash = %hash,
700 error = %e,
701 "nonce recovery: error checking receipt for current hash"
702 );
703 had_rpc_errors = true;
704 }
705 }
706 }
707
708 if tx.hashes.len() > 1 {
710 match self.try_recover_with_historical_hashes(tx, &evm_data).await {
711 Ok(Some(recovered_tx)) => {
712 debug!(
713 tx_id = %tx.id,
714 "nonce recovery: recovered transaction via historical hash"
715 );
716 return Ok(Some(recovered_tx));
717 }
718 Ok(None) => {
719 }
721 Err(e) => {
722 warn!(
723 tx_id = %tx.id,
724 error = %e,
725 "nonce recovery: error during historical hash recovery"
726 );
727 had_rpc_errors = true;
728 }
729 }
730 }
731
732 if had_rpc_errors {
736 warn!(
737 tx_id = %tx.id,
738 "nonce recovery: skipping nonce comparison due to RPC errors during hash checks, deferring to normal flow"
739 );
740 return Ok(None);
741 }
742
743 let tx_nonce = match evm_data.nonce {
744 Some(n) => n,
745 None => {
746 return Ok(None);
748 }
749 };
750
751 let on_chain_nonce = self
752 .provider()
753 .get_transaction_count(&self.relayer().address)
754 .await
755 .map_err(|e| {
756 TransactionError::UnexpectedError(format!(
757 "Failed to get on-chain nonce for recovery: {e}"
758 ))
759 })?;
760
761 if on_chain_nonce > tx_nonce {
762 let reason = format!(
764 "Nonce {tx_nonce} consumed externally (on-chain nonce: {on_chain_nonce}). \
765 No matching transaction hash found on-chain."
766 );
767 warn!(
768 tx_id = %tx.id,
769 relayer_id = %tx.relayer_id,
770 tx_nonce = tx_nonce,
771 on_chain_nonce = on_chain_nonce,
772 "nonce recovery: nonce consumed externally, marking as Failed"
773 );
774
775 let updated_tx = self
776 .update_transaction_status(tx.clone(), TransactionStatus::Failed, Some(reason))
777 .await?;
778
779 if let Err(e) = self.schedule_relayer_nonce_health_job(tx).await {
784 warn!(
785 tx_id = %tx.id,
786 error = %e,
787 "nonce recovery: failed to schedule nonce health after external consumption"
788 );
789 }
790
791 return Ok(Some(updated_tx));
792 }
793
794 debug!(
796 tx_id = %tx.id,
797 tx_nonce = tx_nonce,
798 on_chain_nonce = on_chain_nonce,
799 "nonce recovery: on-chain nonce not past tx nonce, deferring to normal flow"
800 );
801 Ok(None)
802 }
803
804 async fn handle_circuit_breaker_safely(
816 &self,
817 tx: TransactionRepoModel,
818 ctx: &StatusCheckContext,
819 ) -> Result<TransactionRepoModel, TransactionError> {
820 let reason = format!(
821 "Transaction status monitoring failed after {} consecutive errors (total: {}). \
822 Last status: {:?}.",
823 ctx.consecutive_failures, ctx.total_failures, tx.status
824 );
825
826 match tx.status {
827 TransactionStatus::Pending => {
828 debug!(
830 tx_id = %tx.id,
831 relayer_id = %tx.relayer_id,
832 "circuit breaker: Pending transaction (no nonce) - safe to mark as Failed"
833 );
834 self.mark_as_failed(tx, reason).await
835 }
836 TransactionStatus::Sent => {
837 let has_nonce = tx
841 .network_data
842 .get_evm_transaction_data()
843 .map(|d| d.nonce.is_some())
844 .unwrap_or(false);
845
846 if has_nonce {
847 warn!(
848 tx_id = %tx.id,
849 relayer_id = %tx.relayer_id,
850 "circuit breaker: Sent transaction with nonce assigned - triggering NOOP to clear nonce slot"
851 );
852 let noop_reason = Some(format!(
853 "{reason}. Replacing with NOOP to clear nonce slot (Sent state with assigned nonce)."
854 ));
855 let updated_tx = self.process_noop_transaction(&tx, noop_reason).await?;
856 self.send_transaction_resubmit_job(&updated_tx).await?;
860 Ok(updated_tx)
861 } else {
862 debug!(
864 tx_id = %tx.id,
865 relayer_id = %tx.relayer_id,
866 "circuit breaker: Sent transaction without nonce - safe to mark as Failed"
867 );
868 self.mark_as_failed(tx, reason).await
869 }
870 }
871 TransactionStatus::Submitted => {
872 warn!(
878 tx_id = %tx.id,
879 relayer_id = %tx.relayer_id,
880 "circuit breaker: Submitted transaction - triggering NOOP to safely clear nonce"
881 );
882 let noop_reason = Some(format!(
883 "{reason}. Replacing with NOOP to clear nonce slot."
884 ));
885 let updated_tx = self.process_noop_transaction(&tx, noop_reason).await?;
886 self.send_transaction_resubmit_job(&updated_tx).await?;
887 Ok(updated_tx)
888 }
889 _ => {
890 debug!(
892 tx_id = %tx.id,
893 relayer_id = %tx.relayer_id,
894 status = ?tx.status,
895 "circuit breaker: unexpected status, returning transaction unchanged"
896 );
897 Ok(tx)
898 }
899 }
900 }
901
902 pub async fn handle_status_impl(
907 &self,
908 tx: TransactionRepoModel,
909 context: Option<StatusCheckContext>,
910 ) -> Result<TransactionRepoModel, TransactionError> {
911 debug!(
912 tx_id = %tx.id,
913 relayer_id = %tx.relayer_id,
914 status = ?tx.status,
915 "checking transaction status"
916 );
917
918 if is_final_state(&tx.status) {
920 debug!(
921 tx_id = %tx.id,
922 relayer_id = %tx.relayer_id,
923 status = ?tx.status,
924 "transaction already in final state"
925 );
926 return Ok(tx);
927 }
928
929 if let Some(ref ctx) = context {
933 let is_noop_tx = tx
934 .network_data
935 .get_evm_transaction_data()
936 .map(|data| is_noop(&data))
937 .unwrap_or(false);
938
939 if ctx.should_force_finalize() && !is_noop_tx {
940 warn!(
941 tx_id = %tx.id,
942 consecutive_failures = ctx.consecutive_failures,
943 total_failures = ctx.total_failures,
944 max_consecutive = ctx.max_consecutive_failures,
945 status = ?tx.status,
946 "circuit breaker triggered - handling safely based on transaction state"
947 );
948 return self.handle_circuit_breaker_safely(tx, ctx).await;
949 }
950
951 if ctx.should_force_finalize() && is_noop_tx {
952 debug!(
953 tx_id = %tx.id,
954 consecutive_failures = ctx.consecutive_failures,
955 relayer_id = %tx.relayer_id,
956 "circuit breaker would trigger but transaction is NOOP - continuing with normal status logic"
957 );
958 }
959 }
960
961 if let Some(ref ctx) = context {
965 if let Some(ref metadata) = ctx.job_metadata {
966 if let Some(hint) = metadata.get(TX_NONCE_RECONCILE_TRIGGER) {
967 debug!(
968 tx_id = %tx.id,
969 hint = %hint,
970 "nonce recovery hint detected - performing nonce reconciliation"
971 );
972 match self.reconcile_tx_nonce_state(&tx).await {
973 Ok(Some(recovered_tx)) => {
974 return Ok(recovered_tx);
975 }
976 Ok(None) => {
977 debug!(
979 tx_id = %tx.id,
980 "nonce recovery did not resolve transaction, continuing normal flow"
981 );
982 }
983 Err(e) => {
984 warn!(
986 tx_id = %tx.id,
987 error = %e,
988 "nonce recovery failed, falling through to normal status flow"
989 );
990 }
991 }
992 }
993 }
994 }
995
996 let status = self.check_transaction_status(&tx).await?;
1001
1002 debug!(
1003 tx_id = %tx.id,
1004 previous_status = ?tx.status,
1005 new_status = ?status,
1006 relayer_id = %tx.relayer_id,
1007 "transaction status check completed"
1008 );
1009
1010 let tx = if status != tx.status {
1014 debug!(
1015 tx_id = %tx.id,
1016 old_status = ?tx.status,
1017 new_status = ?status,
1018 relayer_id = %tx.relayer_id,
1019 "status changed during check, reloading transaction from DB to ensure fresh data"
1020 );
1021 self.transaction_repository()
1022 .get_by_id(tx.id.clone())
1023 .await?
1024 } else {
1025 tx
1026 };
1027
1028 if is_too_early_to_resubmit(&tx)? && is_pending_transaction(&status) {
1033 return self
1035 .update_transaction_status_if_needed(tx, status, None)
1036 .await;
1037 }
1038
1039 match status {
1041 TransactionStatus::Pending => self.handle_pending_state(tx).await,
1042 TransactionStatus::Sent => self.handle_sent_state(tx).await,
1043 TransactionStatus::Submitted => self.handle_submitted_state(tx).await,
1044 TransactionStatus::Mined => self.handle_mined_state(tx).await,
1045 TransactionStatus::Failed => {
1046 let status_reason = if tx.status != TransactionStatus::Failed {
1049 Some("Transaction reverted on-chain (receipt status: failed)".to_string())
1050 } else {
1051 None
1052 };
1053 self.handle_final_state(tx, status, status_reason).await
1054 }
1055 TransactionStatus::Confirmed
1056 | TransactionStatus::Expired
1057 | TransactionStatus::Canceled => self.handle_final_state(tx, status, None).await,
1058 }
1059 }
1060
1061 async fn handle_sent_state(
1063 &self,
1064 tx: TransactionRepoModel,
1065 ) -> Result<TransactionRepoModel, TransactionError> {
1066 debug!(
1067 tx_id = %tx.id,
1068 relayer_id = %tx.relayer_id,
1069 "handling Sent state"
1070 );
1071
1072 let (should_noop, reason) = self.should_noop(&tx).await?;
1074 if should_noop {
1075 debug!(
1076 tx_id = %tx.id,
1077 relayer_id = %tx.relayer_id,
1078 "preparing NOOP for sent transaction"
1079 );
1080 let update = self.prepare_noop_update_request(&tx, false, reason).await?;
1081 let updated_tx = self
1082 .transaction_repository()
1083 .partial_update(tx.id.clone(), update)
1084 .await?;
1085
1086 self.send_transaction_submit_job(&updated_tx).await?;
1087 let res = self.send_transaction_update_notification(&updated_tx).await;
1088 if let Err(e) = res {
1089 error!(
1090 tx_id = %updated_tx.id,
1091 relayer_id = %updated_tx.relayer_id,
1092 status = ?updated_tx.status,
1093 error = %e,
1094 "sending transaction update notification failed for Sent state NOOP"
1095 );
1096 }
1097 return Ok(updated_tx);
1098 }
1099
1100 let age_since_sent = get_age_since_status_change(&tx)?;
1103
1104 if age_since_sent > get_evm_resend_timeout() {
1105 warn!(
1106 tx_id = %tx.id,
1107 relayer_id = %tx.relayer_id,
1108 age_seconds = age_since_sent.num_seconds(),
1109 "transaction stuck in Sent, queuing resubmit job with repricing"
1110 );
1111
1112 self.send_transaction_resubmit_job(&tx).await?;
1114 }
1115
1116 self.update_transaction_status_if_needed(tx, TransactionStatus::Sent, None)
1117 .await
1118 }
1119
1120 fn should_try_hash_recovery(
1127 &self,
1128 tx: &TransactionRepoModel,
1129 ) -> Result<bool, TransactionError> {
1130 if tx.status != TransactionStatus::Submitted {
1132 return Ok(false);
1133 }
1134
1135 if tx.hashes.len() <= 1 {
1137 return Ok(false);
1138 }
1139
1140 let age = get_age_of_sent_at(tx)?;
1142 let min_age_for_recovery = get_evm_min_age_for_hash_recovery();
1143
1144 if age < min_age_for_recovery {
1145 return Ok(false);
1146 }
1147
1148 if tx.hashes.len() < EVM_MIN_HASHES_FOR_RECOVERY {
1151 return Ok(false);
1152 }
1153
1154 Ok(true)
1155 }
1156
1157 async fn try_recover_with_historical_hashes(
1166 &self,
1167 tx: &TransactionRepoModel,
1168 evm_data: &crate::models::EvmTransactionData,
1169 ) -> Result<Option<TransactionRepoModel>, TransactionError> {
1170 warn!(
1171 tx_id = %tx.id,
1172 relayer_id = %tx.relayer_id,
1173 current_hash = ?evm_data.hash,
1174 total_hashes = %tx.hashes.len(),
1175 "attempting hash recovery - checking historical hashes"
1176 );
1177
1178 for (idx, historical_hash) in tx.hashes.iter().rev().enumerate() {
1180 if Some(historical_hash) == evm_data.hash.as_ref() {
1182 continue;
1183 }
1184
1185 debug!(
1186 tx_id = %tx.id,
1187 relayer_id = %tx.relayer_id,
1188 hash = %historical_hash,
1189 index = %idx,
1190 "checking historical hash"
1191 );
1192
1193 match self
1195 .provider()
1196 .get_transaction_receipt(historical_hash)
1197 .await
1198 {
1199 Ok(Some(receipt)) => {
1200 warn!(
1201 tx_id = %tx.id,
1202 relayer_id = %tx.relayer_id,
1203 mined_hash = %historical_hash,
1204 wrong_hash = ?evm_data.hash,
1205 block_number = ?receipt.block_number,
1206 "RECOVERED: found mined transaction with historical hash - correcting database"
1207 );
1208
1209 let updated_tx = self
1212 .update_transaction_with_corrected_hash(
1213 tx,
1214 evm_data,
1215 historical_hash,
1216 TransactionStatus::Mined,
1217 )
1218 .await?;
1219
1220 return Ok(Some(updated_tx));
1221 }
1222 Ok(None) => {
1223 continue;
1225 }
1226 Err(e) => {
1227 warn!(
1229 tx_id = %tx.id,
1230 relayer_id = %tx.relayer_id,
1231 hash = %historical_hash,
1232 error = %e,
1233 "error checking historical hash, continuing to next"
1234 );
1235 continue;
1236 }
1237 }
1238 }
1239
1240 debug!(
1242 tx_id = %tx.id,
1243 relayer_id = %tx.relayer_id,
1244 "hash recovery completed - no historical hashes found on-chain"
1245 );
1246 Ok(None)
1247 }
1248
1249 async fn update_transaction_with_corrected_hash(
1253 &self,
1254 tx: &TransactionRepoModel,
1255 evm_data: &crate::models::EvmTransactionData,
1256 correct_hash: &str,
1257 status: TransactionStatus,
1258 ) -> Result<TransactionRepoModel, TransactionError> {
1259 let mut corrected_data = evm_data.clone();
1260 corrected_data.hash = Some(correct_hash.to_string());
1261
1262 let updated_tx = self
1263 .transaction_repository()
1264 .partial_update(
1265 tx.id.clone(),
1266 TransactionUpdateRequest {
1267 network_data: Some(NetworkTransactionData::Evm(corrected_data)),
1268 status: Some(status),
1269 ..Default::default()
1270 },
1271 )
1272 .await?;
1273
1274 if let Err(e) = self.send_transaction_update_notification(&updated_tx).await {
1276 error!(
1277 tx_id = %updated_tx.id,
1278 relayer_id = %updated_tx.relayer_id,
1279 error = %e,
1280 "failed to send notification for hash recovery"
1281 );
1282 }
1283
1284 Ok(updated_tx)
1285 }
1286}
1287
1288#[cfg(test)]
1289mod tests {
1290 use crate::{
1291 config::{EvmNetworkConfig, NetworkConfigCommon},
1292 domain::transaction::evm::{EvmRelayerTransaction, MockPriceCalculatorTrait},
1293 jobs::MockJobProducerTrait,
1294 models::{
1295 evm::Speed, EvmTransactionData, NetworkConfigData, NetworkRepoModel,
1296 NetworkTransactionData, NetworkType, RelayerEvmPolicy, RelayerNetworkPolicy,
1297 RelayerRepoModel, RpcConfig, TransactionReceipt, TransactionRepoModel,
1298 TransactionStatus, U256,
1299 },
1300 repositories::{
1301 MockNetworkRepository, MockRelayerRepository, MockTransactionCounterTrait,
1302 MockTransactionRepository,
1303 },
1304 services::{provider::MockEvmProviderTrait, signer::MockSigner},
1305 };
1306 use alloy::{
1307 consensus::{Eip658Value, Receipt, ReceiptWithBloom},
1308 network::AnyReceiptEnvelope,
1309 primitives::{b256, Address, BlockHash, Bloom, TxHash},
1310 };
1311 use chrono::{Duration, Utc};
1312 use std::sync::Arc;
1313
1314 pub struct TestMocks {
1316 pub provider: MockEvmProviderTrait,
1317 pub relayer_repo: MockRelayerRepository,
1318 pub network_repo: MockNetworkRepository,
1319 pub tx_repo: MockTransactionRepository,
1320 pub job_producer: MockJobProducerTrait,
1321 pub signer: MockSigner,
1322 pub counter: MockTransactionCounterTrait,
1323 pub price_calc: MockPriceCalculatorTrait,
1324 }
1325
1326 pub fn default_test_mocks() -> TestMocks {
1329 TestMocks {
1330 provider: MockEvmProviderTrait::new(),
1331 relayer_repo: MockRelayerRepository::new(),
1332 network_repo: MockNetworkRepository::new(),
1333 tx_repo: MockTransactionRepository::new(),
1334 job_producer: MockJobProducerTrait::new(),
1335 signer: MockSigner::new(),
1336 counter: MockTransactionCounterTrait::new(),
1337 price_calc: MockPriceCalculatorTrait::new(),
1338 }
1339 }
1340
1341 pub fn default_test_mocks_with_network() -> TestMocks {
1343 let mut mocks = default_test_mocks();
1344 mocks
1346 .network_repo
1347 .expect_get_by_chain_id()
1348 .returning(|network_type, chain_id| {
1349 if network_type == NetworkType::Evm && chain_id == 1 {
1350 Ok(Some(create_test_network_model()))
1351 } else {
1352 Ok(None)
1353 }
1354 });
1355 mocks
1356 }
1357
1358 pub fn create_test_network_model() -> NetworkRepoModel {
1360 let evm_config = EvmNetworkConfig {
1361 common: NetworkConfigCommon {
1362 network: "mainnet".to_string(),
1363 from: None,
1364 rpc_urls: Some(vec![RpcConfig::new("https://rpc.example.com".to_string())]),
1365 explorer_urls: Some(vec!["https://explorer.example.com".to_string()]),
1366 average_blocktime_ms: Some(12000),
1367 is_testnet: Some(false),
1368 tags: Some(vec!["mainnet".to_string()]),
1369 },
1370 chain_id: Some(1),
1371 required_confirmations: Some(12),
1372 features: Some(vec!["eip1559".to_string()]),
1373 symbol: Some("ETH".to_string()),
1374 gas_price_cache: None,
1375 };
1376 NetworkRepoModel {
1377 id: "evm:mainnet".to_string(),
1378 name: "mainnet".to_string(),
1379 network_type: NetworkType::Evm,
1380 config: NetworkConfigData::Evm(evm_config),
1381 }
1382 }
1383
1384 pub fn create_test_no_mempool_network_model() -> NetworkRepoModel {
1386 let evm_config = EvmNetworkConfig {
1387 common: NetworkConfigCommon {
1388 network: "arbitrum".to_string(),
1389 from: None,
1390 rpc_urls: Some(vec![crate::models::RpcConfig::new(
1391 "https://arb-rpc.example.com".to_string(),
1392 )]),
1393 explorer_urls: Some(vec!["https://arb-explorer.example.com".to_string()]),
1394 average_blocktime_ms: Some(1000),
1395 is_testnet: Some(false),
1396 tags: Some(vec![
1397 "arbitrum".to_string(),
1398 "rollup".to_string(),
1399 "no-mempool".to_string(),
1400 ]),
1401 },
1402 chain_id: Some(42161),
1403 required_confirmations: Some(12),
1404 features: Some(vec!["eip1559".to_string()]),
1405 symbol: Some("ETH".to_string()),
1406 gas_price_cache: None,
1407 };
1408 NetworkRepoModel {
1409 id: "evm:arbitrum".to_string(),
1410 name: "arbitrum".to_string(),
1411 network_type: NetworkType::Evm,
1412 config: NetworkConfigData::Evm(evm_config),
1413 }
1414 }
1415
1416 pub fn make_test_transaction(status: TransactionStatus) -> TransactionRepoModel {
1420 TransactionRepoModel {
1421 id: "test-tx-id".to_string(),
1422 relayer_id: "test-relayer-id".to_string(),
1423 status,
1424 status_reason: None,
1425 created_at: Utc::now().to_rfc3339(),
1426 sent_at: None,
1427 confirmed_at: None,
1428 valid_until: None,
1429 delete_at: None,
1430 network_type: NetworkType::Evm,
1431 network_data: NetworkTransactionData::Evm(EvmTransactionData {
1432 chain_id: 1,
1433 from: "0xSender".to_string(),
1434 to: Some("0xRecipient".to_string()),
1435 value: U256::from(0),
1436 data: Some("0xData".to_string()),
1437 gas_limit: Some(21000),
1438 gas_price: Some(20000000000),
1439 max_fee_per_gas: None,
1440 max_priority_fee_per_gas: None,
1441 nonce: None,
1442 signature: None,
1443 hash: None,
1444 speed: Some(Speed::Fast),
1445 raw: None,
1446 }),
1447 priced_at: None,
1448 hashes: Vec::new(),
1449 noop_count: None,
1450 is_canceled: Some(false),
1451 metadata: None,
1452 }
1453 }
1454
1455 pub fn make_test_evm_relayer_transaction(
1458 relayer: RelayerRepoModel,
1459 mocks: TestMocks,
1460 ) -> EvmRelayerTransaction<
1461 MockEvmProviderTrait,
1462 MockRelayerRepository,
1463 MockNetworkRepository,
1464 MockTransactionRepository,
1465 MockJobProducerTrait,
1466 MockSigner,
1467 MockTransactionCounterTrait,
1468 MockPriceCalculatorTrait,
1469 > {
1470 EvmRelayerTransaction::new(
1471 relayer,
1472 mocks.provider,
1473 Arc::new(mocks.relayer_repo),
1474 Arc::new(mocks.network_repo),
1475 Arc::new(mocks.tx_repo),
1476 Arc::new(mocks.counter),
1477 Arc::new(mocks.job_producer),
1478 mocks.price_calc,
1479 mocks.signer,
1480 )
1481 .unwrap()
1482 }
1483
1484 fn create_test_relayer() -> RelayerRepoModel {
1485 RelayerRepoModel {
1486 id: "test-relayer-id".to_string(),
1487 name: "Test Relayer".to_string(),
1488 paused: false,
1489 system_disabled: false,
1490 network: "test_network".to_string(),
1491 network_type: NetworkType::Evm,
1492 policies: RelayerNetworkPolicy::Evm(RelayerEvmPolicy::default()),
1493 signer_id: "test_signer".to_string(),
1494 address: "0x".to_string(),
1495 notification_id: None,
1496 custom_rpc_urls: None,
1497 ..Default::default()
1498 }
1499 }
1500
1501 fn make_mock_receipt(status: bool, block_number: Option<u64>) -> TransactionReceipt {
1502 let tx_hash = TxHash::from(b256!(
1504 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
1505 ));
1506 let block_hash = BlockHash::from(b256!(
1507 "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
1508 ));
1509 let from_address = Address::from([0x11; 20]);
1510
1511 TransactionReceipt {
1512 inner: alloy::rpc::types::TransactionReceipt {
1513 inner: AnyReceiptEnvelope {
1514 inner: ReceiptWithBloom {
1515 receipt: Receipt {
1516 status: Eip658Value::Eip658(status), cumulative_gas_used: 0,
1518 logs: vec![],
1519 },
1520 logs_bloom: Bloom::ZERO,
1521 },
1522 r#type: 0, },
1524 transaction_hash: tx_hash,
1525 transaction_index: Some(0),
1526 block_hash: block_number.map(|_| block_hash), block_number,
1528 gas_used: 21000,
1529 effective_gas_price: 1000,
1530 blob_gas_used: None,
1531 blob_gas_price: None,
1532 from: from_address,
1533 to: None,
1534 contract_address: None,
1535 },
1536 other: Default::default(),
1537 }
1538 }
1539
1540 mod check_transaction_status_tests {
1542 use super::*;
1543
1544 #[tokio::test]
1545 async fn test_not_mined() {
1546 let mut mocks = default_test_mocks();
1547 let relayer = create_test_relayer();
1548 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1549
1550 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1552 evm_data.hash = Some("0xFakeHash".to_string());
1553 }
1554
1555 mocks
1557 .provider
1558 .expect_get_transaction_receipt()
1559 .returning(|_| Box::pin(async { Ok(None) }));
1560
1561 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1562
1563 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1564 assert_eq!(status, TransactionStatus::Submitted);
1565 }
1566
1567 #[tokio::test]
1568 async fn test_mined_but_not_confirmed() {
1569 let mut mocks = default_test_mocks();
1570 let relayer = create_test_relayer();
1571 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1572
1573 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1574 evm_data.hash = Some("0xFakeHash".to_string());
1575 }
1576
1577 mocks
1579 .provider
1580 .expect_get_transaction_receipt()
1581 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1582
1583 mocks
1585 .provider
1586 .expect_get_block_number()
1587 .return_once(|| Box::pin(async { Ok(100) }));
1588
1589 mocks
1591 .network_repo
1592 .expect_get_by_chain_id()
1593 .returning(|_, _| Ok(Some(create_test_network_model())));
1594
1595 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1596
1597 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1598 assert_eq!(status, TransactionStatus::Mined);
1599 }
1600
1601 #[tokio::test]
1602 async fn test_confirmed() {
1603 let mut mocks = default_test_mocks();
1604 let relayer = create_test_relayer();
1605 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1606
1607 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1608 evm_data.hash = Some("0xFakeHash".to_string());
1609 }
1610
1611 mocks
1613 .provider
1614 .expect_get_transaction_receipt()
1615 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1616
1617 mocks
1619 .provider
1620 .expect_get_block_number()
1621 .return_once(|| Box::pin(async { Ok(113) }));
1622
1623 mocks
1625 .network_repo
1626 .expect_get_by_chain_id()
1627 .returning(|_, _| Ok(Some(create_test_network_model())));
1628
1629 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1630
1631 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1632 assert_eq!(status, TransactionStatus::Confirmed);
1633 }
1634
1635 #[tokio::test]
1639 async fn test_cancellation_noop_confirmed_becomes_canceled() {
1640 let mut mocks = default_test_mocks();
1641 let relayer = create_test_relayer();
1642 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1643 tx.is_canceled = Some(true);
1644
1645 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1647 evm_data.hash = Some("0xNoopHash".to_string());
1648 evm_data.value = U256::from(0);
1649 evm_data.data = Some("0x".to_string());
1650 evm_data.to = Some(evm_data.from.clone());
1651 }
1652
1653 mocks
1654 .provider
1655 .expect_get_transaction_receipt()
1656 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1657 mocks
1658 .provider
1659 .expect_get_block_number()
1660 .return_once(|| Box::pin(async { Ok(113) }));
1661 mocks
1662 .network_repo
1663 .expect_get_by_chain_id()
1664 .returning(|_, _| Ok(Some(create_test_network_model())));
1665
1666 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1667
1668 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1669 assert_eq!(status, TransactionStatus::Canceled);
1670 }
1671
1672 #[tokio::test]
1675 async fn test_non_cancellation_noop_confirmed_stays_confirmed() {
1676 let mut mocks = default_test_mocks();
1677 let relayer = create_test_relayer();
1678 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1679 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1682 evm_data.hash = Some("0xNoopHash".to_string());
1683 evm_data.value = U256::from(0);
1684 evm_data.data = Some("0x".to_string());
1685 evm_data.to = Some(evm_data.from.clone());
1686 }
1687
1688 mocks
1689 .provider
1690 .expect_get_transaction_receipt()
1691 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
1692 mocks
1693 .provider
1694 .expect_get_block_number()
1695 .return_once(|| Box::pin(async { Ok(113) }));
1696 mocks
1697 .network_repo
1698 .expect_get_by_chain_id()
1699 .returning(|_, _| Ok(Some(create_test_network_model())));
1700
1701 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1702
1703 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1704 assert_eq!(status, TransactionStatus::Confirmed);
1705 }
1706
1707 #[tokio::test]
1708 async fn test_failed() {
1709 let mut mocks = default_test_mocks();
1710 let relayer = create_test_relayer();
1711 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1712
1713 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1714 evm_data.hash = Some("0xFakeHash".to_string());
1715 }
1716
1717 mocks
1719 .provider
1720 .expect_get_transaction_receipt()
1721 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
1722
1723 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1724
1725 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
1726 assert_eq!(status, TransactionStatus::Failed);
1727 }
1728 }
1729
1730 mod should_resubmit_tests {
1732 use super::*;
1733 use crate::models::TransactionError;
1734
1735 #[tokio::test]
1736 async fn test_should_resubmit_true() {
1737 let mut mocks = default_test_mocks();
1738 let relayer = create_test_relayer();
1739
1740 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1742 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1743
1744 mocks
1746 .network_repo
1747 .expect_get_by_chain_id()
1748 .returning(|_, _| Ok(Some(create_test_network_model())));
1749
1750 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1751 let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1752 assert!(res, "Transaction should be resubmitted after timeout.");
1753 }
1754
1755 #[tokio::test]
1756 async fn test_should_resubmit_false() {
1757 let mut mocks = default_test_mocks();
1758 let relayer = create_test_relayer();
1759
1760 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1762 tx.sent_at = Some(Utc::now().to_rfc3339());
1763
1764 mocks
1766 .network_repo
1767 .expect_get_by_chain_id()
1768 .returning(|_, _| Ok(Some(create_test_network_model())));
1769
1770 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1771 let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1772 assert!(!res, "Transaction should not be resubmitted immediately.");
1773 }
1774
1775 #[tokio::test]
1776 async fn test_should_resubmit_true_for_no_mempool_network() {
1777 let mut mocks = default_test_mocks();
1778 let relayer = create_test_relayer();
1779
1780 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1782 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1783
1784 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1786 evm_data.chain_id = 42161; }
1788
1789 mocks
1791 .network_repo
1792 .expect_get_by_chain_id()
1793 .returning(|_, _| Ok(Some(create_test_no_mempool_network_model())));
1794
1795 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1796 let res = evm_transaction.should_resubmit(&tx).await.unwrap();
1797 assert!(
1798 res,
1799 "Transaction should be resubmitted for no-mempool networks."
1800 );
1801 }
1802
1803 #[tokio::test]
1804 async fn test_should_resubmit_network_not_found() {
1805 let mut mocks = default_test_mocks();
1806 let relayer = create_test_relayer();
1807
1808 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1809 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1810
1811 mocks
1813 .network_repo
1814 .expect_get_by_chain_id()
1815 .returning(|_, _| Ok(None));
1816
1817 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1818 let result = evm_transaction.should_resubmit(&tx).await;
1819
1820 assert!(
1821 result.is_err(),
1822 "should_resubmit should return error when network not found"
1823 );
1824 let error = result.unwrap_err();
1825 match error {
1826 TransactionError::UnexpectedError(msg) => {
1827 assert!(msg.contains("Network with chain id 1 not found"));
1828 }
1829 _ => panic!("Expected UnexpectedError for network not found"),
1830 }
1831 }
1832
1833 #[tokio::test]
1834 async fn test_should_resubmit_network_conversion_error() {
1835 let mut mocks = default_test_mocks();
1836 let relayer = create_test_relayer();
1837
1838 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1839 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
1840
1841 let invalid_evm_config = EvmNetworkConfig {
1843 common: NetworkConfigCommon {
1844 network: "invalid-network".to_string(),
1845 from: None,
1846 rpc_urls: Some(vec![crate::models::RpcConfig::new(
1847 "https://rpc.example.com".to_string(),
1848 )]),
1849 explorer_urls: Some(vec!["https://explorer.example.com".to_string()]),
1850 average_blocktime_ms: Some(12000),
1851 is_testnet: Some(false),
1852 tags: Some(vec!["testnet".to_string()]),
1853 },
1854 chain_id: None, required_confirmations: Some(12),
1856 features: Some(vec!["eip1559".to_string()]),
1857 symbol: Some("ETH".to_string()),
1858 gas_price_cache: None,
1859 };
1860 let invalid_network = NetworkRepoModel {
1861 id: "evm:invalid".to_string(),
1862 name: "invalid-network".to_string(),
1863 network_type: NetworkType::Evm,
1864 config: NetworkConfigData::Evm(invalid_evm_config),
1865 };
1866
1867 mocks
1869 .network_repo
1870 .expect_get_by_chain_id()
1871 .returning(move |_, _| Ok(Some(invalid_network.clone())));
1872
1873 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1874 let result = evm_transaction.should_resubmit(&tx).await;
1875
1876 assert!(
1877 result.is_err(),
1878 "should_resubmit should return error when network conversion fails"
1879 );
1880 let error = result.unwrap_err();
1881 match error {
1882 TransactionError::UnexpectedError(msg) => {
1883 assert!(msg.contains("Error converting network model to EvmNetwork"));
1884 }
1885 _ => panic!("Expected UnexpectedError for network conversion failure"),
1886 }
1887 }
1888 }
1889
1890 mod should_noop_tests {
1892 use super::*;
1893
1894 #[tokio::test]
1895 async fn test_expired_transaction_triggers_noop() {
1896 let mut mocks = default_test_mocks();
1897 let relayer = create_test_relayer();
1898
1899 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1900 tx.valid_until = Some((Utc::now() - Duration::seconds(10)).to_rfc3339());
1902
1903 mocks
1905 .network_repo
1906 .expect_get_by_chain_id()
1907 .returning(|_, _| Ok(Some(create_test_network_model())));
1908
1909 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1910 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
1911 assert!(res, "Expired transaction should be replaced with a NOOP.");
1912 assert!(
1913 reason.is_some(),
1914 "Reason should be provided for expired transaction"
1915 );
1916 assert!(
1917 reason.unwrap().contains("expired"),
1918 "Reason should mention expiration"
1919 );
1920 }
1921
1922 #[tokio::test]
1923 async fn test_too_many_noop_attempts_returns_false() {
1924 let mocks = default_test_mocks();
1925 let relayer = create_test_relayer();
1926
1927 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1928 tx.noop_count = Some(51); let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1931 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
1932 assert!(
1933 !res,
1934 "Transaction with too many NOOP attempts should not be replaced."
1935 );
1936 assert!(
1937 reason.is_none(),
1938 "Reason should not be provided when should_noop is false"
1939 );
1940 }
1941
1942 #[tokio::test]
1943 async fn test_already_noop_returns_false() {
1944 let mut mocks = default_test_mocks();
1945 let relayer = create_test_relayer();
1946
1947 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1948 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1950 evm_data.to = None;
1951 evm_data.value = U256::from(0);
1952 }
1953
1954 mocks
1955 .network_repo
1956 .expect_get_by_chain_id()
1957 .returning(|_, _| Ok(Some(create_test_network_model())));
1958
1959 mocks.provider.expect_get_block_by_number().returning(|| {
1961 Box::pin(async {
1962 use alloy::{network::AnyRpcBlock, rpc::types::Block};
1963 let mut block: Block = Block::default();
1964 block.header.gas_limit = 30_000_000u64;
1965 Ok(AnyRpcBlock::from(block))
1966 })
1967 });
1968
1969 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
1970 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
1971 assert!(
1972 !res,
1973 "Transaction that is already a NOOP should not be replaced."
1974 );
1975 assert!(
1976 reason.is_none(),
1977 "Reason should not be provided when should_noop is false"
1978 );
1979 }
1980
1981 #[tokio::test]
1982 async fn test_rollup_with_too_many_attempts_triggers_noop() {
1983 let mut mocks = default_test_mocks();
1984 let relayer = create_test_relayer();
1985
1986 let mut tx = make_test_transaction(TransactionStatus::Submitted);
1987 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
1989 evm_data.chain_id = 42161; }
1991 tx.hashes = vec!["0xHash1".to_string(); 51];
1993
1994 mocks
1996 .network_repo
1997 .expect_get_by_chain_id()
1998 .returning(|_, _| Ok(Some(create_test_no_mempool_network_model())));
1999
2000 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2001 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2002 assert!(
2003 res,
2004 "Rollup transaction with too many attempts should be replaced with NOOP."
2005 );
2006 assert!(
2007 reason.is_some(),
2008 "Reason should be provided for rollup transaction"
2009 );
2010 assert!(
2011 reason.unwrap().contains("too many attempts"),
2012 "Reason should mention too many attempts"
2013 );
2014 }
2015
2016 #[tokio::test]
2017 async fn test_pending_state_timeout_triggers_noop() {
2018 let mut mocks = default_test_mocks();
2019 let relayer = create_test_relayer();
2020
2021 let mut tx = make_test_transaction(TransactionStatus::Pending);
2022 tx.created_at = (Utc::now() - Duration::minutes(3)).to_rfc3339();
2024
2025 mocks
2026 .network_repo
2027 .expect_get_by_chain_id()
2028 .returning(|_, _| Ok(Some(create_test_network_model())));
2029
2030 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2031 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2032 assert!(
2033 res,
2034 "Pending transaction stuck for >2 minutes should be replaced with NOOP."
2035 );
2036 assert!(
2037 reason.is_some(),
2038 "Reason should be provided for pending timeout"
2039 );
2040 assert!(
2041 reason.unwrap().contains("Pending state"),
2042 "Reason should mention Pending state"
2043 );
2044 }
2045
2046 #[tokio::test]
2047 async fn test_valid_transaction_returns_false() {
2048 let mut mocks = default_test_mocks();
2049 let relayer = create_test_relayer();
2050
2051 let tx = make_test_transaction(TransactionStatus::Submitted);
2052 mocks
2055 .network_repo
2056 .expect_get_by_chain_id()
2057 .returning(|_, _| Ok(Some(create_test_network_model())));
2058
2059 mocks.provider.expect_get_block_by_number().returning(|| {
2061 Box::pin(async {
2062 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2063 let mut block: Block = Block::default();
2064 block.header.gas_limit = 30_000_000u64;
2065 Ok(AnyRpcBlock::from(block))
2066 })
2067 });
2068
2069 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2070 let (res, reason) = evm_transaction.should_noop(&tx).await.unwrap();
2071 assert!(!res, "Valid transaction should not be replaced with NOOP.");
2072 assert!(
2073 reason.is_none(),
2074 "Reason should not be provided when should_noop is false"
2075 );
2076 }
2077 }
2078
2079 mod update_transaction_status_tests {
2081 use super::*;
2082
2083 #[tokio::test]
2084 async fn test_no_update_when_status_is_same() {
2085 let mocks = default_test_mocks();
2087 let relayer = create_test_relayer();
2088 let tx = make_test_transaction(TransactionStatus::Submitted);
2089 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2090
2091 let updated_tx = evm_transaction
2094 .update_transaction_status_if_needed(tx.clone(), TransactionStatus::Submitted, None)
2095 .await
2096 .unwrap();
2097 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2098 assert_eq!(updated_tx.id, tx.id);
2099 }
2100
2101 #[tokio::test]
2102 async fn test_updates_when_status_differs() {
2103 let mut mocks = default_test_mocks();
2104 let relayer = create_test_relayer();
2105 let tx = make_test_transaction(TransactionStatus::Submitted);
2106
2107 mocks
2109 .tx_repo
2110 .expect_partial_update()
2111 .returning(|_, update| {
2112 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2113 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2114 Ok(updated_tx)
2115 });
2116
2117 mocks
2119 .job_producer
2120 .expect_produce_send_notification_job()
2121 .returning(|_, _| Box::pin(async { Ok(()) }));
2122
2123 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2124 let updated_tx = evm_transaction
2125 .update_transaction_status_if_needed(tx.clone(), TransactionStatus::Mined, None)
2126 .await
2127 .unwrap();
2128
2129 assert_eq!(updated_tx.status, TransactionStatus::Mined);
2130 }
2131
2132 #[tokio::test]
2133 async fn test_updates_with_status_reason() {
2134 let mut mocks = default_test_mocks();
2135 let relayer = create_test_relayer();
2136 let tx = make_test_transaction(TransactionStatus::Submitted);
2137
2138 mocks
2139 .tx_repo
2140 .expect_partial_update()
2141 .withf(|_, update| {
2142 update.status == Some(TransactionStatus::Failed)
2143 && update.status_reason == Some("Transaction reverted on-chain".to_string())
2144 })
2145 .returning(|_, update| {
2146 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2147 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2148 updated_tx.status_reason = update.status_reason.clone();
2149 Ok(updated_tx)
2150 });
2151
2152 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2153 let updated_tx = evm_transaction
2154 .update_transaction_status_if_needed(
2155 tx.clone(),
2156 TransactionStatus::Failed,
2157 Some("Transaction reverted on-chain".to_string()),
2158 )
2159 .await
2160 .unwrap();
2161
2162 assert_eq!(updated_tx.status, TransactionStatus::Failed);
2163 assert_eq!(
2164 updated_tx.status_reason.as_deref(),
2165 Some("Transaction reverted on-chain")
2166 );
2167 }
2168 }
2169
2170 mod handle_sent_state_tests {
2172 use super::*;
2173
2174 #[tokio::test]
2175 async fn test_sent_state_recent_no_resend() {
2176 let mut mocks = default_test_mocks();
2177 let relayer = create_test_relayer();
2178
2179 let mut tx = make_test_transaction(TransactionStatus::Sent);
2180 tx.sent_at = Some((Utc::now() - Duration::seconds(10)).to_rfc3339());
2182
2183 mocks
2185 .network_repo
2186 .expect_get_by_chain_id()
2187 .returning(|_, _| Ok(Some(create_test_network_model())));
2188
2189 mocks.provider.expect_get_block_by_number().returning(|| {
2191 Box::pin(async {
2192 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2193 let mut block: Block = Block::default();
2194 block.header.gas_limit = 30_000_000u64;
2195 Ok(AnyRpcBlock::from(block))
2196 })
2197 });
2198
2199 mocks
2201 .job_producer
2202 .expect_produce_check_transaction_status_job()
2203 .returning(|_, _| Box::pin(async { Ok(()) }));
2204
2205 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2206 let result = evm_transaction.handle_sent_state(tx.clone()).await.unwrap();
2207
2208 assert_eq!(result.status, TransactionStatus::Sent);
2209 }
2210
2211 #[tokio::test]
2212 async fn test_sent_state_stuck_schedules_resubmit() {
2213 let mut mocks = default_test_mocks();
2214 let relayer = create_test_relayer();
2215
2216 let mut tx = make_test_transaction(TransactionStatus::Sent);
2217 tx.sent_at = Some((Utc::now() - Duration::seconds(60)).to_rfc3339());
2219
2220 mocks
2222 .network_repo
2223 .expect_get_by_chain_id()
2224 .returning(|_, _| Ok(Some(create_test_network_model())));
2225
2226 mocks.provider.expect_get_block_by_number().returning(|| {
2228 Box::pin(async {
2229 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2230 let mut block: Block = Block::default();
2231 block.header.gas_limit = 30_000_000u64;
2232 Ok(AnyRpcBlock::from(block))
2233 })
2234 });
2235
2236 mocks
2238 .job_producer
2239 .expect_produce_submit_transaction_job()
2240 .returning(|_, _| Box::pin(async { Ok(()) }));
2241
2242 mocks
2244 .job_producer
2245 .expect_produce_check_transaction_status_job()
2246 .returning(|_, _| Box::pin(async { Ok(()) }));
2247
2248 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2249 let result = evm_transaction.handle_sent_state(tx.clone()).await.unwrap();
2250
2251 assert_eq!(result.status, TransactionStatus::Sent);
2252 }
2253 }
2254
2255 mod prepare_noop_update_request_tests {
2257 use super::*;
2258
2259 #[tokio::test]
2260 async fn test_noop_request_without_cancellation() {
2261 let mocks = default_test_mocks_with_network();
2263 let relayer = create_test_relayer();
2264 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2265 tx.noop_count = Some(2);
2266 tx.is_canceled = Some(false);
2267
2268 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2269 let update_req = evm_transaction
2270 .prepare_noop_update_request(&tx, false, None)
2271 .await
2272 .unwrap();
2273
2274 assert_eq!(update_req.noop_count, Some(3));
2276 assert_eq!(update_req.is_canceled, Some(false));
2278 }
2279
2280 #[tokio::test]
2281 async fn test_noop_request_with_cancellation() {
2282 let mocks = default_test_mocks_with_network();
2284 let relayer = create_test_relayer();
2285 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2286 tx.noop_count = None;
2287 tx.is_canceled = Some(false);
2288
2289 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2290 let update_req = evm_transaction
2291 .prepare_noop_update_request(&tx, true, None)
2292 .await
2293 .unwrap();
2294
2295 assert_eq!(update_req.noop_count, Some(1));
2297 assert_eq!(update_req.is_canceled, Some(true));
2299 }
2300 }
2301
2302 mod handle_submitted_state_tests {
2304 use super::*;
2305
2306 #[tokio::test]
2307 async fn test_schedules_resubmit_job() {
2308 let mut mocks = default_test_mocks();
2309 let relayer = create_test_relayer();
2310
2311 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2313 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2314
2315 mocks
2317 .network_repo
2318 .expect_get_by_chain_id()
2319 .returning(|_, _| Ok(Some(create_test_network_model())));
2320
2321 mocks.provider.expect_get_block_by_number().returning(|| {
2323 Box::pin(async {
2324 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2325 let mut block: Block = Block::default();
2326 block.header.gas_limit = 30_000_000u64;
2327 Ok(AnyRpcBlock::from(block))
2328 })
2329 });
2330
2331 mocks
2333 .provider
2334 .expect_get_transaction_count()
2335 .returning(|_| Box::pin(async { Ok(10) }));
2336
2337 mocks
2339 .job_producer
2340 .expect_produce_submit_transaction_job()
2341 .returning(|_, _| Box::pin(async { Ok(()) }));
2342
2343 mocks
2345 .job_producer
2346 .expect_produce_check_transaction_status_job()
2347 .returning(|_, _| Box::pin(async { Ok(()) }));
2348
2349 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2350 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2351
2352 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2354 }
2355
2356 #[tokio::test]
2359 async fn test_nonce_gap_detected_schedules_health_skips_resubmit() {
2360 let mut mocks = default_test_mocks();
2361 let relayer = create_test_relayer();
2362
2363 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2364 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2365 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
2366 nonce: Some(274),
2367 hash: Some("0xhash".to_string()),
2368 raw: Some(vec![1, 2, 3]),
2369 ..tx.network_data.get_evm_transaction_data().unwrap()
2370 });
2371
2372 mocks
2374 .network_repo
2375 .expect_get_by_chain_id()
2376 .returning(|_, _| Ok(Some(create_test_network_model())));
2377
2378 mocks
2380 .provider
2381 .expect_get_transaction_count()
2382 .returning(|_| Box::pin(async { Ok(269) }));
2383
2384 mocks
2386 .tx_repo
2387 .expect_get_nonce_occupancy()
2388 .withf(|relayer_id, from, to| {
2389 relayer_id == "test-relayer-id" && *from == 269 && *to == 274
2390 })
2391 .returning(|_, from, to| Ok((from..to).map(|n| (n, None)).collect()));
2392
2393 mocks
2395 .job_producer
2396 .expect_produce_relayer_health_check_job()
2397 .withf(|job, _| {
2398 job.metadata.as_ref().map_or(false, |m| {
2399 m.get("health_check_action") == Some(&"nonce_health".to_string())
2400 })
2401 })
2402 .returning(|_, _| Box::pin(async { Ok(()) }));
2403
2404 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2408 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2409
2410 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2411 }
2412
2413 #[tokio::test]
2416 async fn test_nonce_gap_check_rpc_failure_proceeds_to_resubmit() {
2417 let mut mocks = default_test_mocks();
2418 let relayer = create_test_relayer();
2419
2420 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2421 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2422
2423 mocks
2424 .network_repo
2425 .expect_get_by_chain_id()
2426 .returning(|_, _| Ok(Some(create_test_network_model())));
2427
2428 mocks.provider.expect_get_block_by_number().returning(|| {
2429 Box::pin(async {
2430 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2431 let mut block: Block = Block::default();
2432 block.header.gas_limit = 30_000_000u64;
2433 Ok(AnyRpcBlock::from(block))
2434 })
2435 });
2436
2437 mocks
2439 .provider
2440 .expect_get_transaction_count()
2441 .returning(|_| {
2442 Box::pin(async {
2443 Err(crate::services::provider::ProviderError::Other(
2444 "rpc timeout".to_string(),
2445 ))
2446 })
2447 });
2448
2449 mocks
2451 .job_producer
2452 .expect_produce_submit_transaction_job()
2453 .returning(|_, _| Box::pin(async { Ok(()) }));
2454
2455 mocks
2456 .job_producer
2457 .expect_produce_check_transaction_status_job()
2458 .returning(|_, _| Box::pin(async { Ok(()) }));
2459
2460 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2461 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2462
2463 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2464 }
2465
2466 #[tokio::test]
2469 async fn test_no_gap_when_slots_filled_proceeds_to_resubmit() {
2470 let mut mocks = default_test_mocks();
2471 let relayer = create_test_relayer();
2472
2473 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2474 tx.sent_at = Some((Utc::now() - Duration::seconds(600)).to_rfc3339());
2475 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
2476 nonce: Some(270),
2477 hash: Some("0xhash".to_string()),
2478 raw: Some(vec![1, 2, 3]),
2479 ..tx.network_data.get_evm_transaction_data().unwrap()
2480 });
2481
2482 mocks
2483 .network_repo
2484 .expect_get_by_chain_id()
2485 .returning(|_, _| Ok(Some(create_test_network_model())));
2486
2487 mocks.provider.expect_get_block_by_number().returning(|| {
2488 Box::pin(async {
2489 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2490 let mut block: Block = Block::default();
2491 block.header.gas_limit = 30_000_000u64;
2492 Ok(AnyRpcBlock::from(block))
2493 })
2494 });
2495
2496 mocks
2498 .provider
2499 .expect_get_transaction_count()
2500 .returning(|_| Box::pin(async { Ok(269) }));
2501
2502 mocks
2504 .tx_repo
2505 .expect_get_nonce_occupancy()
2506 .returning(|_, from, to| {
2507 Ok((from..to)
2508 .map(|n| (n, Some(TransactionStatus::Submitted)))
2509 .collect())
2510 });
2511
2512 mocks
2514 .job_producer
2515 .expect_produce_submit_transaction_job()
2516 .returning(|_, _| Box::pin(async { Ok(()) }));
2517
2518 mocks
2519 .job_producer
2520 .expect_produce_check_transaction_status_job()
2521 .returning(|_, _| Box::pin(async { Ok(()) }));
2522
2523 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2524 let updated_tx = evm_transaction.handle_submitted_state(tx).await.unwrap();
2525
2526 assert_eq!(updated_tx.status, TransactionStatus::Submitted);
2527 }
2528 }
2529
2530 mod handle_pending_state_tests {
2532 use super::*;
2533
2534 #[tokio::test]
2535 async fn test_pending_state_no_noop() {
2536 let mut mocks = default_test_mocks();
2538 let relayer = create_test_relayer();
2539 let mut tx = make_test_transaction(TransactionStatus::Pending);
2540 tx.created_at = Utc::now().to_rfc3339(); mocks
2544 .network_repo
2545 .expect_get_by_chain_id()
2546 .returning(|_, _| Ok(Some(create_test_network_model())));
2547
2548 mocks.provider.expect_get_block_by_number().returning(|| {
2550 Box::pin(async {
2551 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2552 let mut block: Block = Block::default();
2553 block.header.gas_limit = 30_000_000u64;
2554 Ok(AnyRpcBlock::from(block))
2555 })
2556 });
2557
2558 mocks
2560 .job_producer
2561 .expect_produce_check_transaction_status_job()
2562 .returning(|_, _| Box::pin(async { Ok(()) }));
2563
2564 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2565 let result = evm_transaction
2566 .handle_pending_state(tx.clone())
2567 .await
2568 .unwrap();
2569
2570 assert_eq!(result.id, tx.id);
2572 assert_eq!(result.status, tx.status);
2573 assert_eq!(result.noop_count, tx.noop_count);
2574 }
2575
2576 #[tokio::test]
2577 async fn test_pending_state_with_noop() {
2578 let mut mocks = default_test_mocks();
2580 let relayer = create_test_relayer();
2581 let mut tx = make_test_transaction(TransactionStatus::Pending);
2582 tx.created_at = (Utc::now() - Duration::minutes(2)).to_rfc3339();
2583
2584 mocks
2586 .network_repo
2587 .expect_get_by_chain_id()
2588 .returning(|_, _| Ok(Some(create_test_network_model())));
2589
2590 mocks.provider.expect_get_block_by_number().returning(|| {
2592 Box::pin(async {
2593 use alloy::{network::AnyRpcBlock, rpc::types::Block};
2594 let mut block: Block = Block::default();
2595 block.header.gas_limit = 30_000_000u64;
2596 Ok(AnyRpcBlock::from(block))
2597 })
2598 });
2599
2600 let tx_clone = tx.clone();
2603 mocks
2604 .tx_repo
2605 .expect_partial_update()
2606 .withf(move |id, update| {
2607 id == "test-tx-id"
2608 && update.status == Some(TransactionStatus::Failed)
2609 && update.status_reason.is_some()
2610 })
2611 .returning(move |_, update| {
2612 let mut updated_tx = tx_clone.clone();
2613 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2614 updated_tx.status_reason = update.status_reason.clone();
2615 Ok(updated_tx)
2616 });
2617 mocks
2619 .job_producer
2620 .expect_produce_send_notification_job()
2621 .returning(|_, _| Box::pin(async { Ok(()) }));
2622
2623 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2624 let result = evm_transaction
2625 .handle_pending_state(tx.clone())
2626 .await
2627 .unwrap();
2628
2629 assert_eq!(result.status, TransactionStatus::Failed);
2631 assert!(result.status_reason.is_some());
2632 assert!(result.status_reason.unwrap().contains("Pending state"));
2633 }
2634 }
2635
2636 mod handle_mined_state_tests {
2638 use super::*;
2639
2640 #[tokio::test]
2641 async fn test_updates_status_and_schedules_check() {
2642 let mut mocks = default_test_mocks();
2643 let relayer = create_test_relayer();
2644 let tx = make_test_transaction(TransactionStatus::Submitted);
2646
2647 mocks
2649 .job_producer
2650 .expect_produce_check_transaction_status_job()
2651 .returning(|_, _| Box::pin(async { Ok(()) }));
2652 mocks
2654 .tx_repo
2655 .expect_partial_update()
2656 .returning(|_, update| {
2657 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2658 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2659 Ok(updated_tx)
2660 });
2661
2662 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2663 let result = evm_transaction
2664 .handle_mined_state(tx.clone())
2665 .await
2666 .unwrap();
2667 assert_eq!(result.status, TransactionStatus::Mined);
2668 }
2669 }
2670
2671 mod handle_final_state_tests {
2673 use super::*;
2674
2675 #[tokio::test]
2676 async fn test_final_state_confirmed() {
2677 let mut mocks = default_test_mocks();
2678 let relayer = create_test_relayer();
2679 let tx = make_test_transaction(TransactionStatus::Submitted);
2680
2681 mocks
2683 .tx_repo
2684 .expect_partial_update()
2685 .returning(|_, update| {
2686 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2687 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2688 Ok(updated_tx)
2689 });
2690
2691 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2692 let result = evm_transaction
2693 .handle_final_state(tx.clone(), TransactionStatus::Confirmed, None)
2694 .await
2695 .unwrap();
2696 assert_eq!(result.status, TransactionStatus::Confirmed);
2697 }
2698
2699 #[tokio::test]
2700 async fn test_final_state_failed() {
2701 let mut mocks = default_test_mocks();
2702 let relayer = create_test_relayer();
2703 let tx = make_test_transaction(TransactionStatus::Submitted);
2704
2705 mocks
2707 .tx_repo
2708 .expect_partial_update()
2709 .returning(|_, update| {
2710 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2711 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2712 updated_tx.status_reason = update.status_reason.clone();
2713 Ok(updated_tx)
2714 });
2715
2716 let reason = "Transaction reverted on-chain (receipt status: failed)".to_string();
2717 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2718 let result = evm_transaction
2719 .handle_final_state(tx.clone(), TransactionStatus::Failed, Some(reason.clone()))
2720 .await
2721 .unwrap();
2722 assert_eq!(result.status, TransactionStatus::Failed);
2723 assert_eq!(result.status_reason.as_deref(), Some(reason.as_str()));
2724 }
2725
2726 #[tokio::test]
2727 async fn test_final_state_expired() {
2728 let mut mocks = default_test_mocks();
2729 let relayer = create_test_relayer();
2730 let tx = make_test_transaction(TransactionStatus::Submitted);
2731
2732 mocks
2734 .tx_repo
2735 .expect_partial_update()
2736 .returning(|_, update| {
2737 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2738 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2739 Ok(updated_tx)
2740 });
2741
2742 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2743 let result = evm_transaction
2744 .handle_final_state(tx.clone(), TransactionStatus::Expired, None)
2745 .await
2746 .unwrap();
2747 assert_eq!(result.status, TransactionStatus::Expired);
2748 }
2749 }
2750
2751 mod handle_status_impl_tests {
2753 use super::*;
2754
2755 #[tokio::test]
2756 async fn test_impl_submitted_branch() {
2757 let mut mocks = default_test_mocks();
2758 let relayer = create_test_relayer();
2759 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2760 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
2761 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2763 evm_data.hash = Some("0xFakeHash".to_string());
2764 }
2765 mocks
2767 .provider
2768 .expect_get_transaction_receipt()
2769 .returning(|_| Box::pin(async { Ok(None) }));
2770 mocks
2772 .network_repo
2773 .expect_get_by_chain_id()
2774 .returning(|_, _| Ok(Some(create_test_network_model())));
2775 mocks
2777 .job_producer
2778 .expect_produce_check_transaction_status_job()
2779 .returning(|_, _| Box::pin(async { Ok(()) }));
2780 mocks
2782 .tx_repo
2783 .expect_partial_update()
2784 .returning(|_, update| {
2785 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2786 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2787 Ok(updated_tx)
2788 });
2789
2790 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2791 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2792 assert_eq!(result.status, TransactionStatus::Submitted);
2793 }
2794
2795 #[tokio::test]
2796 async fn test_impl_mined_branch() {
2797 let mut mocks = default_test_mocks();
2798 let relayer = create_test_relayer();
2799 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2800 tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
2802 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2804 evm_data.hash = Some("0xFakeHash".to_string());
2805 }
2806 mocks
2808 .provider
2809 .expect_get_transaction_receipt()
2810 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) }));
2811 mocks
2813 .provider
2814 .expect_get_block_number()
2815 .return_once(|| Box::pin(async { Ok(100) }));
2816 mocks
2818 .network_repo
2819 .expect_get_by_chain_id()
2820 .returning(|_, _| Ok(Some(create_test_network_model())));
2821 mocks
2823 .job_producer
2824 .expect_produce_send_notification_job()
2825 .returning(|_, _| Box::pin(async { Ok(()) }));
2826 mocks.tx_repo.expect_get_by_id().returning(|_| {
2828 let updated_tx = make_test_transaction(TransactionStatus::Mined);
2829 Ok(updated_tx)
2830 });
2831 mocks
2833 .tx_repo
2834 .expect_partial_update()
2835 .returning(|_, update| {
2836 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2837 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2838 Ok(updated_tx)
2839 });
2840
2841 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2842 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2843 assert_eq!(result.status, TransactionStatus::Mined);
2844 }
2845
2846 #[tokio::test]
2847 async fn test_impl_final_confirmed_branch() {
2848 let mut mocks = default_test_mocks();
2849 let relayer = create_test_relayer();
2850 let tx = make_test_transaction(TransactionStatus::Confirmed);
2852
2853 mocks
2856 .tx_repo
2857 .expect_partial_update()
2858 .returning(|_, update| {
2859 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2860 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2861 Ok(updated_tx)
2862 });
2863
2864 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2865 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2866 assert_eq!(result.status, TransactionStatus::Confirmed);
2867 }
2868
2869 #[tokio::test]
2870 async fn test_impl_final_failed_branch() {
2871 let mut mocks = default_test_mocks();
2872 let relayer = create_test_relayer();
2873 let tx = make_test_transaction(TransactionStatus::Failed);
2875
2876 mocks
2877 .tx_repo
2878 .expect_partial_update()
2879 .returning(|_, update| {
2880 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2881 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2882 Ok(updated_tx)
2883 });
2884
2885 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2886 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2887 assert_eq!(result.status, TransactionStatus::Failed);
2888 }
2889
2890 #[tokio::test]
2893 async fn test_impl_submitted_to_failed_sets_status_reason() {
2894 let mut mocks = default_test_mocks();
2895 let relayer = create_test_relayer();
2896 let mut tx = make_test_transaction(TransactionStatus::Submitted);
2897 tx.created_at = (Utc::now() - Duration::minutes(1)).to_rfc3339();
2898 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
2899 evm_data.hash = Some("0xFakeHash".to_string());
2900 }
2901
2902 mocks
2904 .provider
2905 .expect_get_transaction_receipt()
2906 .returning(|_| Box::pin(async { Ok(Some(make_mock_receipt(false, Some(100)))) }));
2907
2908 let tx_clone = tx.clone();
2910 mocks.tx_repo.expect_get_by_id().returning(move |_| {
2911 let mut reloaded = tx_clone.clone();
2912 reloaded.status = TransactionStatus::Submitted;
2913 Ok(reloaded)
2914 });
2915
2916 mocks
2918 .tx_repo
2919 .expect_partial_update()
2920 .withf(|_, update| {
2921 update.status == Some(TransactionStatus::Failed)
2922 && update.status_reason.is_some()
2923 })
2924 .returning(|_, update| {
2925 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2926 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2927 updated_tx.status_reason = update.status_reason.clone();
2928 Ok(updated_tx)
2929 });
2930
2931 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2932 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2933 assert_eq!(result.status, TransactionStatus::Failed);
2934 assert!(result.status_reason.is_some());
2935 assert!(
2936 result
2937 .status_reason
2938 .as_ref()
2939 .unwrap()
2940 .contains("reverted on-chain"),
2941 "Expected on-chain revert reason, got: {:?}",
2942 result.status_reason
2943 );
2944 }
2945
2946 #[tokio::test]
2947 async fn test_impl_final_expired_branch() {
2948 let mut mocks = default_test_mocks();
2949 let relayer = create_test_relayer();
2950 let tx = make_test_transaction(TransactionStatus::Expired);
2952
2953 mocks
2954 .tx_repo
2955 .expect_partial_update()
2956 .returning(|_, update| {
2957 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
2958 updated_tx.status = update.status.unwrap_or(updated_tx.status);
2959 Ok(updated_tx)
2960 });
2961
2962 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
2963 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
2964 assert_eq!(result.status, TransactionStatus::Expired);
2965 }
2966 }
2967
2968 mod circuit_breaker_tests {
2970 use super::*;
2971 use crate::jobs::StatusCheckContext;
2972
2973 fn create_triggered_context() -> StatusCheckContext {
2975 StatusCheckContext::new(
2976 30, 50, 60, 25, 75, NetworkType::Evm,
2982 )
2983 }
2984
2985 fn create_safe_context() -> StatusCheckContext {
2987 StatusCheckContext::new(
2988 5, 10, 15, 25, 75, NetworkType::Evm,
2994 )
2995 }
2996
2997 fn create_total_triggered_context() -> StatusCheckContext {
2999 StatusCheckContext::new(
3000 5, 80, 100, 25, 75, NetworkType::Evm,
3006 )
3007 }
3008
3009 #[tokio::test]
3010 async fn test_circuit_breaker_pending_marks_as_failed() {
3011 let mut mocks = default_test_mocks();
3012 let relayer = create_test_relayer();
3013 let tx = make_test_transaction(TransactionStatus::Pending);
3014
3015 mocks
3017 .tx_repo
3018 .expect_partial_update()
3019 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3020 .returning(|_, update| {
3021 let mut updated_tx = make_test_transaction(TransactionStatus::Pending);
3022 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3023 updated_tx.status_reason = update.status_reason.clone();
3024 Ok(updated_tx)
3025 });
3026
3027 mocks
3029 .job_producer
3030 .expect_produce_send_notification_job()
3031 .returning(|_, _| Box::pin(async { Ok(()) }));
3032
3033 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3034 let ctx = create_triggered_context();
3035
3036 let result = evm_transaction
3037 .handle_status_impl(tx, Some(ctx))
3038 .await
3039 .unwrap();
3040
3041 assert_eq!(result.status, TransactionStatus::Failed);
3042 assert!(result.status_reason.is_some());
3043 assert!(result.status_reason.unwrap().contains("consecutive errors"));
3044 }
3045
3046 #[tokio::test]
3047 async fn test_circuit_breaker_sent_marks_as_failed() {
3048 let mut mocks = default_test_mocks();
3049 let relayer = create_test_relayer();
3050 let tx = make_test_transaction(TransactionStatus::Sent);
3051
3052 mocks
3054 .tx_repo
3055 .expect_partial_update()
3056 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3057 .returning(|_, update| {
3058 let mut updated_tx = make_test_transaction(TransactionStatus::Sent);
3059 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3060 updated_tx.status_reason = update.status_reason.clone();
3061 Ok(updated_tx)
3062 });
3063
3064 mocks
3066 .job_producer
3067 .expect_produce_send_notification_job()
3068 .returning(|_, _| Box::pin(async { Ok(()) }));
3069
3070 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3071 let ctx = create_triggered_context();
3072
3073 let result = evm_transaction
3074 .handle_status_impl(tx, Some(ctx))
3075 .await
3076 .unwrap();
3077
3078 assert_eq!(result.status, TransactionStatus::Failed);
3079 }
3080
3081 #[tokio::test]
3082 async fn test_circuit_breaker_submitted_triggers_noop() {
3083 let mut mocks = default_test_mocks();
3084 let relayer = create_test_relayer();
3085 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3086 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3087
3088 mocks
3090 .network_repo
3091 .expect_get_by_chain_id()
3092 .returning(|_, _| Ok(Some(create_test_network_model())));
3093
3094 mocks
3096 .tx_repo
3097 .expect_partial_update()
3098 .returning(|_, update| {
3099 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3100 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3101 updated_tx.status_reason = update.status_reason.clone();
3102 updated_tx.noop_count = update.noop_count;
3103 Ok(updated_tx)
3104 });
3105
3106 mocks
3108 .job_producer
3109 .expect_produce_submit_transaction_job()
3110 .returning(|_, _| Box::pin(async { Ok(()) }));
3111
3112 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3113 let ctx = create_triggered_context();
3114
3115 let result = evm_transaction
3116 .handle_status_impl(tx, Some(ctx))
3117 .await
3118 .unwrap();
3119
3120 assert!(result.noop_count.is_some());
3122 }
3123
3124 #[tokio::test]
3125 async fn test_circuit_breaker_noop_tx_excluded() {
3126 let mut mocks = default_test_mocks();
3127 let relayer = create_test_relayer();
3128
3129 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3131 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3132 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3133 evm_data.to = Some(evm_data.from.clone()); evm_data.value = U256::from(0);
3135 evm_data.data = Some("0x".to_string());
3136 evm_data.hash = Some("0xFakeHash".to_string());
3137 }
3138
3139 mocks
3142 .provider
3143 .expect_get_transaction_receipt()
3144 .returning(|_| Box::pin(async { Ok(None) }));
3145
3146 mocks
3147 .network_repo
3148 .expect_get_by_chain_id()
3149 .returning(|_, _| Ok(Some(create_test_network_model())));
3150
3151 mocks
3152 .job_producer
3153 .expect_produce_check_transaction_status_job()
3154 .returning(|_, _| Box::pin(async { Ok(()) }));
3155
3156 mocks
3158 .job_producer
3159 .expect_produce_submit_transaction_job()
3160 .returning(|_, _| Box::pin(async { Ok(()) }));
3161
3162 mocks
3163 .tx_repo
3164 .expect_partial_update()
3165 .returning(|_, update| {
3166 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3167 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3168 Ok(updated_tx)
3169 });
3170
3171 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3172 let ctx = create_triggered_context();
3173
3174 let result = evm_transaction
3175 .handle_status_impl(tx, Some(ctx))
3176 .await
3177 .unwrap();
3178
3179 assert_eq!(result.status, TransactionStatus::Submitted);
3181 }
3182
3183 #[tokio::test]
3184 async fn test_circuit_breaker_total_failures_triggers() {
3185 let mut mocks = default_test_mocks();
3186 let relayer = create_test_relayer();
3187 let tx = make_test_transaction(TransactionStatus::Pending);
3188
3189 mocks
3191 .tx_repo
3192 .expect_partial_update()
3193 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
3194 .returning(|_, update| {
3195 let mut updated_tx = make_test_transaction(TransactionStatus::Pending);
3196 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3197 updated_tx.status_reason = update.status_reason.clone();
3198 Ok(updated_tx)
3199 });
3200
3201 mocks
3202 .job_producer
3203 .expect_produce_send_notification_job()
3204 .returning(|_, _| Box::pin(async { Ok(()) }));
3205
3206 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3207 let ctx = create_total_triggered_context();
3209
3210 let result = evm_transaction
3211 .handle_status_impl(tx, Some(ctx))
3212 .await
3213 .unwrap();
3214
3215 assert_eq!(result.status, TransactionStatus::Failed);
3216 assert!(result.status_reason.is_some());
3217 }
3218
3219 #[tokio::test]
3220 async fn test_circuit_breaker_below_threshold_continues_normally() {
3221 let mut mocks = default_test_mocks();
3222 let relayer = create_test_relayer();
3223 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3224 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3225
3226 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3227 evm_data.hash = Some("0xFakeHash".to_string());
3228 }
3229
3230 mocks
3232 .provider
3233 .expect_get_transaction_receipt()
3234 .returning(|_| Box::pin(async { Ok(None) }));
3235
3236 mocks
3237 .network_repo
3238 .expect_get_by_chain_id()
3239 .returning(|_, _| Ok(Some(create_test_network_model())));
3240
3241 mocks
3242 .job_producer
3243 .expect_produce_check_transaction_status_job()
3244 .returning(|_, _| Box::pin(async { Ok(()) }));
3245
3246 mocks
3247 .tx_repo
3248 .expect_partial_update()
3249 .returning(|_, update| {
3250 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3251 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3252 Ok(updated_tx)
3253 });
3254
3255 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3256 let ctx = create_safe_context();
3257
3258 let result = evm_transaction
3259 .handle_status_impl(tx, Some(ctx))
3260 .await
3261 .unwrap();
3262
3263 assert_eq!(result.status, TransactionStatus::Submitted);
3265 }
3266
3267 #[tokio::test]
3268 async fn test_circuit_breaker_no_context_continues_normally() {
3269 let mut mocks = default_test_mocks();
3270 let relayer = create_test_relayer();
3271 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3272 tx.sent_at = Some((Utc::now() - Duration::seconds(120)).to_rfc3339());
3273
3274 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3275 evm_data.hash = Some("0xFakeHash".to_string());
3276 }
3277
3278 mocks
3280 .provider
3281 .expect_get_transaction_receipt()
3282 .returning(|_| Box::pin(async { Ok(None) }));
3283
3284 mocks
3285 .network_repo
3286 .expect_get_by_chain_id()
3287 .returning(|_, _| Ok(Some(create_test_network_model())));
3288
3289 mocks
3290 .job_producer
3291 .expect_produce_check_transaction_status_job()
3292 .returning(|_, _| Box::pin(async { Ok(()) }));
3293
3294 mocks
3295 .tx_repo
3296 .expect_partial_update()
3297 .returning(|_, update| {
3298 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3299 updated_tx.status = update.status.unwrap_or(updated_tx.status);
3300 Ok(updated_tx)
3301 });
3302
3303 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3304
3305 let result = evm_transaction.handle_status_impl(tx, None).await.unwrap();
3307
3308 assert_eq!(result.status, TransactionStatus::Submitted);
3310 }
3311
3312 #[tokio::test]
3313 async fn test_circuit_breaker_final_state_early_return() {
3314 let mocks = default_test_mocks();
3315 let relayer = create_test_relayer();
3316 let tx = make_test_transaction(TransactionStatus::Confirmed);
3318
3319 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3320 let ctx = create_triggered_context();
3321
3322 let result = evm_transaction
3324 .handle_status_impl(tx, Some(ctx))
3325 .await
3326 .unwrap();
3327
3328 assert_eq!(result.status, TransactionStatus::Confirmed);
3329 }
3330 }
3331
3332 mod hash_recovery_tests {
3334 use super::*;
3335
3336 #[tokio::test]
3337 async fn test_should_try_hash_recovery_not_submitted() {
3338 let mocks = default_test_mocks();
3339 let relayer = create_test_relayer();
3340
3341 let mut tx = make_test_transaction(TransactionStatus::Sent);
3342 tx.hashes = vec![
3343 "0xHash1".to_string(),
3344 "0xHash2".to_string(),
3345 "0xHash3".to_string(),
3346 ];
3347
3348 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3349 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3350
3351 assert!(
3352 !result,
3353 "Should not attempt recovery for non-Submitted transactions"
3354 );
3355 }
3356
3357 #[tokio::test]
3358 async fn test_should_try_hash_recovery_not_enough_hashes() {
3359 let mocks = default_test_mocks();
3360 let relayer = create_test_relayer();
3361
3362 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3363 tx.hashes = vec!["0xHash1".to_string()]; tx.sent_at = Some((Utc::now() - Duration::minutes(3)).to_rfc3339());
3365
3366 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3367 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3368
3369 assert!(
3370 !result,
3371 "Should not attempt recovery with insufficient hashes"
3372 );
3373 }
3374
3375 #[tokio::test]
3376 async fn test_should_try_hash_recovery_too_recent() {
3377 let mocks = default_test_mocks();
3378 let relayer = create_test_relayer();
3379
3380 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3381 tx.hashes = vec![
3382 "0xHash1".to_string(),
3383 "0xHash2".to_string(),
3384 "0xHash3".to_string(),
3385 ];
3386 tx.sent_at = Some(Utc::now().to_rfc3339()); let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3389 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3390
3391 assert!(
3392 !result,
3393 "Should not attempt recovery for recently sent transactions"
3394 );
3395 }
3396
3397 #[tokio::test]
3398 async fn test_should_try_hash_recovery_success() {
3399 let mocks = default_test_mocks();
3400 let relayer = create_test_relayer();
3401
3402 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3403 tx.hashes = vec![
3404 "0xHash1".to_string(),
3405 "0xHash2".to_string(),
3406 "0xHash3".to_string(),
3407 ];
3408 tx.sent_at = Some((Utc::now() - Duration::minutes(3)).to_rfc3339());
3409
3410 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3411 let result = evm_transaction.should_try_hash_recovery(&tx).unwrap();
3412
3413 assert!(
3414 result,
3415 "Should attempt recovery for stuck transactions with multiple hashes"
3416 );
3417 }
3418
3419 #[tokio::test]
3420 async fn test_try_recover_no_historical_hash_found() {
3421 let mut mocks = default_test_mocks();
3422 let relayer = create_test_relayer();
3423
3424 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3425 tx.hashes = vec![
3426 "0xHash1".to_string(),
3427 "0xHash2".to_string(),
3428 "0xHash3".to_string(),
3429 ];
3430
3431 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3432 evm_data.hash = Some("0xHash3".to_string());
3433 }
3434
3435 mocks
3437 .provider
3438 .expect_get_transaction_receipt()
3439 .returning(|_| Box::pin(async { Ok(None) }));
3440
3441 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3442 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3443 let result = evm_transaction
3444 .try_recover_with_historical_hashes(&tx, &evm_data)
3445 .await
3446 .unwrap();
3447
3448 assert!(
3449 result.is_none(),
3450 "Should return None when no historical hash is found"
3451 );
3452 }
3453
3454 #[tokio::test]
3455 async fn test_try_recover_finds_mined_historical_hash() {
3456 let mut mocks = default_test_mocks();
3457 let relayer = create_test_relayer();
3458
3459 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3460 tx.hashes = vec![
3461 "0xHash1".to_string(),
3462 "0xHash2".to_string(), "0xHash3".to_string(),
3464 ];
3465
3466 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3467 evm_data.hash = Some("0xHash3".to_string()); }
3469
3470 mocks
3472 .provider
3473 .expect_get_transaction_receipt()
3474 .returning(|hash| {
3475 if hash == "0xHash2" {
3476 Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) })
3477 } else {
3478 Box::pin(async { Ok(None) })
3479 }
3480 });
3481
3482 let tx_clone = tx.clone();
3484 mocks
3485 .tx_repo
3486 .expect_partial_update()
3487 .returning(move |_, update| {
3488 let mut updated_tx = tx_clone.clone();
3489 if let Some(status) = update.status {
3490 updated_tx.status = status;
3491 }
3492 if let Some(NetworkTransactionData::Evm(ref evm_data)) = update.network_data {
3493 if let NetworkTransactionData::Evm(ref mut updated_evm) =
3494 updated_tx.network_data
3495 {
3496 updated_evm.hash = evm_data.hash.clone();
3497 }
3498 }
3499 Ok(updated_tx)
3500 });
3501
3502 mocks
3504 .job_producer
3505 .expect_produce_send_notification_job()
3506 .returning(|_, _| Box::pin(async { Ok(()) }));
3507
3508 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3509 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3510 let result = evm_transaction
3511 .try_recover_with_historical_hashes(&tx, &evm_data)
3512 .await
3513 .unwrap();
3514
3515 assert!(result.is_some(), "Should recover the transaction");
3516 let recovered_tx = result.unwrap();
3517 assert_eq!(recovered_tx.status, TransactionStatus::Mined);
3518 }
3519
3520 #[tokio::test]
3521 async fn test_try_recover_network_error_continues() {
3522 let mut mocks = default_test_mocks();
3523 let relayer = create_test_relayer();
3524
3525 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3526 tx.hashes = vec![
3527 "0xHash1".to_string(),
3528 "0xHash2".to_string(), "0xHash3".to_string(), ];
3531
3532 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3533 evm_data.hash = Some("0xHash1".to_string());
3534 }
3535
3536 mocks
3538 .provider
3539 .expect_get_transaction_receipt()
3540 .returning(|hash| {
3541 if hash == "0xHash2" {
3542 Box::pin(async { Err(crate::services::provider::ProviderError::Timeout) })
3543 } else if hash == "0xHash3" {
3544 Box::pin(async { Ok(Some(make_mock_receipt(true, Some(100)))) })
3545 } else {
3546 Box::pin(async { Ok(None) })
3547 }
3548 });
3549
3550 let tx_clone = tx.clone();
3552 mocks
3553 .tx_repo
3554 .expect_partial_update()
3555 .returning(move |_, update| {
3556 let mut updated_tx = tx_clone.clone();
3557 if let Some(status) = update.status {
3558 updated_tx.status = status;
3559 }
3560 Ok(updated_tx)
3561 });
3562
3563 mocks
3565 .job_producer
3566 .expect_produce_send_notification_job()
3567 .returning(|_, _| Box::pin(async { Ok(()) }));
3568
3569 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3570 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3571 let result = evm_transaction
3572 .try_recover_with_historical_hashes(&tx, &evm_data)
3573 .await
3574 .unwrap();
3575
3576 assert!(
3577 result.is_some(),
3578 "Should continue checking after network error and find mined hash"
3579 );
3580 }
3581
3582 #[tokio::test]
3583 async fn test_update_transaction_with_corrected_hash() {
3584 let mut mocks = default_test_mocks();
3585 let relayer = create_test_relayer();
3586
3587 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3588 if let NetworkTransactionData::Evm(ref mut evm_data) = tx.network_data {
3589 evm_data.hash = Some("0xWrongHash".to_string());
3590 }
3591
3592 mocks
3594 .tx_repo
3595 .expect_partial_update()
3596 .returning(move |_, update| {
3597 let mut updated_tx = make_test_transaction(TransactionStatus::Submitted);
3598 if let Some(status) = update.status {
3599 updated_tx.status = status;
3600 }
3601 if let Some(NetworkTransactionData::Evm(ref evm_data)) = update.network_data {
3602 if let NetworkTransactionData::Evm(ref mut updated_evm) =
3603 updated_tx.network_data
3604 {
3605 updated_evm.hash = evm_data.hash.clone();
3606 }
3607 }
3608 Ok(updated_tx)
3609 });
3610
3611 mocks
3613 .job_producer
3614 .expect_produce_send_notification_job()
3615 .returning(|_, _| Box::pin(async { Ok(()) }));
3616
3617 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3618 let evm_data = tx.network_data.get_evm_transaction_data().unwrap();
3619 let result = evm_transaction
3620 .update_transaction_with_corrected_hash(
3621 &tx,
3622 &evm_data,
3623 "0xCorrectHash",
3624 TransactionStatus::Mined,
3625 )
3626 .await
3627 .unwrap();
3628
3629 assert_eq!(result.status, TransactionStatus::Mined);
3630 if let NetworkTransactionData::Evm(ref updated_evm) = result.network_data {
3631 assert_eq!(updated_evm.hash.as_ref().unwrap(), "0xCorrectHash");
3632 }
3633 }
3634 }
3635
3636 mod check_transaction_status_edge_cases {
3638 use super::*;
3639
3640 #[tokio::test]
3641 async fn test_missing_hash_returns_error() {
3642 let mocks = default_test_mocks();
3643 let relayer = create_test_relayer();
3644
3645 let tx = make_test_transaction(TransactionStatus::Submitted);
3646 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3649 let result = evm_transaction.check_transaction_status(&tx).await;
3650
3651 assert!(result.is_err(), "Should return error when hash is missing");
3652 }
3653
3654 #[tokio::test]
3655 async fn test_pending_status_early_return() {
3656 let mocks = default_test_mocks();
3657 let relayer = create_test_relayer();
3658
3659 let tx = make_test_transaction(TransactionStatus::Pending);
3660
3661 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3662 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
3663
3664 assert_eq!(
3665 status,
3666 TransactionStatus::Pending,
3667 "Should return Pending without querying blockchain"
3668 );
3669 }
3670
3671 #[tokio::test]
3672 async fn test_sent_status_early_return() {
3673 let mocks = default_test_mocks();
3674 let relayer = create_test_relayer();
3675
3676 let tx = make_test_transaction(TransactionStatus::Sent);
3677
3678 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3679 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
3680
3681 assert_eq!(
3682 status,
3683 TransactionStatus::Sent,
3684 "Should return Sent without querying blockchain"
3685 );
3686 }
3687
3688 #[tokio::test]
3689 async fn test_final_state_early_return() {
3690 let mocks = default_test_mocks();
3691 let relayer = create_test_relayer();
3692
3693 let tx = make_test_transaction(TransactionStatus::Confirmed);
3694
3695 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3696 let status = evm_transaction.check_transaction_status(&tx).await.unwrap();
3697
3698 assert_eq!(
3699 status,
3700 TransactionStatus::Confirmed,
3701 "Should return final state without querying blockchain"
3702 );
3703 }
3704 }
3705
3706 mod nonce_recovery_tests {
3707 use super::*;
3708 use crate::domain::transaction::evm::evm_transaction::TX_NONCE_RECONCILE_TRIGGER;
3709 use crate::jobs::StatusCheckContext;
3710
3711 #[tokio::test]
3713 async fn test_nonce_recovery_nonce_consumed_externally() {
3714 let mut mocks = default_test_mocks();
3715 let relayer = create_test_relayer();
3716
3717 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3718 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3719 nonce: Some(5),
3720 hash: Some("0xhash".to_string()),
3721 raw: Some(vec![1, 2, 3]),
3722 ..tx.network_data.get_evm_transaction_data().unwrap()
3723 });
3724 tx.sent_at = Some(Utc::now().to_rfc3339());
3725
3726 mocks
3728 .provider
3729 .expect_get_transaction_receipt()
3730 .returning(|_| Box::pin(async { Ok(None) }));
3731
3732 mocks
3734 .provider
3735 .expect_get_transaction_count()
3736 .returning(|_| Box::pin(async { Ok(10) }));
3737
3738 let tx_clone = tx.clone();
3740 mocks
3741 .tx_repo
3742 .expect_partial_update()
3743 .withf(|_, update| {
3744 update.status == Some(TransactionStatus::Failed)
3745 && update
3746 .status_reason
3747 .as_ref()
3748 .map(|r| r.contains("consumed externally"))
3749 .unwrap_or(false)
3750 })
3751 .returning(move |_, update| {
3752 let mut updated_tx = tx_clone.clone();
3753 updated_tx.status = update.status.unwrap();
3754 updated_tx.status_reason = update.status_reason.clone();
3755 Ok(updated_tx)
3756 });
3757
3758 mocks
3760 .job_producer
3761 .expect_produce_relayer_health_check_job()
3762 .withf(|job, scheduled_on| {
3763 job.relayer_id == "test-relayer-id"
3764 && job.metadata.as_ref().map_or(false, |m| {
3765 m.get("health_check_action") == Some(&"nonce_health".to_string())
3766 })
3767 && scheduled_on.is_none()
3768 })
3769 .returning(|_, _| Box::pin(async { Ok(()) }));
3770
3771 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3772 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3773
3774 assert!(result.is_ok());
3775 let recovered = result.unwrap();
3776 assert!(recovered.is_some(), "Expected Some(tx) for consumed nonce");
3777 assert_eq!(recovered.unwrap().status, TransactionStatus::Failed);
3778 }
3779
3780 #[tokio::test]
3782 async fn test_nonce_recovery_nonce_not_consumed() {
3783 let mut mocks = default_test_mocks();
3784 let relayer = create_test_relayer();
3785
3786 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3787 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3788 nonce: Some(5),
3789 hash: Some("0xhash".to_string()),
3790 raw: Some(vec![1, 2, 3]),
3791 ..tx.network_data.get_evm_transaction_data().unwrap()
3792 });
3793
3794 mocks
3796 .provider
3797 .expect_get_transaction_receipt()
3798 .returning(|_| Box::pin(async { Ok(None) }));
3799
3800 mocks
3802 .provider
3803 .expect_get_transaction_count()
3804 .returning(|_| Box::pin(async { Ok(5) }));
3805
3806 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3807 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3808
3809 assert!(result.is_ok());
3810 assert!(
3811 result.unwrap().is_none(),
3812 "Expected None when nonce not consumed"
3813 );
3814 }
3815
3816 #[tokio::test]
3818 async fn test_nonce_recovery_receipt_found() {
3819 let mut mocks = default_test_mocks();
3820 let relayer = create_test_relayer();
3821
3822 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3823 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3824 nonce: Some(5),
3825 hash: Some("0xhash".to_string()),
3826 raw: Some(vec![1, 2, 3]),
3827 ..tx.network_data.get_evm_transaction_data().unwrap()
3828 });
3829
3830 mocks
3832 .provider
3833 .expect_get_transaction_receipt()
3834 .returning(|_| {
3835 let receipt = make_mock_receipt(true, Some(100));
3836 Box::pin(async move { Ok(Some(receipt)) })
3837 });
3838
3839 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3840 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3841
3842 assert!(result.is_ok());
3843 assert!(
3844 result.unwrap().is_none(),
3845 "Expected None when receipt found — defer to normal flow"
3846 );
3847 }
3848
3849 #[tokio::test]
3852 async fn test_nonce_recovery_rpc_error_prevents_force_fail() {
3853 let mut mocks = default_test_mocks();
3854 let relayer = create_test_relayer();
3855
3856 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3857 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3858 nonce: Some(5),
3859 hash: Some("0xhash".to_string()),
3860 raw: Some(vec![1, 2, 3]),
3861 ..tx.network_data.get_evm_transaction_data().unwrap()
3862 });
3863
3864 mocks
3866 .provider
3867 .expect_get_transaction_receipt()
3868 .returning(|_| {
3869 Box::pin(async {
3870 Err(crate::services::provider::ProviderError::Other(
3871 "RPC timeout".to_string(),
3872 ))
3873 })
3874 });
3875
3876 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3880 let result = evm_transaction.reconcile_tx_nonce_state(&tx).await;
3881
3882 assert!(result.is_ok());
3883 assert!(
3884 result.unwrap().is_none(),
3885 "Expected None when RPC errors occurred — must not force-fail on incomplete data"
3886 );
3887 }
3888
3889 #[tokio::test]
3891 async fn test_handle_status_impl_nonce_recovery_hint() {
3892 let mut mocks = default_test_mocks();
3893 let relayer = create_test_relayer();
3894
3895 let mut tx = make_test_transaction(TransactionStatus::Submitted);
3896 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3897 nonce: Some(5),
3898 hash: Some("0xhash".to_string()),
3899 raw: Some(vec![1, 2, 3]),
3900 ..tx.network_data.get_evm_transaction_data().unwrap()
3901 });
3902 tx.sent_at = Some(Utc::now().to_rfc3339());
3903
3904 mocks
3906 .provider
3907 .expect_get_transaction_receipt()
3908 .returning(|_| Box::pin(async { Ok(None) }));
3909
3910 mocks
3912 .provider
3913 .expect_get_transaction_count()
3914 .returning(|_| Box::pin(async { Ok(10) }));
3915
3916 let tx_clone = tx.clone();
3918 mocks
3919 .tx_repo
3920 .expect_partial_update()
3921 .returning(move |_, update| {
3922 let mut updated_tx = tx_clone.clone();
3923 if let Some(status) = update.status {
3924 updated_tx.status = status;
3925 }
3926 updated_tx.status_reason = update.status_reason.clone();
3927 Ok(updated_tx)
3928 });
3929
3930 mocks
3932 .job_producer
3933 .expect_produce_relayer_health_check_job()
3934 .returning(|_, _| Box::pin(async { Ok(()) }));
3935
3936 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3937
3938 let mut metadata = std::collections::HashMap::new();
3940 metadata.insert(
3941 TX_NONCE_RECONCILE_TRIGGER.to_string(),
3942 "NonceTooLow".to_string(),
3943 );
3944 let context = StatusCheckContext::default().with_job_metadata(Some(metadata));
3945
3946 let result = evm_transaction.handle_status_impl(tx, Some(context)).await;
3947 assert!(result.is_ok());
3948 assert_eq!(result.unwrap().status, TransactionStatus::Failed);
3949 }
3950 }
3951
3952 mod circuit_breaker_sent_state_tests {
3953 use super::*;
3954 use crate::jobs::StatusCheckContext;
3955
3956 #[tokio::test]
3958 async fn test_circuit_breaker_sent_with_nonce_issues_noop() {
3959 let mut mocks = default_test_mocks_with_network();
3960 let relayer = create_test_relayer();
3961
3962 let mut tx = make_test_transaction(TransactionStatus::Sent);
3963 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
3964 nonce: Some(5),
3965 hash: Some("0xhash".to_string()),
3966 raw: Some(vec![1, 2, 3]),
3967 ..tx.network_data.get_evm_transaction_data().unwrap()
3968 });
3969 tx.sent_at = Some(Utc::now().to_rfc3339());
3970
3971 let tx_clone = tx.clone();
3973 mocks
3974 .tx_repo
3975 .expect_partial_update()
3976 .returning(move |_, update| {
3977 let mut updated_tx = tx_clone.clone();
3978 if let Some(status) = update.status {
3979 updated_tx.status = status;
3980 }
3981 updated_tx.status_reason = update.status_reason.clone();
3982 Ok(updated_tx)
3983 });
3984
3985 mocks
3987 .job_producer
3988 .expect_produce_submit_transaction_job()
3989 .times(1)
3990 .returning(|_, _| Box::pin(async { Ok(()) }));
3991
3992 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
3993
3994 let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
3996
3997 let result = evm_transaction
3998 .handle_circuit_breaker_safely(tx, &ctx)
3999 .await;
4000 assert!(result.is_ok());
4001 }
4002
4003 #[tokio::test]
4005 async fn test_circuit_breaker_sent_without_nonce_marks_failed() {
4006 let mut mocks = default_test_mocks();
4007 let relayer = create_test_relayer();
4008
4009 let mut tx = make_test_transaction(TransactionStatus::Sent);
4010 tx.network_data = NetworkTransactionData::Evm(EvmTransactionData {
4012 nonce: None,
4013 hash: None,
4014 raw: None,
4015 ..tx.network_data.get_evm_transaction_data().unwrap()
4016 });
4017 tx.sent_at = Some(Utc::now().to_rfc3339());
4018
4019 let tx_clone = tx.clone();
4021 mocks
4022 .tx_repo
4023 .expect_partial_update()
4024 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
4025 .returning(move |_, update| {
4026 let mut updated_tx = tx_clone.clone();
4027 updated_tx.status = update.status.unwrap();
4028 Ok(updated_tx)
4029 });
4030
4031 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4032
4033 let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4034
4035 let result = evm_transaction
4036 .handle_circuit_breaker_safely(tx, &ctx)
4037 .await;
4038 assert!(result.is_ok());
4039 assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4040 }
4041
4042 #[tokio::test]
4044 async fn test_circuit_breaker_pending_marks_failed() {
4045 let mut mocks = default_test_mocks();
4046 let relayer = create_test_relayer();
4047
4048 let tx = make_test_transaction(TransactionStatus::Pending);
4049
4050 let tx_clone = tx.clone();
4052 mocks
4053 .tx_repo
4054 .expect_partial_update()
4055 .withf(|_, update| update.status == Some(TransactionStatus::Failed))
4056 .returning(move |_, update| {
4057 let mut updated_tx = tx_clone.clone();
4058 updated_tx.status = update.status.unwrap();
4059 Ok(updated_tx)
4060 });
4061
4062 let evm_transaction = make_test_evm_relayer_transaction(relayer, mocks);
4063
4064 let ctx = StatusCheckContext::new(100, 200, 250, 15, 45, NetworkType::Evm);
4065
4066 let result = evm_transaction
4067 .handle_circuit_breaker_safely(tx, &ctx)
4068 .await;
4069 assert!(result.is_ok());
4070 assert_eq!(result.unwrap().status, TransactionStatus::Failed);
4071 }
4072 }
4073}