ef_tests/
result.rs

1//! Test results and errors
2
3use crate::Case;
4use reth_db::DatabaseError;
5use reth_provider::ProviderError;
6use std::path::{Path, PathBuf};
7use thiserror::Error;
8
9/// Test errors
10///
11/// # Note
12///
13/// `Error::Skipped` should not be treated as a test failure.
14#[derive(Debug, Error)]
15#[non_exhaustive]
16pub enum Error {
17    /// The test was skipped
18    #[error("test was skipped")]
19    Skipped,
20    /// No post state found in test
21    #[error("no post state found for validation")]
22    MissingPostState,
23    /// Block processing failed
24    /// Note: This includes but is not limited to execution.
25    /// For example, the header number could be incorrect.
26    #[error("block {block_number} failed to process")]
27    BlockProcessingFailed {
28        /// The block number for the block that failed
29        block_number: u64,
30    },
31    /// An IO error occurred
32    #[error("an error occurred interacting with the file system at {path}: {error}")]
33    Io {
34        /// The path to the file or directory
35        path: PathBuf,
36        /// The specific error
37        #[source]
38        error: std::io::Error,
39    },
40    /// A deserialization error occurred
41    #[error("an error occurred deserializing the test at {path}: {error}")]
42    CouldNotDeserialize {
43        /// The path to the file we wanted to deserialize
44        path: PathBuf,
45        /// The specific error
46        #[source]
47        error: serde_json::Error,
48    },
49    /// A database error occurred.
50    #[error(transparent)]
51    Database(#[from] DatabaseError),
52    /// A test assertion failed.
53    #[error("test failed: {0}")]
54    Assertion(String),
55    /// An error internally in reth occurred.
56    #[error("test failed: {0}")]
57    Provider(#[from] ProviderError),
58    /// An error occurred while decoding RLP.
59    #[error("an error occurred deserializing RLP: {0}")]
60    RlpDecodeError(#[from] alloy_rlp::Error),
61    /// A consensus error occurred.
62    #[error("an error occurred during consensus checks: {0}")]
63    ConsensusError(#[from] reth_consensus::ConsensusError),
64}
65
66/// The result of running a test.
67#[derive(Debug)]
68pub struct CaseResult {
69    /// A description of the test.
70    pub desc: String,
71    /// The full path to the test.
72    pub path: PathBuf,
73    /// The result of the test.
74    pub result: Result<(), Error>,
75}
76
77impl CaseResult {
78    /// Create a new test result.
79    pub fn new(path: &Path, case: &impl Case, result: Result<(), Error>) -> Self {
80        Self { desc: case.description(), path: path.into(), result }
81    }
82}
83
84/// Assert that all the given tests passed and print the results to stdout.
85pub(crate) fn assert_tests_pass(suite_name: &str, path: &Path, results: &[CaseResult]) {
86    let (passed, failed, skipped) = categorize_results(results);
87
88    print_results(suite_name, path, &passed, &failed, &skipped);
89
90    assert!(failed.is_empty(), "Some tests failed (see above)");
91}
92
93/// Categorize test results into `(passed, failed, skipped)`.
94pub(crate) fn categorize_results(
95    results: &[CaseResult],
96) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) {
97    let mut passed = Vec::new();
98    let mut failed = Vec::new();
99    let mut skipped = Vec::new();
100
101    for case in results {
102        match case.result.as_ref().err() {
103            Some(Error::Skipped) => skipped.push(case),
104            Some(_) => failed.push(case),
105            None => passed.push(case),
106        }
107    }
108
109    (passed, failed, skipped)
110}
111
112/// Display the given test results to stdout.
113pub(crate) fn print_results(
114    suite_name: &str,
115    path: &Path,
116    passed: &[&CaseResult],
117    failed: &[&CaseResult],
118    skipped: &[&CaseResult],
119) {
120    println!("Suite: {suite_name} (at {})", path.display());
121    println!(
122        "Ran {} tests ({} passed, {} failed, {} skipped)",
123        passed.len() + failed.len() + skipped.len(),
124        passed.len(),
125        failed.len(),
126        skipped.len()
127    );
128
129    for case in skipped {
130        println!("[S] Case {} skipped", case.path.display());
131    }
132
133    for case in failed {
134        let error = case.result.as_ref().unwrap_err();
135        println!("[!] Case {} failed (description: {}): {}", case.path.display(), case.desc, error);
136    }
137}
ef_tests/result.rs

ef_tests/
result.rs
