use YAML expectation tests with fixed seeds

build.rs

@@ -7,6 +7,7 @@ const TEST_PREFIX: &str = "
 #[cfg(test)]
 use pretty_assertions::assert_eq;
 use crate::{grammar,lexer};
+use std::collections::BTreeMap;
 use std::io::Write;
 
 // to let us use pretty_assertions with strings, we write a newtype
@@ -25,6 +26,15 @@ impl<'a> core::fmt::Debug for StringWrapper<'a> {
 fn assert_eq(x: &str, y: &str) {
   assert_eq!(StringWrapper {wrapped: x}, StringWrapper {wrapped: y});
 }
+
+fn get_expectations(contents: &str) -> BTreeMap<u64, String> {
+  let map: BTreeMap<String, String> = serde_yaml::from_str(contents).unwrap();
+  let mut result = BTreeMap::new();
+  for (k, v) in map.into_iter() {
+    result.insert(k.parse().unwrap(), v);
+  }
+  result
+}
 ";
 
 const TEST_TEMPLATE: &str = "
@@ -48,18 +58,15 @@ fn test_%PREFIX%() {
     );
   }
 
-  if let Ok(expected) = std::fs::read_to_string(\"%ROOT%/tests/%PREFIX%.output\") {
-    let possibilities = expected.lines().collect::<Vec<&str>>();
-    let mut buf = Vec::new();
-    for stmt in stmts {
-      state.execute(&stmt, &mut buf).unwrap();
-    }
-
-    let out = std::str::from_utf8(&buf).unwrap();
-    if !possibilities.contains(&out.trim()) {
-      panic!(\"Got output\\n  `{}`\\nbut expected one of the following:\\n{}\\n\",
-        &out.trim(),
-        possibilities.iter().map(|x| format!(\"  `{}`\\n\", x)).collect::<Vec<String>>().join(\", \"),
+  if let Ok(contents) = std::fs::read_to_string(\"%ROOT%/tests/%PREFIX%.output\") {
+    let expectations = get_expectations(&contents);
+    for (seed, expectation) in expectations {
+      let st = crate::interp::State::new_from_seed(seed);
+      let mut out = Vec::new();
+      st.run_with_writer(source, &mut out).unwrap();
+      assert_eq(
+        std::str::from_utf8(&out).unwrap().trim(),
+        expectation.trim(),
       );
     }
   }

src/interp.rs

@@ -347,6 +347,20 @@ impl State {
         Ok(())
     }
 
+    /// Evaluate this string as a standalone program, writing the
+    /// results to the provided writer.
+    pub fn run_with_writer(&self, src: &str, w: &mut impl std::io::Write) -> Result<(), Error> {
+        let lexed = crate::lexer::tokens(src);
+        let stmts = self
+            .parser
+            .parse(&mut self.ast.borrow_mut(), lexed)
+            .map_err(|err| anyhow!("Got {:?}", err))?;
+        for stmt in stmts {
+            self.execute(&stmt, &mut *w)?;
+        }
+        Ok(())
+    }
+
     /// Evaluate this string as a fragment in a REPL, writing the
     /// results to stdout. One way this differs from the standalone
     /// program is that it actually tries parsing twice: first it

src/main.rs

@@ -34,7 +34,7 @@ impl Opts {
             None
         };
         let mut files = Vec::new();
-        if let Some(fs) = matches.values_of("FILES") {
+        if let Some(fs) = matches.values_of("input") {
             files.extend(fs.map(|x| x.to_string()));
         }
         Opts { seed, files }

tests/fixed.output

@@ -1,4 +1,4 @@
-papa
-pupu
-baba
-bubu
+0: baba
+2: papa
+4: bubu
+5: pupu

tests/hello.output

@@ -1 +1 @@
-Hello, world!
+'0': 'Hello, world!'

tests/simple.output

@@ -1,4 +1,4 @@
-pa
-pu
-ba
-bu
+2: pa
+5: pu
+0: ba
+4: bu