Add mandatory mode parameter support to evaluation function and tests

Refactors the evaluation function to require a `mode` parameter (`demo` or `io_test`) for defining behavior. Updates unit tests and Postman collection to include the required `mode` parameter. Adds error handling for missing/unknown mode values with appropriate feedback.

Author: m-messer

evaluation_function/evaluation.py

@@ -76,31 +76,28 @@ def _upload_plots(images: list[Image.Image]) -> list[str]:
     return result
 
 
-def evaluation_function(response: Any, answer: Any, params: Params) -> Result:
-    tests = params.get("tests", [])
-    result = Result()
-
-    if not tests:
-        stdout, stderr, timed_out, images = _run_code(str(response), "")
-        if timed_out:
-            result.add_feedback("error", f"Code timed out after {_TIMEOUT}s.")
-        elif stderr and not stdout:
-            result.add_feedback("error", _code_block("Error", stderr.strip()))
-        else:
-            parts = [_code_block("Output", stdout.rstrip() or "(no output)")]
-            parts.extend(_upload_plots(images))
-            result.add_feedback("output", "\n\n".join(parts))
-        return result
+def _evaluate_demo(response: str, result: Result) -> Result:
+    stdout, stderr, timed_out, images = _run_code(response, "")
+    if timed_out:
+        result.add_feedback("error", f"Code timed out after {_TIMEOUT}s.")
+    elif stderr and not stdout:
+        result.add_feedback("error", _code_block("Error", stderr.strip()))
+    else:
+        parts = [_code_block("Output", stdout.rstrip() or "(no output)")]
+        parts.extend(_upload_plots(images))
+        result.add_feedback("output", "\n\n".join(parts))
+    return result
 
 
+def _evaluate_io(response: str, tests: list, result: Result) -> Result:
     passed = 0
 
     for i, test in enumerate(tests, 1):
         stdin = test.get("input", "")
         expected = test.get("expected_output", "").rstrip()
         hidden = test.get("hidden", False)
 
-        stdout, stderr, timed_out, images = _run_code(str(response), stdin)
+        stdout, stderr, timed_out, images = _run_code(response, stdin)
         actual = stdout.rstrip()
         label = f"Hidden test {i}" if hidden else f"Test {i}"
 
@@ -143,4 +140,16 @@ def evaluation_function(response: Any, answer: Any, params: Params) -> Result:
 
     result.is_correct = passed == len(tests)
     result.add_feedback("summary", f"{passed}/{len(tests)} tests passed.")
-    return result
+    return result
+
+
+def evaluation_function(response: Any, answer: Any, params: Params) -> Result:
+    result = Result()
+    mode = params.get("mode")
+    if mode not in ("demo", "io_test"):
+        result.add_feedback("error", f"Unknown or missing mode: {mode!r}. Expected 'demo' or 'io_test'.")
+        return result
+
+    if mode == "demo":
+        return _evaluate_demo(str(response), result)
+    return _evaluate_io(str(response), params.get("tests", []), result)

evaluation_function/evaluation_test.py

@@ -11,7 +11,7 @@
 
 
 def _params(*tests):
-    return {"tests": list(tests)}
+    return {"mode": "io_test", "tests": list(tests)}
 
 
 def _test(inp, expected, hidden=False):
@@ -54,11 +54,17 @@ def test_runtime_error(self):
         self.assertIn("```", result["feedback"])
 
     def test_no_tests(self):
-        result = evaluation_function(_SQUARE_CODE, None, {}).to_dict()
+        result = evaluation_function(_SQUARE_CODE, None, {"mode": "demo"}).to_dict()
 
         self.assertFalse(result["is_correct"])
         self.assertIn("```", result["feedback"])
 
+    def test_missing_mode(self):
+        result = evaluation_function(_SQUARE_CODE, None, {}).to_dict()
+
+        self.assertFalse(result["is_correct"])
+        self.assertIn("mode", result["feedback"])
+
 
 _PLOT_CODE = "import matplotlib.pyplot as plt\nplt.plot([1, 2, 3])\n"
 _MULTI_PLOT_CODE = (
@@ -73,7 +79,7 @@ class TestImageGeneration(unittest.TestCase):
 
     @patch("evaluation_function.evaluation.upload_image", return_value=_FAKE_URL)
     def test_single_plot_demo_mode(self, mock_upload):
-        result = evaluation_function(_PLOT_CODE, None, {}).to_dict()
+        result = evaluation_function(_PLOT_CODE, None, {"mode": "demo"}).to_dict()
         mock_upload.assert_called_once()
         self.assertIn("![Plot 1]", result["feedback"])
 
@@ -91,14 +97,14 @@ def test_single_plot_failing_test(self, mock_upload):
 
     @patch("evaluation_function.evaluation.upload_image", return_value=_FAKE_URL)
     def test_multiple_plots(self, mock_upload):
-        result = evaluation_function(_MULTI_PLOT_CODE, None, {}).to_dict()
+        result = evaluation_function(_MULTI_PLOT_CODE, None, {"mode": "demo"}).to_dict()
         self.assertEqual(mock_upload.call_count, 2)
         self.assertIn("![Plot 1]", result["feedback"])
         self.assertIn("![Plot 2]", result["feedback"])
 
     @patch("evaluation_function.evaluation.upload_image", side_effect=ImageUploadError)
     def test_upload_failure_graceful(self, mock_upload):
-        result = evaluation_function(_PLOT_CODE, None, {}).to_dict()
+        result = evaluation_function(_PLOT_CODE, None, {"mode": "demo"}).to_dict()
         self.assertNotIn("![Plot", result["feedback"])
 
     def test_run_code_captures_images(self):