feat(Mocking): allow functions to be marked as mockable

Initial design wanted to let every tracable elements to also be mockable.
However, langgraph and `@traced` annotations uses different tracing mechanisms.
Keeping `@mockable` separate from the tracing system will create a cleaner separation of concern.

The `mockingStrategy` can be extended to support different types of strategies.
Currently, LLM strategy and mockito strategy is supported.

The LLM strategy is backwards compatible, however the feature is incomplete:

1. The prompt used to generate LLM responses need to be 1:1 with the c# world.
Specially, the current run history may be an important part of the prompt.
2. Dynamic input generation is currently unsupported.
3. Model selection is currently unsupported.
4. Caching is currently unsupported.

Author: akshaylive

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.1.72"
+version = "2.1.73"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.10"
@@ -19,6 +19,9 @@ dependencies = [
   "truststore>=0.10.1",
   "textual>=5.3.0",
   "pyperclip>=1.9.0",
+  "mockito>=1.5.4",
+  "hydra-core>=1.3.2",
+  "pydantic-function-models>=0.1.10",
 ]
 classifiers = [
   "Development Status :: 3 - Alpha",

samples/calculator/evals/eval-sets/default.json

@@ -10,14 +10,42 @@
     {
       "id": "test",
       "name": "Test Addition",
-      "inputs": {"a": 1, "b": 1, "operator":  "+"},
+      "inputs": {"a": 1, "b": 1, "operator":  "random"},
       "expectedOutput": {"result": 2},
-      "simulationInstructions": "",
       "expectedAgentBehavior": "",
-      "simulateInput": false,
-      "inputGenerationInstructions": "",
-      "simulateTools": false,
-      "toolsToSimulate": [],
+      "mockingStrategy": {
+        "type": "mockito",
+        "behaviors": [
+          {
+            "function": "get_random_operator",
+            "arguments": {
+              "args": [],
+              "kwargs": {}
+            },
+            "then": [
+              {
+                "type": "return",
+                "value": {"result": "+"}
+              }
+            ]
+          }
+        ]
+      },
+      "evalSetId": "default-eval-set-id",
+      "createdAt": "2025-09-04T18:54:58.378Z",
+      "updatedAt": "2025-09-04T18:55:55.416Z"
+    },
+    {
+      "id": "test",
+      "name": "Test Addition",
+      "inputs": {"a": 1, "b": 1, "operator":  "random"},
+      "expectedOutput": {"result": 2},
+      "expectedAgentBehavior": "",
+      "mockingStrategy": {
+        "type": "llm",
+        "prompt": "The random operator is '+'.",
+        "toolsToSimulate": [{"name":  "get_random_operator"}]
+      },
       "evalSetId": "default-eval-set-id",
       "createdAt": "2025-09-04T18:54:58.378Z",
       "updatedAt": "2025-09-04T18:55:55.416Z"

samples/calculator/main.py

@@ -1,16 +1,21 @@
+import random
+
 from pydantic.dataclasses import dataclass
 from enum import Enum
 
 from uipath.tracing import traced
 import logging
 
+from uipath._cli._evals.mocks.mocks import mockable
+
 logger = logging.getLogger(__name__)
 
 class Operator(Enum):
     ADD = "+"
     SUBTRACT = "-"
     MULTIPLY = "*"
     DIVIDE = "/"
+    RANDOM = "random"
 
 @dataclass
 class CalculatorInput:
@@ -22,13 +27,28 @@ class CalculatorInput:
 class CalculatorOutput:
     result: float
 
-# use InputTriggerEventArgs when called by UiPath EventTriggers
+@dataclass
+class Wrapper:
+    # Testing nested objects
+    result: Operator
+
+@traced()
+@mockable()
+def get_random_operator() -> Wrapper:
+    """Get a random operator."""
+    return Wrapper(result=random.choice([Operator.ADD, Operator.SUBTRACT, Operator.MULTIPLY, Operator.DIVIDE]))
+
+
 @traced()
-def main(input: CalculatorInput) -> CalculatorOutput:
-    result = 0.0
-    match input.operator:
+async def main(input: CalculatorInput) -> CalculatorOutput:
+    if input.operator == Operator.RANDOM:
+        operator = get_random_operator().result
+    else:
+        operator = input.operator
+    match operator:
         case Operator.ADD: result = input.a + input.b
         case Operator.SUBTRACT: result = input.a - input.b
         case Operator.MULTIPLY: result = input.a * input.b
-        case Operator.DIVIDE: result = input.a / input.b if input.b != 0 else 0
+        case Operator.DIVIDE: result = input.a / input.b if input.b != 0.0 else 0.0
+        case _: raise ValueError("Unknown operator")
     return CalculatorOutput(result=result)

src/uipath/_cli/_evals/_models/_evaluation_set.py

@@ -1,5 +1,5 @@
-from enum import IntEnum
-from typing import Any, Dict, List
+from enum import Enum, IntEnum
+from typing import Annotated, Any, Dict, List, Literal, Optional, Union
 
 from pydantic import BaseModel, ConfigDict, Field
 from pydantic.alias_generators import to_camel
@@ -9,28 +9,127 @@ class EvaluationSimulationTool(BaseModel):
     name: str = Field(..., alias="name")
 
 
+class MockingStrategyType(str, Enum):
+    LLM = "llm"
+    MOCKITO = "mockito"
+    UNKNOWN = "unknown"
+
+
+class BaseMockingStrategy(BaseModel):
+    pass
+
+
+class LLMMockingStrategy(BaseMockingStrategy):
+    type: Literal[MockingStrategyType.LLM] = MockingStrategyType.LLM
+    prompt: str = Field(..., alias="prompt")
+    tools_to_simulate: list[EvaluationSimulationTool] = Field(
+        ..., alias="toolsToSimulate"
+    )
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+"""
+{
+            "function": "postprocess",
+            "arguments": {
+              "args": [],
+              "kwargs": {"x": 3}
+            },
+            "then": [
+              {
+                "return": 3
+              },
+              {
+                "raise": {
+                  "__target__": "NotImplementedError"
+                }
+              }
+            ]
+          }
+          """
+
+
+class MockingArgument(BaseModel):
+    args: List[Any] = Field(default_factory=lambda: [], alias="args")
+    kwargs: Dict[str, Any] = Field(default_factory=lambda: {}, alias="kwargs")
+
+
+class MockingAnswerType(str, Enum):
+    RETURN = "return"
+    RAISE = "raise"
+
+
+class MockingAnswer(BaseModel):
+    type: MockingAnswerType
+    value: Any = Field(..., alias="value")
+
+
+class MockingBehavior(BaseModel):
+    function: str = Field(..., alias="function")
+    arguments: MockingArgument = Field(..., alias="arguments")
+    then: List[MockingAnswer] = Field(..., alias="then")
+
+
+class MockitoMockingStrategy(BaseMockingStrategy):
+    type: Literal[MockingStrategyType.MOCKITO] = MockingStrategyType.MOCKITO
+    behaviors: List[MockingBehavior] = Field(..., alias="config")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+KnownMockingStrategy = Annotated[
+    Union[LLMMockingStrategy, MockitoMockingStrategy],
+    Field(discriminator="type"),
+]
+
+
+class UnknownMockingStrategy(BaseMockingStrategy):
+    type: str = Field(..., alias="type")
+
+    model_config = ConfigDict(
+        validate_by_name=True, validate_by_alias=True, extra="allow"
+    )
+
+
+MockingStrategy = Union[KnownMockingStrategy, UnknownMockingStrategy]
+
+
+def migrate_mocking_strategy(data) -> MockingStrategy:
+    if data.get("simulate_tools") and "tools_to_simulate" in data:
+        return LLMMockingStrategy(
+            **{
+                "prompt": data["simulation_instructions"],
+                "toolsToSimulate": data["tools_to_simulate"],
+            }
+        )
+    else:
+        return UnknownMockingStrategy(type=MockingStrategyType.UNKNOWN)
+
+
 class EvaluationItem(BaseModel):
     """Individual evaluation item within an evaluation set."""
 
-    model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True)
+    model_config = ConfigDict(
+        alias_generator=to_camel, populate_by_name=True, extra="allow"
+    )
 
     id: str
     name: str
     inputs: Dict[str, Any]
     expected_output: Dict[str, Any]
     expected_agent_behavior: str = Field(default="", alias="expectedAgentBehavior")
-    simulation_instructions: str = Field(default="", alias="simulationInstructions")
-    simulate_input: bool = Field(default=False, alias="simulateInput")
-    input_generation_instructions: str = Field(
-        default="", alias="inputGenerationInstructions"
-    )
-    simulate_tools: bool = Field(default=False, alias="simulateTools")
-    tools_to_simulate: List[EvaluationSimulationTool] = Field(
-        default_factory=list, alias="toolsToSimulate"
-    )
     eval_set_id: str = Field(alias="evalSetId")
     created_at: str = Field(alias="createdAt")
     updated_at: str = Field(alias="updatedAt")
+    mocking_strategy: Optional[MockingStrategy] = Field(
+        default=None,
+        alias="mockingStrategy",
+    )
 
 
 class EvaluationSet(BaseModel):

src/uipath/_cli/_evals/_runtime.py

@@ -35,6 +35,7 @@
     UiPathEvalOutput,
     UiPathEvalRunExecutionOutput,
 )
+from .mocks.mocks import set_evaluation_item
 
 T = TypeVar("T", bound=UiPathBaseRuntime)
 C = TypeVar("C", bound=UiPathRuntimeContext)
@@ -137,6 +138,7 @@ async def execute(self) -> Optional[UiPathRuntimeResult]:
             evaluation_set_name=evaluation_set.name, score=0, evaluation_set_results=[]
         )
         for eval_item in evaluation_set.evaluations:
+            set_evaluation_item(eval_item)
             await event_bus.publish(
                 EvaluationEvents.CREATE_EVAL_RUN,
                 EvalRunCreatedEvent(

src/uipath/_cli/_evals/mocks/__init__.py

@@ -0,0 +1 @@
+"""UiPath mocking framework."""