Gym/resources_servers/terminus_judge/configs/terminus_judge_simple.yaml at main · hackIDLE/Gym · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
terminus_judge_resources_server:
  resources_servers:
    terminus_judge:
      entrypoint: app.py
      judge_model_server:
        type: responses_api_models
        name: policy_model
      judge_responses_create_params:
        input: []
        max_output_tokens: 1024
      judge_prompt_template_fpath: prompt_templates/terminus_prompt.txt
      judge_endpoint_max_concurrency: null
      judge_system_message: null
      judge_equal_label: "[[A=B]]"
      judge_not_equal_label: "[[A!=B]]"

      # Swap check: Run second judge pass with swapped expected/generated to detect positional bias
      # Enabled for simple prompt which uses symmetric A/B labels
      check_twice_swap: true
      # Reward when the second (swap) pass fails; default 0.0, can be -1.0
      reward_if_swap_fails: 0.0

      # Verification options (independent, can enable any combination)
      # - Both off: Only schema + task_complete check, reward=1.0 if pass
      # - String sim only: Fast string comparison, reward=1.0 if similarity >= threshold
      # - LLM judge only: Always call LLM judge for semantic equivalence
      # - Both on: String sim as fast-path, LLM judge as fallback when sim fails
      enable_string_similarity: true
      string_similarity_threshold: 0.9
      enable_llm_judge: true

      domain: agent
      verified: false
      description: single-step terminal based task (simple judge prompt)
      value: Improve on terminal-style tasks

terminus_judge_simple_agent:
  responses_api_agents:
    simple_agent:
      entrypoint: app.py
      max_steps: 1
      resources_server:
        type: resources_servers
        name: terminus_judge_resources_server
      model_server:
        type: responses_api_models
        name: policy_model
      datasets:
      - name: train
        type: train
        jsonl_fpath: resources_servers/terminus_judge/data/train.jsonl
        num_repeats: 1
        gitlab_identifier:
          dataset_name: terminus_judge
          version: 0.0.1
          artifact_fpath: train.jsonl
        license: Apache 2.0
      - name: validation
        type: validation
        jsonl_fpath: resources_servers/terminus_judge/data/validation.jsonl
        num_repeats: 1
        gitlab_identifier:
          dataset_name: terminus_judge
          version: 0.0.1
          artifact_fpath: validation.jsonl
        license: Apache 2.0
      - name: example
        type: example
        jsonl_fpath: resources_servers/terminus_judge/data/example.jsonl
        num_repeats: 1