Gym/resources_servers/abstention/configs/abstention.yaml at main · hackIDLE/Gym · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
# Abstention Environment
# Trains a policy model to abstain from answering when unsure rather than
# hallucinating.  Uses a three-tier reward: correct (1.0) > abstain (lambda) >
# incorrect (0.0).  Verification uses an LLM judge (OMNISCIENCE_GRADER) instead
# of string matching.

abstention:
  resources_servers:
    abstention:
      entrypoint: app.py
      domain: rlhf
      verified: false
      description: Train models to abstain when unsure using three-tier reward on HotPotQA with LLM judge
      value: Improve calibration by rewarding abstention over incorrect answers

      # LLM judge model server
      judge_model_server:
        type: responses_api_models
        name: genrm_model
      judge_responses_create_params:
        input: []
        max_output_tokens: 64
        temperature: 0.0
        top_p: 1.0

      # Reward for abstaining (lambda). Must satisfy 0 < lambda < 1.
      abstention_reward: 0.5

      # Token the model outputs inside \boxed{} to signal abstention
      abstention_token: "[IDK]"

      # Reward for correct and incorrect answers
      correct_reward: 1.0
      incorrect_reward: 0.0

abstention_simple_agent:
  responses_api_agents:
    simple_agent:
      entrypoint: app.py
      resources_server:
        type: resources_servers
        name: abstention
      model_server:
        type: responses_api_models
        name: policy_model
      datasets:
        - name: hotpotqa_train
          type: train
          jsonl_fpath: resources_servers/abstention/data/hotpotqa_train.jsonl
          license: Creative Commons Attribution-ShareAlike 4.0 International
        - name: hotpotqa_val
          type: validation
          jsonl_fpath: resources_servers/abstention/data/hotpotqa_val.jsonl
          license: Creative Commons Attribution-ShareAlike 4.0 International
        - name: example
          type: example
          jsonl_fpath: resources_servers/abstention/data/example.jsonl