Skip to content

Commit fd1f3dd

Browse files
FScholPerFScholPer
committed
feat(traceability): add coverage checker and reporting docs
1 parent 810199a commit fd1f3dd

6 files changed

Lines changed: 704 additions & 0 deletions

File tree

‎docs/how-to/test_to_doc_links.rst‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,43 @@ Limitations
5353
- Partial properties will lead to no Testlink creation.
5454
If you want a test to be linked, please ensure all requirement properties are provided.
5555
- Tests must be executed by Bazel first so `test.xml` files exist.
56+
57+
58+
CI/CD Gate for Linkage Percentage
59+
---------------------------------
60+
61+
To enforce traceability in CI:
62+
63+
1. Run tests.
64+
2. Generate ``needs.json``.
65+
3. Execute the traceability checker.
66+
67+
.. code-block:: bash
68+
69+
bazel test //...
70+
bazel build //:needs_json
71+
bazel run //scripts_bazel:traceability_coverage -- \
72+
--needs-json bazel-bin/needs_json/_build/needs/needs.json \
73+
--min-req-code 100 \
74+
--min-req-test 100 \
75+
--min-req-fully-linked 100 \
76+
--min-tests-linked 100 \
77+
--fail-on-broken-test-refs
78+
79+
The checker reports:
80+
81+
- Percentage of implemented requirements with ``source_code_link``
82+
- Percentage of implemented requirements with ``testlink``
83+
- Percentage of implemented requirements with both links (fully linked)
84+
- Percentage of test cases linked to at least one requirement
85+
- Broken testcase references to unknown requirement IDs
86+
87+
To check only unit tests, filter testcase types:
88+
89+
.. code-block:: bash
90+
91+
bazel run //scripts_bazel:traceability_coverage -- \
92+
--needs-json bazel-bin/needs_json/_build/needs/needs.json \
93+
--test-types unit-test
94+
95+
Use lower thresholds during rollout and tighten towards 100% over time.

‎docs/reference/commands.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
| `bazel run //:docs` | Builds documentation |
66
| `bazel run //:docs_check` | Verifies documentation correctness |
77
| `bazel run //:docs_combo` | Builds combined documentation with all external dependencies included |
8+
| `bazel run //scripts_bazel:traceability_coverage -- --needs-json bazel-bin/needs_json/needs.json --min-req-code 100 --min-req-test 100 --min-req-fully-linked 100 --min-tests-linked 100 --fail-on-broken-test-refs` | Calculates requirement/test traceability percentages and fails if thresholds are not met |
89
| `bazel run //:live_preview` | Creates a live_preview of the documentation viewable in a local server |
910
| `bazel run //:live_preview_combo_experimental` | Creates a live_preview of the full documentation with all dependencies viewable in a local server |
1011
| `bazel run //:ide_support` | Sets up a Python venv for esbonio (Remember to restart VS Code!) |

‎scripts_bazel/BUILD‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,3 +37,10 @@ py_binary(
3737
main = "merge_sourcelinks.py",
3838
visibility = ["//visibility:public"],
3939
)
40+
41+
py_binary(
42+
name = "traceability_coverage",
43+
srcs = ["traceability_coverage.py"],
44+
main = "traceability_coverage.py",
45+
visibility = ["//visibility:public"],
46+
)

‎scripts_bazel/tests/BUILD‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,12 @@ score_pytest(
3232
] + all_requirements,
3333
pytest_config = "//:pyproject.toml",
3434
)
35+
36+
score_pytest(
37+
name = "traceability_coverage_test",
38+
srcs = ["traceability_coverage_test.py"],
39+
deps = [
40+
"//scripts_bazel:traceability_coverage",
41+
] + all_requirements,
42+
pytest_config = "//:pyproject.toml",
43+
)

‎scripts_bazel/tests/traceability_coverage_test.py‎

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
# *******************************************************************************
2+
# Copyright (c) 2026 Contributors to the Eclipse Foundation
3+
#
4+
# See the NOTICE file(s) distributed with this work for additional
5+
# information regarding copyright ownership.
6+
#
7+
# This program and the accompanying materials are made available under the
8+
# terms of the Apache License Version 2.0 which is available at
9+
# https://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# SPDX-License-Identifier: Apache-2.0
12+
# *******************************************************************************
13+
14+
"""Tests for traceability_coverage.py."""
15+
16+
import json
17+
import os
18+
import subprocess
19+
import sys
20+
from pathlib import Path
21+
22+
_MY_PATH = Path(__file__).parent
23+
24+
25+
def _write_needs_json(tmp_path: Path) -> Path:
26+
needs_json = tmp_path / "needs.json"
27+
payload = {
28+
"current_version": "main",
29+
"versions": {
30+
"main": {
31+
"needs": {
32+
"REQ_1": {
33+
"id": "REQ_1",
34+
"type": "tool_req",
35+
"implemented": "YES",
36+
"source_code_link": "src/foo.py:10",
37+
"testlink": "",
38+
},
39+
"REQ_2": {
40+
"id": "REQ_2",
41+
"type": "tool_req",
42+
"implemented": "PARTIAL",
43+
"source_code_link": "",
44+
"testlink": "tests/test_foo.py::test_bar",
45+
},
46+
"REQ_3": {
47+
"id": "REQ_3",
48+
"type": "tool_req",
49+
"implemented": "NO",
50+
"source_code_link": "",
51+
"testlink": "",
52+
},
53+
"TC_1": {
54+
"id": "TC_1",
55+
"type": "testcase",
56+
"partially_verifies": "REQ_1, REQ_2",
57+
"fully_verifies": "",
58+
},
59+
"TC_2": {
60+
"id": "TC_2",
61+
"type": "testcase",
62+
"partially_verifies": "",
63+
"fully_verifies": "",
64+
},
65+
"TC_3": {
66+
"id": "TC_3",
67+
"type": "testcase",
68+
"partially_verifies": "",
69+
"fully_verifies": "REQ_UNKNOWN",
70+
},
71+
}
72+
}
73+
},
74+
}
75+
needs_json.write_text(json.dumps(payload), encoding="utf-8")
76+
return needs_json
77+
78+
79+
def test_traceability_coverage_thresholds_pass(tmp_path: Path) -> None:
80+
needs_json = _write_needs_json(tmp_path)
81+
output_json = tmp_path / "summary.json"
82+
83+
result = subprocess.run(
84+
[
85+
sys.executable,
86+
_MY_PATH.parent / "traceability_coverage.py",
87+
"--needs-json",
88+
str(needs_json),
89+
"--min-req-code",
90+
"50",
91+
"--min-req-test",
92+
"50",
93+
"--min-req-fully-linked",
94+
"0",
95+
"--min-tests-linked",
96+
"60",
97+
"--json-output",
98+
str(output_json),
99+
],
100+
capture_output=True,
101+
text=True,
102+
)
103+
104+
assert result.returncode == 0
105+
assert "Threshold check passed." in result.stdout
106+
assert output_json.exists()
107+
108+
summary = json.loads(output_json.read_text(encoding="utf-8"))
109+
assert summary["requirements"]["total"] == 2
110+
assert summary["requirements"]["with_code_link"] == 1
111+
assert summary["requirements"]["with_test_link"] == 1
112+
assert summary["requirements"]["fully_linked"] == 0
113+
assert summary["tests"]["total"] == 3
114+
assert summary["tests"]["linked_to_requirements"] == 2
115+
assert len(summary["tests"]["broken_references"]) == 1
116+
117+
118+
def test_traceability_coverage_thresholds_fail(tmp_path: Path) -> None:
119+
needs_json = _write_needs_json(tmp_path)
120+
121+
result = subprocess.run(
122+
[
123+
sys.executable,
124+
_MY_PATH.parent / "traceability_coverage.py",
125+
"--needs-json",
126+
str(needs_json),
127+
"--min-req-code",
128+
"80",
129+
"--min-req-test",
130+
"80",
131+
"--min-req-fully-linked",
132+
"80",
133+
"--min-tests-linked",
134+
"80",
135+
],
136+
capture_output=True,
137+
text=True,
138+
)
139+
140+
assert result.returncode == 2
141+
assert "Threshold check failed:" in result.stdout
142+
143+
144+
def test_traceability_coverage_fails_on_broken_refs(tmp_path: Path) -> None:
145+
needs_json = _write_needs_json(tmp_path)
146+
147+
result = subprocess.run(
148+
[
149+
sys.executable,
150+
_MY_PATH.parent / "traceability_coverage.py",
151+
"--needs-json",
152+
str(needs_json),
153+
"--min-req-code",
154+
"0",
155+
"--min-req-test",
156+
"0",
157+
"--min-req-fully-linked",
158+
"0",
159+
"--min-tests-linked",
160+
"0",
161+
"--fail-on-broken-test-refs",
162+
],
163+
capture_output=True,
164+
text=True,
165+
)
166+
167+
assert result.returncode == 2
168+
assert "broken testcase references found:" in result.stdout
169+
170+
171+
def test_traceability_coverage_prints_unlinked_requirements(tmp_path: Path) -> None:
172+
needs_json = _write_needs_json(tmp_path)
173+
174+
result = subprocess.run(
175+
[
176+
sys.executable,
177+
_MY_PATH.parent / "traceability_coverage.py",
178+
"--needs-json",
179+
str(needs_json),
180+
"--min-req-code",
181+
"0",
182+
"--min-req-test",
183+
"0",
184+
"--min-req-fully-linked",
185+
"0",
186+
"--min-tests-linked",
187+
"0",
188+
"--print-unlinked-requirements",
189+
],
190+
capture_output=True,
191+
text=True,
192+
)
193+
194+
assert result.returncode == 0
195+
assert "Unlinked requirement details:" in result.stdout
196+
assert "Missing source_code_link: REQ_2" in result.stdout
197+
assert "Missing testlink: REQ_1" in result.stdout
198+
assert "Not fully linked: REQ_1, REQ_2" in result.stdout
199+
200+
201+
def test_traceability_coverage_accepts_workspace_relative_needs_json(tmp_path: Path) -> None:
202+
workspace = tmp_path / "workspace"
203+
workspace.mkdir()
204+
needs_json = _write_needs_json(workspace)
205+
206+
env = dict(os.environ)
207+
env["BUILD_WORKSPACE_DIRECTORY"] = str(workspace)
208+
209+
result = subprocess.run(
210+
[
211+
sys.executable,
212+
_MY_PATH.parent / "traceability_coverage.py",
213+
"--needs-json",
214+
"needs.json",
215+
"--min-req-code",
216+
"0",
217+
"--min-req-test",
218+
"0",
219+
"--min-req-fully-linked",
220+
"0",
221+
"--min-tests-linked",
222+
"0",
223+
],
224+
capture_output=True,
225+
text=True,
226+
cwd=tmp_path,
227+
env=env,
228+
)
229+
230+
assert result.returncode == 0
231+
assert f"Traceability input: {needs_json}" in result.stdout

0 commit comments

Comments
 (0)