commit f074273939ce18cf8935b040d093e9d15a91855a
parent e85b72901bbce9af9470099b45ad51bc7686ef95
Author: andrewlaack-collab <andrew.laack@imbue.com>
Date: Tue, 10 Feb 2026 09:40:41 +0000
Update issues found to use github review format instead of pr comments. (#46)
* Update issues found to use github review format instead of pr comments.
* Update wording
* Better wording again
* Added tests for github formatting
* Added comment, updated wording to be more consistent
* whitespace
* Don't redirect errors, useful for debugging.
* Refactoring to make exit status codes more sensible (realized 1 was default error for python failures so shouldn't be using that for issues found).
* Fix comment fallback behavior
* Updated exit status code descriptors
Diffstat:
8 files changed, 169 insertions(+), 24 deletions(-)
diff --git a/.github/workflows/vet.yml b/.github/workflows/vet.yml
@@ -3,7 +3,6 @@ name: Vet
permissions:
contents: read
pull-requests: write
- issues: write
on:
pull_request:
@@ -20,7 +19,9 @@ jobs:
- uses: actions/setup-python@v5
with:
python-version: "3.11"
- - run: pip install git+https://github.com/imbue-ai/vet.git
+ # we want to run the current version of vet against the PR, not necessarily the deployed / main version.
+ # this is the only difference between this vet.yml and the recommended one in the readme.
+ - run: pip install .
- name: Run vet
if: github.event.pull_request.head.repo.full_name == github.repository
env:
@@ -33,11 +34,17 @@ jobs:
${{ github.event.pull_request.body }}
run: |
set +e
- vet "$VET_GOAL" --quiet --base-commit "${{ github.event.pull_request.base.sha }}" > "$RUNNER_TEMP/vet-output.txt" 2>&1
+ vet "$VET_GOAL" --quiet --output-format github \
+ --base-commit "${{ github.event.pull_request.base.sha }}" \
+ > "$RUNNER_TEMP/review.json"
status=$?
- if [ ! -s "$RUNNER_TEMP/vet-output.txt" ]; then
- echo "Vet found no issues." > "$RUNNER_TEMP/vet-output.txt"
- fi
- gh pr comment "${{ github.event.pull_request.number }}" --body-file "$RUNNER_TEMP/vet-output.txt"
- if [ "$status" -eq 1 ]; then exit 0; fi
- exit "$status"
+ if [ "$status" -ne 0 ] && [ "$status" -ne 10 ]; then exit "$status"; fi
+
+ jq --arg sha "${{ github.event.pull_request.head.sha }}" \
+ '. + {commit_id: $sha}' "$RUNNER_TEMP/review.json" > "$RUNNER_TEMP/review-final.json"
+
+ gh api "repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews" \
+ --method POST --input "$RUNNER_TEMP/review-final.json" || \
+ gh pr comment "${{ github.event.pull_request.number }}" \
+ --body "$(jq -r '[.body] + [.comments[] | "**\(.path):\(.line)**\n\n\(.body)"] | join("\n\n---\n\n")' "$RUNNER_TEMP/review-final.json")"
+ exit 0
diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md
@@ -74,9 +74,10 @@ Example configuration:
The following are the **expected** exit status codes for vet:
-- `0` - Success, no issues found
-- `1` - Issues were found in the code
+- `0` - No issues found
+- `1` - Unexpected runtime error
- `2` - Invalid arguments or configuration
+- `10` - Issues were found in the code
## Concepts
diff --git a/README.md b/README.md
@@ -36,7 +36,6 @@ name: Vet
permissions:
contents: read
pull-requests: write
- issues: write
on:
pull_request:
@@ -66,17 +65,23 @@ jobs:
${{ github.event.pull_request.body }}
run: |
set +e
- vet "$VET_GOAL" --quiet --base-commit "${{ github.event.pull_request.base.sha }}" > "$RUNNER_TEMP/vet-output.txt" 2>&1
+ vet "$VET_GOAL" --quiet --output-format github \
+ --base-commit "${{ github.event.pull_request.base.sha }}" \
+ > "$RUNNER_TEMP/review.json"
status=$?
- if [ ! -s "$RUNNER_TEMP/vet-output.txt" ]; then
- echo "Vet found no issues." > "$RUNNER_TEMP/vet-output.txt"
- fi
- gh pr comment "${{ github.event.pull_request.number }}" --body-file "$RUNNER_TEMP/vet-output.txt"
- if [ "$status" -eq 1 ]; then exit 0; fi
- exit "$status"
+ if [ "$status" -ne 0 ] && [ "$status" -ne 10 ]; then exit "$status"; fi
+
+ jq --arg sha "${{ github.event.pull_request.head.sha }}" \
+ '. + {commit_id: $sha}' "$RUNNER_TEMP/review.json" > "$RUNNER_TEMP/review-final.json"
+
+ gh api "repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews" \
+ --method POST --input "$RUNNER_TEMP/review-final.json" || \
+ gh pr comment "${{ github.event.pull_request.number }}" \
+ --body "$(jq -r '[.body] + [.comments[] | "**\(.path):\(.line)**\n\n\(.body)"] | join("\n\n---\n\n")' "$RUNNER_TEMP/review-final.json")"
+ exit 0
```
-NOTE: This will not fail in CI if Vet finds an issue. This will only add a comment to the PR.
+NOTE: This will not fail in CI if Vet finds an issue.
#### Environment variables
@@ -119,12 +124,14 @@ Vet snapshots the repo and diff, optionally adds a goal and agent conversation,
## Output & exit codes
- Exit code `0`: no issues found
-- Exit code `1`: issues found
+- Exit code `1`: unexpected runtime error
- Exit code `2`: invalid usage/configuration error
+- Exit code `10`: issues found
Output formats:
- `text`
- `json`
+- `github`
## Configuration
diff --git a/skills/vet/SKILL.md b/skills/vet/SKILL.md
@@ -70,6 +70,6 @@ Vet analyzes the full git diff from the base commit. This may include changes fr
- `--base-commit REF`: Git ref for diff base (default: HEAD)
- `--model MODEL`: LLM model to use (default: claude-4-6-opus)
- `--confidence-threshold N`: Minimum confidence 0.0-1.0 (default: 0.8)
-- `--output-format FORMAT`: Output as `text` or `json`
+- `--output-format FORMAT`: Output as `text`, `json`, or `github`
- `--quiet`: Suppress progress output
- `--help`: Show comprehensive list of options
diff --git a/vet/__snapshots__/formatters_test.ambr b/vet/__snapshots__/formatters_test.ambr
@@ -0,0 +1,35 @@
+# serializer version: 1
+# name: test_format_github_review_mixed_issues
+ dict({
+ 'body': '''
+ **Vet found 2 issues.**
+
+ ---
+
+ **[incorrect_function_implementation]** (severity 2/5) (confidence 0.75)
+
+ General architecture concern
+ ''',
+ 'comments': list([
+ dict({
+ 'body': '''
+ **[incorrect_function_implementation]** (severity 2/5) (confidence 0.75)
+
+ This function has a bug
+ ''',
+ 'line': 10,
+ 'path': 'src/app.py',
+ 'side': 'RIGHT',
+ }),
+ ]),
+ 'event': 'COMMENT',
+ })
+# ---
+# name: test_format_github_review_no_issues
+ dict({
+ 'body': '**Vet found 0 issues.**',
+ 'comments': list([
+ ]),
+ 'event': 'COMMENT',
+ })
+# ---
diff --git a/vet/cli/main.py b/vet/cli/main.py
@@ -33,6 +33,7 @@ from vet.cli.models import get_models_by_provider
from vet.cli.models import validate_model_id
from vet.formatters import OUTPUT_FIELDS
from vet.formatters import OUTPUT_FORMATS
+from vet.formatters import format_github_review
from vet.formatters import format_issue_text
from vet.formatters import issue_to_dict
from vet.formatters import validate_output_fields
@@ -503,6 +504,9 @@ def main(argv: list[str] | None = None) -> int:
if not issues:
if args.output_format == "json":
print(json.dumps({"issues": []}, indent=2), file=output_stream)
+ elif args.output_format == "github":
+ payload = format_github_review(issues, output_fields)
+ print(json.dumps(payload, indent=2), file=output_stream)
elif not args.quiet:
print("No issues found.", file=output_stream)
return 0
@@ -510,12 +514,15 @@ def main(argv: list[str] | None = None) -> int:
if args.output_format == "json":
issues_list = [issue_to_dict(issue, output_fields) for issue in issues]
print(json.dumps({"issues": issues_list}, indent=2), file=output_stream)
+ elif args.output_format == "github":
+ payload = format_github_review(issues, output_fields)
+ print(json.dumps(payload, indent=2), file=output_stream)
else:
for issue in issues:
print(format_issue_text(issue, output_fields), file=output_stream)
print(file=output_stream)
- return 1
+ return 10
finally:
if output_file is not None:
output_file.close()
diff --git a/vet/formatters.py b/vet/formatters.py
@@ -4,7 +4,7 @@ from pydantic import BaseModel
from vet.imbue_core.data_types import IdentifiedVerifyIssue
-OUTPUT_FORMATS = ["text", "json"]
+OUTPUT_FORMATS = ["text", "json", "github"]
OUTPUT_FIELDS = [
"issue_code",
@@ -91,3 +91,47 @@ def issue_to_dict(issue: IdentifiedVerifyIssue, fields: list[str]) -> dict:
if "line_number" in fields and output.line_number_end is not None:
include_fields.add("line_number_end")
return output.model_dump(mode="json", include=include_fields)
+
+
+def _format_review_comment_body(issue: IdentifiedVerifyIssue, fields: list[str]) -> str:
+ parts: list[str] = []
+ header = []
+ if "issue_code" in fields:
+ header.append(f"**[{issue.code}]**")
+ if "severity" in fields and issue.severity_score:
+ header.append(f"(severity {issue.severity_score.raw:.0f}/5)")
+ if "confidence" in fields and issue.confidence_score:
+ header.append(f"(confidence {issue.confidence_score.normalized:.2f})")
+ if header:
+ parts.append(" ".join(header))
+ if "description" in fields:
+ parts.append(issue.description)
+ return "\n\n".join(parts)
+
+
+def format_github_review(
+ issues: tuple[IdentifiedVerifyIssue, ...],
+ fields: list[str],
+) -> dict:
+ inline = [i for i in issues if i.location and i.location[0].filename]
+ body_only = [i for i in issues if not i.location or not i.location[0].filename]
+
+ count = len(issues)
+ noun = "issue" if count == 1 else "issues"
+ body = f"**Vet found {count} {noun}.**"
+ if body_only:
+ sections = [_format_review_comment_body(i, fields) for i in body_only]
+ body += "\n\n---\n\n" + "\n\n".join(sections)
+
+ comments = []
+ for issue in inline:
+ loc = issue.location[0]
+ comment: dict = {
+ "path": loc.filename,
+ "line": loc.line_start,
+ "side": "RIGHT",
+ "body": _format_review_comment_body(issue, fields),
+ }
+ comments.append(comment)
+
+ return {"body": body, "event": "COMMENT", "comments": comments}
diff --git a/vet/formatters_test.py b/vet/formatters_test.py
@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from syrupy.assertion import SnapshotAssertion
+
+from vet.formatters import OUTPUT_FIELDS, format_github_review
+from vet.imbue_core.data_types import (
+ ConfidenceScore,
+ IdentifiedVerifyIssue,
+ IssueCode,
+ IssueLocation,
+ SeverityScore,
+)
+
+
+def _make_issue(
+ *,
+ description: str = "Buffer overflow",
+ severity_raw: float = 2.5,
+ confidence_raw: float = 0.75,
+ filename: str | None = "src/foo.py",
+ line_start: int = 40,
+ line_end: int = 50,
+) -> IdentifiedVerifyIssue:
+ location = ()
+ if filename is not None:
+ location = (IssueLocation(line_start=line_start, line_end=line_end, filename=filename),)
+ return IdentifiedVerifyIssue(
+ issue_id="test-issue-id",
+ code=IssueCode.INCORRECT_FUNCTION_IMPLEMENTATION,
+ description=description,
+ severity_score=SeverityScore(raw=severity_raw, normalized=severity_raw / 5.0),
+ confidence_score=ConfidenceScore(raw=confidence_raw, normalized=confidence_raw),
+ location=location,
+ )
+
+
+def test_format_github_review_no_issues(snapshot: SnapshotAssertion) -> None:
+ assert format_github_review((), OUTPUT_FIELDS) == snapshot
+
+
+def test_format_github_review_mixed_issues(snapshot: SnapshotAssertion) -> None:
+ inline_issue = _make_issue(description="This function has a bug", filename="src/app.py", line_start=10)
+ body_issue = _make_issue(description="General architecture concern", filename=None)
+ assert format_github_review((inline_issue, body_issue), OUTPUT_FIELDS) == snapshot
