vet

commit c75bc4ae63accf875958cd258fa74c96e8be7ae2
parent 5b9719eb475b1f45dd3403fe1c219634f7c20f9f
Author: andrewlaack-collab <andrew.laack@imbue.com>
Date:   Tue,  3 Feb 2026 18:55:46 +0000

Fix PR process (#11)

* Fix PR process

* Updated github action

* Added github pr comment

* Better output

* Black

* Improve output without changing vet

* Updated wording

* Updated gitignore

* Adds goal specification.
Diffstat:
M
.github/workflows/test.yml
 | 
23
+++++++++++++++++++++--
M
.gitignore
 | 
1
+
M
vet.toml
 | 
1
-
M
vet/truncation_test.py
 | 
24
++++++------------------

4 files changed, 28 insertions(+), 21 deletions(-)
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -27,9 +27,28 @@ jobs:
       - name: Run tests
         run: uv run pytest
 
+      - name: Fetch main branch for vet comparison
+        if: github.event_name == 'pull_request'
+        run: git fetch origin main:main
+
       - name: Run vet
         if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
+        continue-on-error: true
         env:
           ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-        run: uv run vet --config ci
-        continue-on-error: true
+          VET_GOAL: |
+            ${{ github.event.pull_request.title }}
+
+            Additional context (not necessarily part of the goal):
+            ${{ github.event.pull_request.body }}
+        run: uv run vet "$VET_GOAL" --base-commit main --confidence-threshold 0.9 --max-workers 4 --quiet --model opus > vet-output.txt 2>&1
+
+      - name: Comment vet results
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if [ ! -s vet-output.txt ]; then
+            echo "Vet found no issues." > vet-output.txt
+          fi
+          gh pr comment ${{ github.event.pull_request.number }} --body-file vet-output.txt
diff --git a/.gitignore b/.gitignore
@@ -79,3 +79,4 @@ wandb/
 
 # Project-specific
 .imbue/
+vet-output.txt
diff --git a/vet.toml b/vet.toml
@@ -2,7 +2,6 @@
 confidence_threshold = 0.9
 max_workers = 4
 quiet = true
-output_format = "json"
 base_commit = "main"
 model = "opus"
 
diff --git a/vet/truncation_test.py b/vet/truncation_test.py
@@ -31,9 +31,7 @@ SIMPLE_TOKEN_COUNTERS: list[Callable[[str], int]] = [
     char_div4_count,
 ]
 
-ALL_TOKEN_COUNTERS: list[Callable[[str], int]] = SIMPLE_TOKEN_COUNTERS + [
-    tiktoken_count
-]
+ALL_TOKEN_COUNTERS: list[Callable[[str], int]] = SIMPLE_TOKEN_COUNTERS + [tiktoken_count]
 
 ascii_text = st.text(
     alphabet=st.characters(min_codepoint=32, max_codepoint=126),
@@ -79,9 +77,7 @@ def test_context_budgets_sum_to_100():
     total_tokens=st.integers(min_value=0, max_value=1_000_000),
     budget=st.sampled_from(list(ContextBudget)),
 )
-def test_get_token_budget_is_mathematically_correct(
-    total_tokens: int, budget: ContextBudget
-):
+def test_get_token_budget_is_mathematically_correct(total_tokens: int, budget: ContextBudget):
     result = get_token_budget(total_tokens, budget)
     expected = total_tokens * budget.value // 100
     assert result == expected
@@ -93,9 +89,7 @@ def test_get_token_budget_is_mathematically_correct(
     truncate_end=st.booleans(),
     count_tokens=st.sampled_from(SIMPLE_TOKEN_COUNTERS),
 )
-def test_truncate_always_respects_token_limit_simple(
-    text: str, max_tokens: int, truncate_end: bool, count_tokens
-):
+def test_truncate_always_respects_token_limit_simple(text: str, max_tokens: int, truncate_end: bool, count_tokens):
     result, _ = truncate_to_token_limit(
         text,
         max_tokens=max_tokens,
@@ -123,9 +117,7 @@ def test_truncate_end_produces_prefix(text: str, max_tokens: int):
         truncate_end=True,
     )
 
-    assert text.startswith(result), (
-        f"Result '{result[:50]}...' is not a prefix of original"
-    )
+    assert text.startswith(result), f"Result '{result[:50]}...' is not a prefix of original"
 
     if was_truncated:
         assert len(result) < len(text)
@@ -144,9 +136,7 @@ def test_truncate_start_produces_suffix(text: str, max_tokens: int):
         truncate_end=False,
     )
 
-    assert text.endswith(result), (
-        f"Result '...{result[-50:]}' is not a suffix of original"
-    )
+    assert text.endswith(result), f"Result '...{result[-50:]}' is not a suffix of original"
 
     if was_truncated:
         assert len(result) < len(text)
@@ -210,9 +200,7 @@ def test_zero_budget_returns_empty_and_truncated(text: str, count_tokens):
     max_tokens=st.integers(min_value=0, max_value=500),
     truncate_end=st.booleans(),
 )
-def test_truncate_respects_limit_tiktoken(
-    text: str, max_tokens: int, truncate_end: bool
-):
+def test_truncate_respects_limit_tiktoken(text: str, max_tokens: int, truncate_end: bool):
     result, _ = truncate_to_token_limit(
         text,
         max_tokens=max_tokens,