commit e9ce6487366fd383e4c541fb29f7ac053315b74b
parent 676fdf85fe655e65fc0d46cb26766d2cb42a7742
Author: andrewlaack-collab <andrew.laack@imbue.com>
Date: Thu, 12 Feb 2026 00:53:49 +0000
Updated post processing steps to respect custom configs (#64)
Diffstat:
4 files changed, 34 insertions(+), 13 deletions(-)
diff --git a/vet/issue_identifiers/agentic_issue_collation.py b/vet/issue_identifiers/agentic_issue_collation.py
@@ -22,7 +22,7 @@ from vet.issue_identifiers.common import format_issue_identification_guide_for_l
from vet.issue_identifiers.common import generate_issues_from_response_texts
from vet.issue_identifiers.common import generate_response_from_claude_code
from vet.issue_identifiers.common import get_claude_code_options
-from vet.issue_identifiers.identification_guides import ISSUE_IDENTIFICATION_GUIDES_BY_ISSUE_CODE
+from vet.issue_identifiers.identification_guides import IssueIdentificationGuide
from vet.issue_identifiers.utils import ReturnCapturingGenerator
COLLATION_PROMPT_TEMPLATE = """You are reviewing the results from parallel code analysis for potential issues.
@@ -72,12 +72,12 @@ def _get_collation_prompt(
identifier_inputs: CommitInputs,
enabled_issue_codes: tuple[IssueCode, ...],
generated_issues: str,
+ guides_by_issue_code: dict[IssueCode, IssueIdentificationGuide],
) -> str:
# Sort issue codes to make the resulting prompts deterministic (for snapshot tests and LLM caching)
sorted_issue_codes = sorted(enabled_issue_codes)
formatted_guides = {
- code: format_issue_identification_guide_for_llm(ISSUE_IDENTIFICATION_GUIDES_BY_ISSUE_CODE[code])
- for code in sorted_issue_codes
+ code: format_issue_identification_guide_for_llm(guides_by_issue_code[code]) for code in sorted_issue_codes
}
env = jinja2.Environment(undefined=jinja2.StrictUndefined)
@@ -118,6 +118,7 @@ def collate_issues_with_agent(
project_context: ProjectContext,
config: VetConfig,
enabled_issue_codes: tuple[IssueCode, ...],
+ guides_by_issue_code: dict[IssueCode, IssueIdentificationGuide],
) -> Generator[GeneratedIssueSchema, None, IssueIdentificationDebugInfo]:
"""
Collate issues from multiple issue identifiers.
@@ -128,6 +129,7 @@ def collate_issues_with_agent(
project_context: Loaded data corresponding to the inputs, e.g. diffs or files.
config: Settings
enabled_issue_codes: The issue types used by the issue identifiers.
+ guides_by_issue_code: Mapping from issue codes to their identification guides (including any custom overrides).
Returns:
A generator of collated issues. Returns IssueIdentificationDebugInfo after the generator is exhausted.
@@ -149,7 +151,11 @@ def collate_issues_with_agent(
)
combined_issues_string = _convert_parsed_issues_to_combined_string(all_issues)
collation_prompt = _get_collation_prompt(
- project_context, collation_inputs, enabled_issue_codes, combined_issues_string
+ project_context,
+ collation_inputs,
+ enabled_issue_codes,
+ combined_issues_string,
+ guides_by_issue_code,
)
claude_response = generate_response_from_claude_code(collation_prompt, options)
assert claude_response is not None
diff --git a/vet/issue_identifiers/issue_deduplication.py b/vet/issue_identifiers/issue_deduplication.py
@@ -19,7 +19,7 @@ from vet.issue_identifiers.common import GeneratedResponseSchema
from vet.issue_identifiers.common import extract_invocation_info_from_costed_response
from vet.issue_identifiers.common import format_issue_identification_guide_for_llm
from vet.issue_identifiers.common import generate_issues_from_response_texts
-from vet.issue_identifiers.identification_guides import ISSUE_IDENTIFICATION_GUIDES_BY_ISSUE_CODE
+from vet.issue_identifiers.identification_guides import IssueIdentificationGuide
from vet.issue_identifiers.utils import ReturnCapturingGenerator
DEDUPLICATION_PROMPT_TEMPLATE = """[ROLE=USER]
@@ -69,12 +69,12 @@ Do not output any other JSON, only the consolidated issues in the specified form
def _get_deduplication_prompt(
enabled_issue_codes: Iterable[IssueCode],
generated_issues: str,
+ guides_by_issue_code: dict[IssueCode, IssueIdentificationGuide],
) -> str:
# Sort issue codes to make the resulting prompts deterministic (for snapshot tests and LLM caching)
sorted_issue_codes = sorted(enabled_issue_codes)
formatted_guides = {
- code: format_issue_identification_guide_for_llm(ISSUE_IDENTIFICATION_GUIDES_BY_ISSUE_CODE[code])
- for code in sorted_issue_codes
+ code: format_issue_identification_guide_for_llm(guides_by_issue_code[code]) for code in sorted_issue_codes
}
env = jinja2.Environment(undefined=jinja2.StrictUndefined)
@@ -107,6 +107,7 @@ def deduplicate_issues(
issue_generator: Generator[GeneratedIssueSchema, None, IssueIdentificationDebugInfo],
config: VetConfig,
enabled_issue_codes: Iterable[IssueCode],
+ guides_by_issue_code: dict[IssueCode, IssueIdentificationGuide],
) -> Generator[GeneratedIssueSchema, None, IssueIdentificationDebugInfo]:
"""
Deduplicate issues from multiple issue identifiers.
@@ -115,6 +116,7 @@ def deduplicate_issues(
issues: The issues to deduplicate.
config: Settings
enabled_issue_codes: The issue types used by the issue identifiers.
+ guides_by_issue_code: Mapping from issue codes to their identification guides (including any custom overrides).
Returns:
A generator of deduplicated issues. Returns IssueIdentificationDebugInfo after the generator is exhausted.
@@ -149,7 +151,7 @@ def deduplicate_issues(
# As per above TODO, only deduplicate over issues that passed filtration
combined_issues_string = _convert_parsed_issues_to_combined_string(issues_passing_filtration)
- prompt = _get_deduplication_prompt(enabled_issue_codes, combined_issues_string)
+ prompt = _get_deduplication_prompt(enabled_issue_codes, combined_issues_string, guides_by_issue_code)
costed_response = language_model.complete_with_usage_sync(
prompt,
diff --git a/vet/issue_identifiers/issue_evaluation.py b/vet/issue_identifiers/issue_evaluation.py
@@ -24,7 +24,7 @@ from vet.issue_identifiers.common import GeneratedIssueSchema
from vet.issue_identifiers.common import extract_invocation_info_from_costed_response
from vet.issue_identifiers.common import format_issue_identification_guide_for_llm
from vet.issue_identifiers.harnesses.single_prompt import USER_REQUEST_PREFIX_TEMPLATE
-from vet.issue_identifiers.identification_guides import ISSUE_IDENTIFICATION_GUIDES_BY_ISSUE_CODE
+from vet.issue_identifiers.identification_guides import IssueIdentificationGuide
from vet.issue_identifiers.utils import ReturnCapturingGenerator
from vet.truncation import ContextBudget
from vet.truncation import get_available_tokens
@@ -110,12 +110,12 @@ def _format_prompt(
config: VetConfig,
inputs: IdentifierInputs,
is_code_based_issue: bool,
+ formatted_guide: str,
) -> str:
env = jinja2.Environment(undefined=jinja2.StrictUndefined)
prompt_template = _get_full_prompt_template(is_code_based_issue)
jinja_template = env.from_string(prompt_template)
issue_code = IssueCode(issue.issue_code)
- guide = format_issue_identification_guide_for_llm(ISSUE_IDENTIFICATION_GUIDES_BY_ISSUE_CODE[issue_code])
criteria = CODE_BASED_CRITERIA if is_code_based_issue else CONVERSATION_BASED_CRITERIA
response_class = CodeBasedEvaluationResponse if is_code_based_issue else ConversationBasedEvaluationResponse
@@ -125,7 +125,7 @@ def _format_prompt(
"cache_full_prompt": config.cache_full_prompt,
"issue_description": issue.description,
"issue_code": issue_code,
- "guide": guide,
+ "guide": formatted_guide,
"criteria": criteria,
"response_schema": response_class.model_json_schema(),
"is_code_based_issue": is_code_based_issue,
@@ -180,6 +180,7 @@ def evaluate_code_issue_through_llm(
project_context: ProjectContext,
config: VetConfig,
is_code_based_issue: bool,
+ formatted_guide: str,
) -> tuple[bool, tuple[LLMResponse, ...]]:
"""
Args:
@@ -206,7 +207,7 @@ def evaluate_code_issue_through_llm(
language_model = build_language_model_from_config(config.language_model_generation_config)
- prompt = _format_prompt(issue, project_context, config, inputs, is_code_based_issue)
+ prompt = _format_prompt(issue, project_context, config, inputs, is_code_based_issue, formatted_guide)
costed_response = language_model.complete_with_usage_sync(
prompt,
params=LanguageModelGenerationParams(temperature=0.0, max_tokens=config.max_output_tokens),
@@ -260,6 +261,7 @@ def filter_issues(
config: VetConfig,
# Currently, the LLM-based filter only works reliably for code-related issue types.
is_code_based_issue_generator: bool,
+ guides_by_issue_code: dict[IssueCode, IssueIdentificationGuide],
) -> Generator[GeneratedIssueSchema, None, IssueIdentificationDebugInfo]:
"""
Filter issues based on the evaluation.
@@ -269,6 +271,7 @@ def filter_issues(
inputs: The inputs which determine the content provided to the evaluator.
project_context: Loaded data corresponding to the inputs, e.g. diffs or files.
config: Settings
+ guides_by_issue_code: Mapping from issue codes to their identification guides (including any custom overrides).
Returns:
A generator of issues with the passes_filtration flag set.
@@ -282,8 +285,15 @@ def filter_issues(
for issue in issue_generator_with_capture:
passes_filtration = evaluate_issue_through_confidence(issue, config)
if passes_filtration:
+ issue_code = IssueCode(issue.issue_code)
+ formatted_guide = format_issue_identification_guide_for_llm(guides_by_issue_code[issue_code])
passes_filtration, llm_responses = evaluate_code_issue_through_llm(
- issue, inputs, project_context, config, is_code_based_issue_generator
+ issue,
+ inputs,
+ project_context,
+ config,
+ is_code_based_issue_generator,
+ formatted_guide,
)
filter_llm_responses.extend(llm_responses)
issue.set_passes_filtration(passes_filtration)
diff --git a/vet/issue_identifiers/registry.py b/vet/issue_identifiers/registry.py
@@ -206,6 +206,7 @@ def run(
project_context,
config,
identifier.enabled_issue_codes,
+ guides_by_issue_code,
)
except IdentifierInputsMissingError as e:
logger.warning(
@@ -224,6 +225,7 @@ def run(
project_context,
config,
is_code_based_issue_generator=identifier.identifies_code_issues,
+ guides_by_issue_code=guides_by_issue_code,
)
else:
filtered_results_generator = collated_issues_generator
@@ -244,6 +246,7 @@ def run(
multiplexed_generators_with_combined_debug_info,
config,
tuple(detectable_issue_codes),
+ guides_by_issue_code,
)
else:
deduplicated_generator = multiplexed_generators_with_combined_debug_info
