commit 895b4fc9a75458ddbd1c9a37d2fe0d6434d23afa
parent af6535311f960693a15d9b7360ac7f0140897613
Author: Andrew Laack <andrew@laack.co>
Date: Tue, 17 Jun 2025 19:59:53 -0500
Updated tests to only fail when count increases. Added the ability to blame issues that are found
Diffstat:
| M | src/run_tests.py | | | 243 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------- |
| M | src/test_ratchet.py | | | 92 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------- |
2 files changed, 296 insertions(+), 39 deletions(-)
diff --git a/src/run_tests.py b/src/run_tests.py
@@ -1,5 +1,6 @@
import os
import pathspec
+from datetime import datetime
from pathlib import Path
import toml
import argparse
@@ -7,10 +8,27 @@ import json
import re
import subprocess
+def print_diff(current_json, previous_json):
+ all_keys = set(current_json.keys()) | set(previous_json.keys())
+
+ diff_count = 0
+
+ for key in sorted(all_keys):
+ current_value = current_json.get(key, 0)
+ previous_value = previous_json.get(key, 0)
+
+ if current_value != previous_value:
+ diff_count += 1
+ diff = current_value - previous_value
+ sign = "+" if diff > 0 else "-"
+ print(f" {key}: {previous_value} → {current_value} ({sign}{abs(diff)})")
+
+ if diff_count == 0:
+ print("There are no differences.")
+
def find_project_root(start_path=None, markers=None):
if start_path is None:
start_path = os.getcwd()
- # check if we are at the root of the project.
if markers is None:
markers = ['.git', 'pyproject.toml', 'setup.py', 'tests.toml']
@@ -49,10 +67,14 @@ def get_python_files(directory):
python_files = set([path.absolute() for path in directory.rglob("*.py") if not path.is_symlink()])
return list(python_files)
-def filter_excluded_files(files, excluded_path):
+def filter_excluded_files(files, excluded_path, ignore_path):
with open(excluded_path, 'r') as f:
patterns = f.read().splitlines()
+ if os.path.isfile(ignore_path):
+ with open(ignore_path, 'r') as f:
+ patterns += f.read().splitlines()
+
spec = pathspec.PathSpec.from_lines('gitwildmatch', patterns)
files = [f for f in files if not spec.match_file(f)]
@@ -73,7 +95,9 @@ def evaluate_tests(path, cmd_only, regex_only):
EXCLUDED_PATH = "ratchet_excluded.txt"
excluded_path = os.path.join(root, EXCLUDED_PATH)
- files = filter_excluded_files(files, excluded_path)
+ ignore_path = os.path.join(root, ".gitignore")
+
+ files = filter_excluded_files(files, excluded_path, ignore_path)
test_issues = {}
custom_issues = {}
@@ -103,7 +127,7 @@ def print_issues(issues):
print(f"\n{test_name} — no issues found.")
-def previous_results():
+def load_ratchet_results():
path = get_ratchet_path()
with open(path, 'r') as file:
data = json.load(file)
@@ -176,28 +200,16 @@ def evaluate_command_tests(files, test_str):
return results
def results_to_json(results):
- """
- Convert the tuple returned by evaluate_tests (test_issues, custom_issues)
- into a JSON string of the form:
- {
- "test_name_1": count1,
- "test_name_2": count2,
- ...
- }
- where count is the number of matches/issues for that test.
- """
+
test_issues, custom_issues = results
counts = {}
- # Count regex-based test issues
for name, matches in test_issues.items():
counts[name] = len(matches)
- # Count command-based test issues, summing if a name overlaps
for name, matches in custom_issues.items():
counts[name] = counts.get(name, 0) + len(matches)
- # Return a pretty-printed JSON string (sorted keys for consistency)
return json.dumps(counts, indent=2, sort_keys=True)
def update_ratchets(test_path, cmd_mode, regex_mode):
@@ -206,17 +218,183 @@ def update_ratchets(test_path, cmd_mode, regex_mode):
path = get_ratchet_path()
with open(path, 'w') as file:
file.writelines(results_json)
+
+
+
+from datetime import datetime
+
+def print_issues_with_blames(results, max_count):
+ enriched_test_issues, enriched_custom_issues = add_blames(results)
+
+ def _parse_time(ts):
+ if not ts:
+ return datetime.max
+ try:
+ return datetime.fromisoformat(ts)
+ except Exception:
+ return datetime.max
+
+ def _print_section(section_name, issues_dict):
+ for test_name, matches in issues_dict.items():
+ if matches:
+ sorted_matches = sorted(matches, key=lambda m: _parse_time(m.get("blame_time")))
+ print("\n" + "-"*40)
+ print(f"{section_name} — {test_name} ({len(sorted_matches)} issue{'s' if len(sorted_matches)!=1 else ''}):")
+ print("-"*40)
+ count = 0
+ for match in sorted_matches:
+ count += 1
+ if count > max_count:
+ break
+ file_path = match.get("file", "<unknown>")
+ line_no = match.get("line")
+ content = match.get("content", "").strip()
+ truncated = content if len(content) <= 80 else content[:80] + "..."
+ author = match.get("blame_author") or "Unknown"
+ ts = match.get("blame_time") or "Unknown"
+ if line_no is not None:
+ print(f" → {file_path}:{line_no} by {author} at {ts}")
+ print(f" {truncated}")
+ else:
+ print(f" → {file_path} by {author} at {ts}")
+ print(f" {truncated}")
+ else:
+ # No matches for this test
+ print(f"\n{section_name} — {test_name}: no issues found.")
+
+ _print_section("Regex Test", enriched_test_issues)
+ _print_section("Command Test", enriched_custom_issues)
+
+def add_blames(results):
+
+ test_issues, custom_issues = results
+
+ # Determine repo root to run git commands in
+ try:
+ repo_root = find_project_root()
+ except Exception:
+ repo_root = None # if not in a git repo, blame will fail
+
+ def get_blame_for_line(file_path, line_no):
+ """
+ Returns (author, timestamp_iso) for a given file and line number via git blame.
+ If anything fails, returns (None, None).
+ """
+ if repo_root is None:
+ return None, None
+ # Use porcelain format for easier parsing
+ cmd = ["git", "blame", "-L", f"{line_no},{line_no}", "--porcelain", file_path]
+ try:
+ res = subprocess.run(cmd, capture_output=True, text=True, cwd=repo_root, timeout=5)
+ if res.returncode != 0:
+ return None, None
+ author = None
+ author_time = None
+ for l in res.stdout.splitlines():
+ if l.startswith("author "):
+ author = l[len("author "):].strip()
+ elif l.startswith("author-time "):
+ # author-time is a Unix timestamp (seconds since epoch)
+ try:
+ ts = int(l[len("author-time "):].strip())
+ # convert to ISO 8601; uses local timezone
+ author_time = datetime.fromtimestamp(ts).isoformat()
+ except Exception:
+ author_time = None
+ # once we have both, we can break
+ if author is not None and author_time is not None:
+ break
+ return author, author_time
+ except Exception:
+ return None, None
+
+ def get_last_commit_for_file(file_path):
+ """
+ Returns (author, timestamp_iso) for the last commit touching this file via git log.
+ If fails, returns (None, None).
+ """
+ if repo_root is None:
+ return None, None
+ cmd = ["git", "log", "-1", "--format=%an;%at", "--", file_path]
+ try:
+ res = subprocess.run(cmd, capture_output=True, text=True, cwd=repo_root, timeout=5)
+ if res.returncode != 0 or not res.stdout.strip():
+ return None, None
+ out = res.stdout.strip()
+ # format is "Author Name;timestamp"
+ parts = out.split(";", 1)
+ if len(parts) != 2:
+ return None, None
+ author = parts[0].strip()
+ try:
+ ts = int(parts[1].strip())
+ author_time = datetime.fromtimestamp(ts).isoformat()
+ except Exception:
+ author_time = None
+ return author, author_time
+ except Exception:
+ return None, None
+
+ # Process both test_issues and custom_issues
+ for issues in (test_issues, custom_issues):
+ for test_name, matches in issues.items():
+ for match in matches:
+ file_path = match.get("file")
+ line_no = match.get("line")
+ # Only proceed if file_path exists
+ if not file_path:
+ continue
+ # If it's an absolute path, convert to relative to repo_root if possible
+ # Git commands accept absolute paths if cwd is repo root, so this is OK.
+ if line_no is not None:
+ # try blame for the specific line
+ author, author_time = get_blame_for_line(file_path, line_no)
+ else:
+ # fallback to last commit touching file
+ author, author_time = get_last_commit_for_file(file_path)
+ # Attach blame info if found
+ if author is not None:
+ match["blame_author"] = author
+ else:
+ match["blame_author"] = None
+ if author_time is not None:
+ match["blame_time"] = author_time
+ else:
+ match["blame_time"] = None
+
+ return (test_issues, custom_issues)
+
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Python ratchet testing")
parser.add_argument("-f", "--file", help="Specify .toml file with tests")
parser.add_argument(
+ "-b", "--blame",
+ action="store_true",
+ help="Run only custom command-based tests"
+ )
+
+ parser.add_argument(
+ "--max-count", "-m",
+ type=int,
+ default=None,
+ help="Maximum infractions to display per test (only applies with --blame; default is 10)"
+ )
+
+ parser.add_argument(
"-c", "--command-only",
action="store_true",
help="Run only custom command-based tests"
)
+
+ parser.add_argument(
+ "--compare-counts",
+ action="store_true",
+ help="Compare the counts between the current test and the last saved"
+ )
+
parser.add_argument(
"-r", "--regex-only",
action="store_true",
@@ -235,12 +413,33 @@ if __name__ == "__main__":
cmd_mode = args.command_only
regex_mode = args.regex_only
update = args.update_ratchets
+ compare_counts = args.compare_counts
+ blame = args.blame
+ max_count = args.max_count
+
+ if not max_count:
+ max_count = 10
test_path = get_file_path(file)
- if update:
- update_ratchets(test_path, cmd_mode, regex_mode)
- else:
+ if blame:
issues = evaluate_tests(test_path, cmd_mode, regex_mode)
- for issue_type in issues:
- print_issues(issue_type)
+ with_blames = add_blames(issues)
+ print_issues_with_blames(issues, max_count)
+
+ else:
+ if compare_counts:
+ issues = evaluate_tests(test_path, cmd_mode, regex_mode)
+
+ current_json = json.loads(results_to_json(issues))
+ previous_json = load_ratchet_results()
+
+ print_diff(current_json, previous_json)
+
+ else:
+ if update:
+ update_ratchets(test_path, cmd_mode, regex_mode)
+ else:
+ issues = evaluate_tests(test_path, cmd_mode, regex_mode)
+ for issue_type in issues:
+ print_issues(issue_type)
diff --git a/src/test_ratchet.py b/src/test_ratchet.py
@@ -1,14 +1,17 @@
+import os
+import json
import pytest
import toml
from pathlib import Path
from run_tests import (
evaluate_python_tests,
evaluate_command_tests,
+ filter_excluded_files,
+ find_project_root,
get_python_files,
- find_project_root
+ get_ratchet_path
)
-# Load the TOML config once
ROOT = find_project_root()
TOML_PATH = Path(ROOT) / "tests.toml"
CONFIG = toml.load(TOML_PATH)
@@ -16,28 +19,83 @@ CONFIG = toml.load(TOML_PATH)
PYTHON_TESTS = CONFIG.get("python-tests", {})
COMMAND_TESTS = CONFIG.get("custom-tests", {})
+def load_baseline_counts():
+ """
+ Load previous counts from ratchet_values.json.
+ Return a dict mapping test_name -> count. If file missing or malformed, return empty dict.
+ """
+ try:
+ ratchet_path = get_ratchet_path()
+ if os.path.isfile(ratchet_path):
+ with open(ratchet_path, 'r', encoding='utf-8') as f:
+ data = json.load(f)
+ if isinstance(data, dict):
+ # Ensure values are ints
+ return {k: int(v) for k, v in data.items()}
+ except Exception:
+ pass
+ return {}
+
+BASELINE_COUNTS = load_baseline_counts()
@pytest.mark.parametrize("test_name,rule", PYTHON_TESTS.items())
def test_python_regex_rule(test_name, rule):
- """
- Fails if any regex match is found in the project files.
- """
- files = get_python_files(ROOT)
+ # Prepare file list
+ root = find_project_root()
+ files = get_python_files(root)
+ EXCLUDED_PATH = "ratchet_excluded.txt"
+ excluded_path = os.path.join(root, EXCLUDED_PATH)
+ ignore_path = os.path.join(root, ".gitignore")
+ files = filter_excluded_files(files, excluded_path, ignore_path)
+
+ # Evaluate current results
results = evaluate_python_tests(files, {test_name: rule})
- assert not results[test_name], (
- f"Regex violations found for '{test_name}':\n" +
- "\n".join(f"{r['file']}:{r['line']} — {r['content']}" for r in results[test_name])
- )
+ current_matches = results.get(test_name, [])
+ current_count = len(current_matches)
+ # Baseline
+ baseline_count = BASELINE_COUNTS.get(test_name, 0)
+
+ # If increased, fail.
+ assert current_count <= baseline_count, (
+ f"Regex violations for '{test_name}' increased: "
+ f"baseline={baseline_count}, current={current_count}\n"
+ + (
+ "\n".join(
+ f"{r['file']}:{r['line']} — {r['content']}"
+ for r in current_matches
+ )
+ if current_count > 0 else ""
+ )
+ )
@pytest.mark.parametrize("test_name,test_dict", COMMAND_TESTS.items())
def test_custom_command_rule(test_name, test_dict):
- """
- Fails if custom shell command detects output in any file.
- """
- files = get_python_files(ROOT)
+ # Prepare file list
+ root = find_project_root()
+ files = get_python_files(root)
+ EXCLUDED_PATH = "ratchet_excluded.txt"
+ excluded_path = os.path.join(root, EXCLUDED_PATH)
+ ignore_path = os.path.join(root, ".gitignore")
+ files = filter_excluded_files(files, excluded_path, ignore_path)
+
+ # Evaluate current results
results = evaluate_command_tests(files, {test_name: test_dict})
- assert not results[test_name], (
- f"Command violations found for '{test_name}':\n" +
- "\n".join(f"{r['file']} — {r['content']}" for r in results[test_name])
+ current_matches = results.get(test_name, [])
+ current_count = len(current_matches)
+
+ # Baseline
+ baseline_count = BASELINE_COUNTS.get(test_name, 0)
+
+ # If increased, fail.
+ assert current_count <= baseline_count, (
+ f"Command violations for '{test_name}' increased: "
+ f"baseline={baseline_count}, current={current_count}\n"
+ + (
+ "\n".join(
+ f"{r['file']} — {r['content']}"
+ for r in current_matches
+ )
+ if current_count > 0 else ""
+ )
)
