commit 72552a7fdef6a453ee9d649fddde01dd3bfb7cc9
parent 60996488d196cc1f2cdf4320ac815864a6bb92cc
Author: Andrew Laack <andrew@laack.co>
Date: Tue, 17 Jun 2025 17:06:35 -0500
Added .toml validation, set up command evaluation, and finished regex checks
Diffstat:
| A | src/run_tests.py | | | 200 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| D | src/test_regexp.py | | | 113 | ------------------------------------------------------------------------------- |
| A | src/validate.py | | | 56 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
3 files changed, 256 insertions(+), 113 deletions(-)
diff --git a/src/run_tests.py b/src/run_tests.py
@@ -0,0 +1,200 @@
+import os
+import pathspec
+from pathlib import Path
+import toml
+import argparse
+import json
+import re
+import subprocess
+
+def find_project_root(start_path=None, markers=None):
+ if start_path is None:
+ start_path = os.getcwd()
+ # check if we are at the root of the project.
+ if markers is None:
+ markers = ['.git', 'pyproject.toml', 'setup.py', 'tests.toml']
+
+ current = os.path.abspath(start_path)
+ while True:
+ for marker in markers:
+ if os.path.exists(os.path.join(current, marker)):
+ return current
+ parent = os.path.dirname(current)
+ if parent == current:
+ raise FileNotFoundError("Project root not found.")
+ current = parent
+
+
+def get_excludes_path():
+ DEFAULT_FILENAME = "ratchet_excluded.txt"
+ root = find_project_root(file)
+ return os.path.join(root, DEFAULT_FILENAME)
+
+
+def get_file_path(file):
+
+ DEFAULT_FILENAME = "tests.toml"
+
+ if not file:
+ file = DEFAULT_FILENAME
+
+ if "/" in file:
+ return file
+ else:
+ root = find_project_root(file)
+ return os.path.join(root, file)
+
+
+def get_python_files(directory):
+ directory = Path(directory)
+ python_files = set([path.absolute() for path in directory.rglob("*.py") if not path.is_symlink()])
+ return list(python_files)
+
+def filter_excluded_files(files, excluded_path):
+ with open(excluded_path, 'r') as f:
+ patterns = f.read().splitlines()
+
+ spec = pathspec.PathSpec.from_lines('gitwildmatch', patterns)
+
+ files = [f for f in files if not spec.match_file(f)]
+
+ return files
+
+def evaluate_tests(path, cmd_only, regex_only):
+
+ assert (os.path.isfile(path))
+
+ config = toml.load(path)
+ python_tests = config.get("python-tests")
+ custom_tests = config.get("custom-tests")
+ root = find_project_root()
+
+ files = get_python_files(root)
+
+ EXCLUDED_PATH = "ratchet_excluded.txt"
+ excluded_path = os.path.join(root, EXCLUDED_PATH)
+
+ files = filter_excluded_files(files, excluded_path)
+
+ if python_tests and not cmd_only:
+ issues = evaluate_python_tests(files, python_tests)
+ for test_name, matches in issues.items():
+ if matches:
+ print(f"\n{test_name} — matched {len(matches)} issue(s):")
+ for match in matches:
+ print(f" → {match['file']}:{match['line']}: {match['content']}")
+ else:
+ print(f"\n{test_name} — no issues found.")
+
+ if custom_tests and not regex_only:
+ issues = evaluate_command_tests(files, custom_tests)
+ if issues:
+ print("\ncustom-tests — matched issue(s):")
+ for file, lines in issues.items():
+ print(f"\n{file}:")
+ for line in lines:
+ truncated = f"{line[0:80]}..." if len(line) > 80 else line
+ print(f" → {truncated}")
+ else:
+ print("\ncustom-tests — no issues found.")
+
+def previous_results():
+ path = get_ratchet_path()
+ with open(path, 'r') as file:
+ data = json.load(file)
+ return data
+
+
+def evaluate_python_tests(files, test_str):
+ assert len(files) != 0
+ assert len(test_str) != 0
+
+ results = {}
+
+ for test_name, rule in test_str.items():
+ pattern = re.compile(rule["regex"])
+ results[test_name] = []
+
+ for file_path in files:
+ with open(file_path, 'r', encoding='utf-8') as f:
+ for lineno, line in enumerate(f, 1):
+ if pattern.search(line):
+ results[test_name].append({
+ "file": str(file_path),
+ "line": lineno,
+ "content": line.strip()
+ })
+
+ return results
+
+def get_ratchet_path():
+ root = find_project_root()
+ RATCHET_NAME = "ratchet_values.json"
+ ratchet_file_path = os.path.join(root, RATCHET_NAME)
+ return ratchet_file_path
+
+
+# add timeout.
+def evaluate_command_tests(files, test_str):
+ assert len(test_str) != 0
+
+ issues = {} # filename -> list of issues
+
+ for test_name, test_dict in test_str.items():
+ command_template = test_dict["command"]
+
+ for file in files:
+ # Build the full command using pipe: echo file | <command>
+ cmd_str = f"echo {file} | {command_template}"
+
+ try:
+ # Run the command and capture output
+ result = subprocess.run(
+ cmd_str,
+ shell=True,
+ text=True,
+ capture_output=True,
+ timeout=5
+ )
+
+ output = result.stdout.strip()
+ if output:
+ lines = output.splitlines()
+ if file not in issues:
+ issues[file] = []
+ issues[file].extend(lines)
+
+ except subprocess.TimeoutExpired:
+ print(f"Timeout while running test '{test_name}' on {file}")
+
+ return issues
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser(description="Python ratchet testing")
+ parser.add_argument("-f", "--file",)
+
+ parser.add_argument(
+ "-c", "--command-only",
+ action="store_true",
+ help="Run only custom command-based tests"
+ )
+
+ parser.add_argument(
+ "-r", "--regex-only",
+ action="store_true",
+ help="Run only regex-based tests"
+ )
+
+
+
+ args = parser.parse_args()
+ file = args.file
+
+ cmd_mode = args.command_only
+ regex_mode = args.regex_only
+
+
+ test_path = get_file_path(file)
+
+ evaluate_tests(test_path, cmd_mode, regex_mode)
diff --git a/src/test_regexp.py b/src/test_regexp.py
@@ -1,113 +0,0 @@
-import os
-import pathspec
-from pathlib import Path
-import toml
-import argparse
-import json
-import re
-
-def find_project_root(start_path=None, markers=None):
- if start_path is None:
- start_path = os.getcwd()
- # check if we are at the root of the project.
- if markers is None:
- markers = ['.git', 'pyproject.toml', 'setup.py', 'tests.toml']
-
- current = os.path.abspath(start_path)
- while True:
- for marker in markers:
- if os.path.exists(os.path.join(current, marker)):
- return current
- parent = os.path.dirname(current)
- if parent == current:
- raise FileNotFoundError("Project root not found.")
- current = parent
-
-
-def get_excludes_path():
- DEFAULT_FILENAME = "ratchet_excluded.txt"
- root = find_project_root(file)
- return os.path.join(root, DEFAULT_FILENAME)
-
-
-def get_file_path(file):
-
- DEFAULT_FILENAME = "tests.toml"
-
- if not file:
- file = DEFAULT_FILENAME
-
- if "/" in file:
- return file
- else:
- root = find_project_root(file)
- return os.path.join(root, file)
-
-
-def get_python_files(directory):
- directory = Path(directory)
- python_files = set([path.absolute() for path in directory.rglob("*.py") if not path.is_symlink()])
- return list(python_files)
-
-def filter_excluded_files(files, excluded_path):
- with open(excluded_path, 'r') as f:
- patterns = f.read().splitlines()
-
- spec = pathspec.PathSpec.from_lines('gitwildmatch', patterns)
-
- files = [f for f in files if not spec.match_file(f)]
-
- return files
-
-def evaluate_tests(path):
-
- assert (os.path.isfile(path))
-
- config = toml.load(path)
- python_tests = config.get("python-tests")
- custom_tests = config.get("custom-tests")
- root = find_project_root()
-
- files = get_python_files(root)
-
- EXCLUDED_PATH = "ratchet_excluded.txt"
- excluded_path = os.path.join(root, EXCLUDED_PATH)
-
- files = filter_excluded_files(files, excluded_path)
-
- if python_tests:
- evaluate_python_tests(python_tests)
-
- if custom_tests:
- evaluate_command_tests(custom_tests)
-
-def previous_results():
- path = get_ratchet_path()
- with open(path, 'r') as file:
- data = json.load(file)
- return data
-
-def evaluate_python_tests(test_str):
- assert (len(test_str) != 0)
-
-def get_ratchet_path():
- root = find_project_root()
- RATCHET_NAME = "ratchet_values.json"
- ratchet_file_path = os.path.join(root, RATCHET_NAME)
- return ratchet_file_path
-
-
-def evaluate_command_tests(test_str):
- assert (len(test_str) != 0)
-
-if __name__ == "__main__":
-
- parser = argparse.ArgumentParser(description="Python ratchet testing")
- parser.add_argument("-f", "--file",)
-
- args = parser.parse_args()
- file = args.file
-
- test_path = get_file_path(file)
-
- evaluate_tests(test_path)
diff --git a/src/validate.py b/src/validate.py
@@ -0,0 +1,56 @@
+import run_tests
+import re
+import toml
+import argparse
+
+# validate rules
+def evaluate_single_regex(regex, custom_str):
+ pattern = re.compile(regex)
+ return pattern.search(custom_str)
+
+def check_valid(python_tests):
+ for test in python_tests:
+ regex = python_tests[test]["regex"]
+ for validation in python_tests[test]["valid"]:
+ for line in validation.splitlines():
+ if evaluate_single_regex(regex, line):
+ raise AssertionError(f"Regex: {regex} matched {line}")
+
+
+def check_invalid(python_tests):
+
+ for test in python_tests:
+ regex = python_tests[test]["regex"]
+ for validation in python_tests[test]["invalid"]:
+
+ found = False
+
+ for line in validation.splitlines():
+ if evaluate_single_regex(regex, line):
+ found = True
+
+ if not found:
+ raise AssertionError(f"Regex: {regex} not matched in {validation}")
+ return 0
+
+if __name__ == "__main__":
+
+ parser = argparse.ArgumentParser(description="Python ratchet testing")
+ parser.add_argument("-f", "--file",)
+
+ args = parser.parse_args()
+ file = args.file
+
+ test_path = run_tests.get_file_path(file)
+
+ config = toml.load(test_path)
+ python_tests = config.get("python-tests")
+
+ if python_tests is None:
+ print("No python tests found, there is nothing to validate.")
+ exit()
+
+ check_valid(python_tests)
+ check_invalid(python_tests)
+
+ print(f"{test_path} is configured correctly!")
