commit 2614a6b9f1fe3b9d25091fe500df7ef0b0ba7812
parent 0ee96767e2914b09fa776969cdafbefb734508aa
Author: Andrew Laack <andrew@laack.co>
Date: Wed, 18 Jun 2025 16:16:04 -0500
Added readme, wrote tests, documented install
Diffstat:
6 files changed, 146 insertions(+), 12 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -192,7 +192,5 @@ cython_debug/
# refer to https://docs.cursor.com/context/ignore-files
.cursorignore
.cursorindexingignore
-ratchet_values.json
-ratchet_excluded.txt
notes/
data/
diff --git a/README.md b/README.md
@@ -1,2 +1,124 @@
# ratchets
-Ratcheted testing with Python.
+
+Tests that lazily enforce a requirement across the entire repo.
+
+# What is it?
+
+ratchets is a lazy way to enforce code compliance on an ongoing basis. This is done by defining regular expressions or commands to run against all non-excluded python files in a given repository. Tests pass when the number of non-compliant instances of code decreases and fail when they increase. This ensures that subsequent code does not have bad patterns, while still allowing old code to coexist until it is phased out.
+
+# Installation
+
+```bash
+pip install ratchets
+```
+
+# Usage
+
+You first need to create a tests.toml file at the root of your repository. See [tests.toml](tests.toml) for an example of how this should look. There are two primary rule types that can be defined in the tests.toml file.
+
+## python-tests
+
+These are tests that check regular expressions on the basis of each line of each file being examined.
+
+**Example:**
+```toml
+
+[python-tests.exceptions]
+regex = "except:"
+valid = [
+ """try:
+ x = 1
+except ValueError:
+ pass""",
+ """try:
+ do_something()
+except (IOError, ValueError):
+ handle()"""
+]
+invalid = [
+ """
+try:
+ pass
+except:
+ pass""",
+ """try:
+ dangerous()
+except:
+ recover()"""
+]
+
+```
+
+The valid and invalid entries are not necessary, but we provide a CLI utility, ran with ```python3 -m ratchets.validate``` to verify the regular expressions don't exist in the valid string and do exist in the invalid string. If you are testing the tests.toml file in the current git repository or ```python3 -m ratchets.validate -f FILENAME``` if you need to test a specific toml file.
+
+
+## custom-tests
+
+These are tests that run against each file where each evaluation is of the form:
+
+```bash
+FILEPATH | COMMAND
+
+```
+It is assumed the standard output of the command describes each of the issues where each line is counted as an infraction.
+
+**Example:**
+
+```toml
+
+[custom-tests.line_too_long]
+command = "xargs -n1 awk 'length($0) > 80'"
+
+```
+
+This is an example of an `awk` command being used to print each line that has more than 80 characters. As these are printed, they are counted as infractions.
+
+## Updating Ratchets
+
+Once your rules are defined, you need to count the infractions. This is done by running ```bash python3 -m ratchets -u```. This creates a ratchet_values.json file in the root of your project. This will be checked into git and how the previous number of infractions is tracked to ensure it the never increase.
+
+## Excluding Files
+
+Once you run the update command, you should see a file in the root of your repository titled `ratchet_excluded.txt`. By default, this file is empty, but you can use standard .gitignore syntax to specify files that shouldn't be included in your tests. Additionally, all files specified by the gitignore of your project or that don't have the .py extension will not be included in the evaluation.
+
+## Setting Up Tests
+
+To set up tests, we provide an example file at [examples/example_test_ratchet.py](examples/example_test_ratchet.py), which defines tests to be ran with PyTest. In this file there are two uncommented methods that runs one test per rule in both sections (Python and command).
+
+The commented methods aggregate these tests together into two total tests (Python and command).
+
+When creating your testing file, ensure it is being indexed by pytest. If you are unsure what this means, create a file named `test_ratchet.py` in the root of your project.
+
+## Running Tests
+
+Running tests is as simple as running ```pytest``` from the root of the repository or specifying the testing file with ```pytest test_ratchet.py```.
+
+## Finding Issues
+
+At this point, your project has been set up, but as these tests are ran, and further infringements are found, there is a need to identify them. This, along with many other pieces of functionality can be viewed by running:
+
+```
+python -m ratchets --help
+```
+
+Where you will see the following help message describing CLI usage for ratchets:
+
+```
+usage: __main__.py [-h] [-f FILE] [-c] [-r] [-v] [-b] [-m MAX_COUNT] [--compare-counts] [-u] [--validate]
+
+Python ratchet testing
+
+options:
+ -h, --help show this help message and exit
+ -f FILE, --file FILE specify .toml file with tests
+ -c, --command-only run only custom command-based tests
+ -r, --regex-only run only regex-based tests
+ -v, --verbose run verbose tests, printing each infringing line
+ -b, --blame run an additional git-blame for each infraction, ordering results by timestamp
+ -m MAX_COUNT, --max-count MAX_COUNT
+ maximum infractions to display per test (only applies with --blame; default is 10)
+ --compare-counts show only the differences in infraction counts between the current and last saved tests
+ -u, --update-ratchets
+```
+
+Of these, the -b option is particularly useful. When PyTests fail due to infringement counts increasing, it is necessary to identify where the new infringement occurred. By using the -b option you will, by default, see the 10 most recent changes that caused infringements for each rule.
diff --git a/ratchet_excluded.txt b/ratchet_excluded.txt
@@ -0,0 +1,2 @@
+*1*
+ex_dir/
diff --git a/ratchet_values.json b/ratchet_values.json
@@ -0,0 +1 @@
+{}+
\ No newline at end of file
diff --git a/src/ratchets/run_tests.py b/src/ratchets/run_tests.py
@@ -93,9 +93,9 @@ def evaluate_tests(path: str, cmd_only: bool, regex_only: bool) -> Tuple[Dict[st
test_issues: Dict[str, List[Dict[str, Any]]] = {}
custom_issues: Dict[str, List[Dict[str, Any]]] = {}
if python_tests and not cmd_only:
- test_issues = evaluate_python_tests(files, python_tests) # type: ignore
+ test_issues = evaluate_python_tests(files, python_tests)
if custom_tests and not regex_only:
- custom_issues = evaluate_command_tests(files, custom_tests) # type: ignore
+ custom_issues = evaluate_command_tests(files, custom_tests)
return test_issues, custom_issues
@@ -117,7 +117,12 @@ def print_issues(issues: Dict[str, List[Dict[str, Any]]]) -> None:
def load_ratchet_results() -> Dict[str, Any]:
+
path = get_ratchet_path()
+
+ if not os.path.isfile(path):
+ return {}
+
with open(path, 'r') as file:
data = json.load(file)
return data
@@ -392,12 +397,6 @@ def cli():
help="update ratchets_values.json"
)
- parser.add_argument(
- "--validate",
- action="store_true",
- help="validate toml regex patterns"
- )
-
args = parser.parse_args()
file: Optional[str] = args.file
cmd_mode: bool = args.command_only
@@ -406,9 +405,14 @@ def cli():
compare_counts: bool = args.compare_counts
blame: bool = args.blame
verbose: bool = args.verbose
- validate: bool = args.validate
max_count: Optional[int] = args.max_count
+ excludes_path = get_excludes_path()
+
+ if not os.path.isfile(excludes_path):
+ with open(excludes_path, 'a'):
+ pass
+
if not max_count:
max_count = 10
test_path = get_file_path(file)
diff --git a/tests/test_files/test_toml_configs.py b/tests/test_files/test_toml_configs.py
@@ -77,6 +77,12 @@ def test_ratchet_excluded_missing():
assert False, "Unable to delete ratchet_values.json"
test_path = run_tests.get_file_path(None)
+
+ try:
+ previous = run_tests.load_ratchet_results()
+ except Exception as e:
+ assert False, "If ratchet_values.json does not exist, we don't throw, assume all 0's"
+
issues = run_tests.evaluate_tests(test_path, True, True)
# writes back json file
