ratchets

Mirror of ratchets, my python testing utility
git clone git://git.laack.co/ratchets.git
Log | Files | Refs | README | LICENSE

test_toml_configs.py (4651B)

      1 from ratchets import run_tests
      2 from ratchets import abstracted_tests
      3 import os
      4 import toml
      5 from typing import Dict, Any
      6 import json
      7 
      8 # verify code still runs as expected even
      9 # if only shell or regex sections are defined
     10 
     11 # also, ensure sufficiently informative message is shown when
     12 # invalid toml is used.
     13 
     14 # ensure environment is configured correctly for other tests to run.
     15 # this creates a ratchet_excluded.txt file,
     16 # creates the output json, and verifies there is a default tests.toml file
     17 
     18 
     19 def test_config():
     20     test_path = os.path.join(
     21         run_tests.find_project_root(), "tests/toml_files/default.toml"
     22     )
     23 
     24     assert os.path.isfile(test_path), "default.toml not found"
     25 
     26     try:
     27         issues = run_tests.evaluate_tests(test_path, True, True, None)
     28         run_tests.update_ratchets(
     29             test_path,
     30             True,
     31             True,
     32             None,
     33             run_tests.find_project_root() + "/tests/test_files/temp_ratchet1.json",
     34         )
     35     except Exception as e:
     36         assert False, f"Unable to update ratchets using 'tests.toml': {e}"
     37 
     38 
     39 def test_formatting():
     40     current_file_directory = os.path.dirname(os.path.abspath(__file__))
     41     toml_file_directory = os.path.abspath(
     42         os.path.join(current_file_directory, "..", "toml_files")
     43     )
     44 
     45     for filename in os.listdir(toml_file_directory):
     46         if filename == "invalid.toml":
     47             try:
     48                 full_path = os.path.abspath(os.path.join(toml_file_directory, filename))
     49                 run_tests.evaluate_tests(full_path, True, True, None)
     50             except Exception as e:
     51                 assert isinstance(
     52                     e, toml.TomlDecodeError
     53                 ), f"Expected TomlDecodeError, got {type(e)}: {e}"
     54             else:
     55                 assert False, "Expected error to be thrown for invalid toml file."
     56 
     57         else:
     58             full_path = os.path.abspath(os.path.join(toml_file_directory, filename))
     59 
     60             # there is a directory in there
     61             if os.path.isfile(full_path):
     62                 run_tests.evaluate_tests(full_path, True, True, None)
     63 
     64         full_path = os.path.abspath(os.path.join(toml_file_directory, filename))
     65 
     66 
     67 # ensure updated values match subsequent runs.
     68 def verify_updating():
     69     test_path = run_tests.find_project_root() + "/tests/toml_files/default.toml"
     70 
     71     run_tests.update_ratchets(
     72         test_path,
     73         True,
     74         True,
     75         None,
     76         run_tests.find_project_root() + "/tests/test_files/temp_ratchet2.json",
     77     )
     78 
     79     # if one is false then the results are guaranteed
     80     # to be either the same or lower.
     81 
     82     issues = run_tests.evaluate_tests(test_path, True, True, None)
     83     current_json: Dict[str, Any] = json.loads(run_tests.results_to_json(issues))
     84     previous_json: Dict[str, Any] = run_tests.load_ratchet_results()
     85 
     86     assert (
     87         current_json == previous_json
     88     ), "JSON should be identical when running evals and updating ratchets."
     89 
     90 
     91 # test how things behave when ratchet_values.json does not exist
     92 def test_ratchet_excluded_missing():
     93 
     94     ratchet_path = abstracted_tests.get_ratchet_path()
     95 
     96     if os.path.isfile(ratchet_path):
     97         try:
     98             os.remove(ratchet_path)
     99         except Exception as e:
    100             assert False, "Unable to delete ratchet_values.json"
    101 
    102     test_path = run_tests.find_project_root() + "/tests/toml_files/default.toml"
    103 
    104     try:
    105         previous = run_tests.load_ratchet_results()
    106     except Exception:
    107         assert (
    108             False
    109         ), "If ratchet_values.json does not exist, we don't throw, assume all 0's"
    110 
    111     issues = run_tests.evaluate_tests(test_path, True, True, None)
    112 
    113     # writes back json file
    114     run_tests.update_ratchets(
    115         test_path,
    116         True,
    117         True,
    118         None,
    119         run_tests.find_project_root() + "/tests/test_files/temp_ratchet3.json",
    120     )
    121 
    122     return
    123 
    124 
    125 # test when there are additional values,
    126 # less values, no values (in current).
    127 
    128 
    129 def test_ratchet_values_differ():
    130 
    131     # ensure clean start
    132     test_config()
    133 
    134     current_file_directory = os.path.dirname(os.path.abspath(__file__))
    135     toml_file_directory = os.path.abspath(
    136         os.path.join(current_file_directory, "..", "toml_files/different")
    137     )
    138 
    139     for filename in os.listdir(toml_file_directory):
    140         full_path = os.path.abspath(os.path.join(toml_file_directory, filename))
    141         run_tests.evaluate_tests(full_path, True, True, None)
    142         full_path = os.path.abspath(os.path.join(toml_file_directory, filename))
    143 
    144     return
    145 
    146 
    147 if __name__ == "__main__":
    148     test_config()
    149     test_formatting()
    150     test_ratchet_excluded_missing()
    151     test_ratchet_values_differ()