blog

Personal blog
git clone git://git.laack.co/blog.git
Log | Files | Refs

commit 8431b8620b0901ccf6cd1f5dc96aee43d96b984f
parent f81c6112f8cce04dc3bc32957953331b1863bb83
Author: Andrew Laack <andrew@laack.co>
Date:   Sun, 27 Sep 2026 11:01:38 -0500

Robustness

Diffstat:
Mpython/geminispace/search.py | 19+++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/python/geminispace/search.py b/python/geminispace/search.py @@ -13,14 +13,17 @@ def can_fetch(robot_dir, domain, target): if not os.path.exists(pth): subprocess.run(['timeout', '10', 'gemget', domain + "/robots.txt", '-o', pth, "--max-size", "1.0 MB"]) - with open(pth, 'r') as f: - lines = f.readlines() - - for line in lines: - if line.startswith("Disallow:"): - if line[10: len(line) - 1] in target: - print("Can't fetch: " + target) - return False + try: + with open(pth, 'r') as f: + lines = f.readlines() + + for line in lines: + if line.startswith("Disallow:"): + if line[10: len(line) - 1] in target: + print("Can't fetch: " + target) + return False + except: + pass # can happen when no robots.txt file exists return True