commit 8431b8620b0901ccf6cd1f5dc96aee43d96b984f
parent f81c6112f8cce04dc3bc32957953331b1863bb83
Author: Andrew Laack <andrew@laack.co>
Date: Sun, 27 Sep 2026 11:01:38 -0500
Robustness
Diffstat:
1 file changed, 11 insertions(+), 8 deletions(-)
diff --git a/python/geminispace/search.py b/python/geminispace/search.py
@@ -13,14 +13,17 @@ def can_fetch(robot_dir, domain, target):
if not os.path.exists(pth):
subprocess.run(['timeout', '10', 'gemget', domain + "/robots.txt", '-o', pth, "--max-size", "1.0 MB"])
- with open(pth, 'r') as f:
- lines = f.readlines()
-
- for line in lines:
- if line.startswith("Disallow:"):
- if line[10: len(line) - 1] in target:
- print("Can't fetch: " + target)
- return False
+ try:
+ with open(pth, 'r') as f:
+ lines = f.readlines()
+
+ for line in lines:
+ if line.startswith("Disallow:"):
+ if line[10: len(line) - 1] in target:
+ print("Can't fetch: " + target)
+ return False
+ except:
+ pass # can happen when no robots.txt file exists
return True