commit f81c6112f8cce04dc3bc32957953331b1863bb83
parent d710635bd4c30359d1b4d059ab09ced1bd461688
Author: Andrew Laack <andrew@laack.co>
Date: Sun, 27 Sep 2026 11:00:39 -0500
Respect robots.txt
Diffstat:
1 file changed, 33 insertions(+), 3 deletions(-)
diff --git a/python/geminispace/search.py b/python/geminispace/search.py
@@ -8,7 +8,32 @@ import urllib.parse
urllib.parse.uses_relative.append('gemini')
urllib.parse.uses_netloc.append('gemini')
+def can_fetch(robot_dir, domain, target):
+ pth = robot_dir + domain
+ if not os.path.exists(pth):
+ subprocess.run(['timeout', '10', 'gemget', domain + "/robots.txt", '-o', pth, "--max-size", "1.0 MB"])
+
+ with open(pth, 'r') as f:
+ lines = f.readlines()
+
+ for line in lines:
+ if line.startswith("Disallow:"):
+ if line[10: len(line) - 1] in target:
+ print("Can't fetch: " + target)
+ return False
+ return True
+
+
+
+
+
def download(data_dir, site):
+
+ domain = urllib.parse.urlparse(site).netloc
+ if not can_fetch("robots/", domain, site):
+ return
+
+
cuuid = uuid.uuid4()
subprocess.run(['timeout', '10', 'gemget', site, '-o', os.path.join(data_dir, str(cuuid)), "--max-size", "1.0 MB"])
return cuuid
@@ -64,6 +89,7 @@ def init_db(filename):
def index(conn, data_dir, site_names):
curs = conn.cursor()
+
for site in site_names:
cuuid = download(data_dir, site)
print(site)
@@ -74,9 +100,13 @@ def index(conn, data_dir, site_names):
INSERT OR IGNORE INTO link (source, destination, visited) VALUES (?, ?, 0)
''', (site,link))
- curs.execute('''
- INSERT INTO site (id, url) VALUES (?, ?)
- ''', (str(cuuid), site))
+ try:
+ curs.execute('''
+ INSERT INTO site (id, url) VALUES (?, ?)
+ ''', (str(cuuid), site))
+ except:
+ pass
+
curs.close()
conn.commit()