blog

Personal blog
git clone git://git.laack.co/blog.git
Log | Files | Refs

commit f81c6112f8cce04dc3bc32957953331b1863bb83
parent d710635bd4c30359d1b4d059ab09ced1bd461688
Author: Andrew Laack <andrew@laack.co>
Date:   Sun, 27 Sep 2026 11:00:39 -0500

Respect robots.txt

Diffstat:
Mpython/geminispace/search.py | 36+++++++++++++++++++++++++++++++++---
1 file changed, 33 insertions(+), 3 deletions(-)

diff --git a/python/geminispace/search.py b/python/geminispace/search.py @@ -8,7 +8,32 @@ import urllib.parse urllib.parse.uses_relative.append('gemini') urllib.parse.uses_netloc.append('gemini') +def can_fetch(robot_dir, domain, target): + pth = robot_dir + domain + if not os.path.exists(pth): + subprocess.run(['timeout', '10', 'gemget', domain + "/robots.txt", '-o', pth, "--max-size", "1.0 MB"]) + + with open(pth, 'r') as f: + lines = f.readlines() + + for line in lines: + if line.startswith("Disallow:"): + if line[10: len(line) - 1] in target: + print("Can't fetch: " + target) + return False + return True + + + + + def download(data_dir, site): + + domain = urllib.parse.urlparse(site).netloc + if not can_fetch("robots/", domain, site): + return + + cuuid = uuid.uuid4() subprocess.run(['timeout', '10', 'gemget', site, '-o', os.path.join(data_dir, str(cuuid)), "--max-size", "1.0 MB"]) return cuuid @@ -64,6 +89,7 @@ def init_db(filename): def index(conn, data_dir, site_names): curs = conn.cursor() + for site in site_names: cuuid = download(data_dir, site) print(site) @@ -74,9 +100,13 @@ def index(conn, data_dir, site_names): INSERT OR IGNORE INTO link (source, destination, visited) VALUES (?, ?, 0) ''', (site,link)) - curs.execute(''' - INSERT INTO site (id, url) VALUES (?, ?) - ''', (str(cuuid), site)) + try: + curs.execute(''' + INSERT INTO site (id, url) VALUES (?, ?) + ''', (str(cuuid), site)) + except: + pass + curs.close() conn.commit()