blog

Personal blog
git clone git://git.laack.co/blog.git
Log | Files | Refs

commit 34adb7a20c4a98f2cdbd30542153c5a0272f2295
parent 8431b8620b0901ccf6cd1f5dc96aee43d96b984f
Author: Andrew Laack <andrew@laack.co>
Date:   Sun, 27 Sep 2026 11:31:22 -0500

Improved parsing

Diffstat:
Mpython/geminispace/search.py | 61+++++++++++++++++++++++++++++++++++++++++++++----------------
1 file changed, 45 insertions(+), 16 deletions(-)

diff --git a/python/geminispace/search.py b/python/geminispace/search.py @@ -8,28 +8,48 @@ import urllib.parse urllib.parse.uses_relative.append('gemini') urllib.parse.uses_netloc.append('gemini') +def parse_robots(text, agents=("*", "researcher")): + rules = [] + group_agents = [] + in_rules = False + for raw in text.splitlines(): + line = raw.split("#", 1)[0].strip() + if not line or ":" not in line: + continue + key, val = line.split(":", 1) + key, val = key.strip().lower(), val.strip() + if key == "user-agent": + if in_rules: + group_agents = [] + in_rules = False + group_agents.append(val.lower()) + elif key == "disallow": + in_rules = True + if val and any(a in agents for a in group_agents): + rules.append(val) + return rules + def can_fetch(robot_dir, domain, target): - pth = robot_dir + domain + os.makedirs(robot_dir, exist_ok=True) + pth = os.path.join(robot_dir, domain) if not os.path.exists(pth): - subprocess.run(['timeout', '10', 'gemget', domain + "/robots.txt", '-o', pth, "--max-size", "1.0 MB"]) - + subprocess.run(['timeout', '10', 'gemget', f"gemini://{domain}/robots.txt", + '-o', pth, "--max-size", "1.0 MB"]) try: - with open(pth, 'r') as f: - lines = f.readlines() - - for line in lines: - if line.startswith("Disallow:"): - if line[10: len(line) - 1] in target: - print("Can't fetch: " + target) - return False - except: - pass # can happen when no robots.txt file exists + with open(pth, 'r', errors='replace') as f: + rules = parse_robots(f.read()) + except FileNotFoundError: + return True + + path = urllib.parse.urlparse(target).path or "/" + for rule in rules: + if path.startswith(rule): + print("Can't fetch: " + target) + return False return True - - def download(data_dir, site): domain = urllib.parse.urlparse(site).netloc @@ -67,7 +87,7 @@ def grab_links(prior, gem_file_path): def init_db(filename): - conn = sqlite3.connect(filename) + conn = sqlite3.connect(filename, timeout=30) curs = conn.cursor() curs.execute(''' @@ -86,6 +106,10 @@ def init_db(filename): ) ''') + curs.execute('CREATE INDEX IF NOT EXISTS link_dest ON link(destination)') + curs.execute('CREATE INDEX IF NOT EXISTS site_url ON site(url)') + curs.execute('CREATE INDEX IF NOT EXISTS link_unvisited ON link(destination)') + curs.close() conn.commit() return conn @@ -102,11 +126,13 @@ def index(conn, data_dir, site_names): curs.execute(''' INSERT OR IGNORE INTO link (source, destination, visited) VALUES (?, ?, 0) ''', (site,link)) + conn.commit() try: curs.execute(''' INSERT INTO site (id, url) VALUES (?, ?) ''', (str(cuuid), site)) + conn.commit() except: pass @@ -125,11 +151,14 @@ while True: SELECT DISTINCT destination FROM link WHERE visited = 0 AND destination NOT IN (SELECT url FROM site) + ORDER BY RANDOM() LIMIT 10 ''') urls = [row[0] for row in curs.fetchall()] if not urls: break + conn.commit() + index(conn, data_dir, urls) curs.executemany('UPDATE link SET visited = 1 WHERE destination = ?', [(u,) for u in urls])