commit 34adb7a20c4a98f2cdbd30542153c5a0272f2295
parent 8431b8620b0901ccf6cd1f5dc96aee43d96b984f
Author: Andrew Laack <andrew@laack.co>
Date: Sun, 27 Sep 2026 11:31:22 -0500
Improved parsing
Diffstat:
1 file changed, 45 insertions(+), 16 deletions(-)
diff --git a/python/geminispace/search.py b/python/geminispace/search.py
@@ -8,28 +8,48 @@ import urllib.parse
urllib.parse.uses_relative.append('gemini')
urllib.parse.uses_netloc.append('gemini')
+def parse_robots(text, agents=("*", "researcher")):
+ rules = []
+ group_agents = []
+ in_rules = False
+ for raw in text.splitlines():
+ line = raw.split("#", 1)[0].strip()
+ if not line or ":" not in line:
+ continue
+ key, val = line.split(":", 1)
+ key, val = key.strip().lower(), val.strip()
+ if key == "user-agent":
+ if in_rules:
+ group_agents = []
+ in_rules = False
+ group_agents.append(val.lower())
+ elif key == "disallow":
+ in_rules = True
+ if val and any(a in agents for a in group_agents):
+ rules.append(val)
+ return rules
+
def can_fetch(robot_dir, domain, target):
- pth = robot_dir + domain
+ os.makedirs(robot_dir, exist_ok=True)
+ pth = os.path.join(robot_dir, domain)
if not os.path.exists(pth):
- subprocess.run(['timeout', '10', 'gemget', domain + "/robots.txt", '-o', pth, "--max-size", "1.0 MB"])
-
+ subprocess.run(['timeout', '10', 'gemget', f"gemini://{domain}/robots.txt",
+ '-o', pth, "--max-size", "1.0 MB"])
try:
- with open(pth, 'r') as f:
- lines = f.readlines()
-
- for line in lines:
- if line.startswith("Disallow:"):
- if line[10: len(line) - 1] in target:
- print("Can't fetch: " + target)
- return False
- except:
- pass # can happen when no robots.txt file exists
+ with open(pth, 'r', errors='replace') as f:
+ rules = parse_robots(f.read())
+ except FileNotFoundError:
+ return True
+
+ path = urllib.parse.urlparse(target).path or "/"
+ for rule in rules:
+ if path.startswith(rule):
+ print("Can't fetch: " + target)
+ return False
return True
-
-
def download(data_dir, site):
domain = urllib.parse.urlparse(site).netloc
@@ -67,7 +87,7 @@ def grab_links(prior, gem_file_path):
def init_db(filename):
- conn = sqlite3.connect(filename)
+ conn = sqlite3.connect(filename, timeout=30)
curs = conn.cursor()
curs.execute('''
@@ -86,6 +106,10 @@ def init_db(filename):
)
''')
+ curs.execute('CREATE INDEX IF NOT EXISTS link_dest ON link(destination)')
+ curs.execute('CREATE INDEX IF NOT EXISTS site_url ON site(url)')
+ curs.execute('CREATE INDEX IF NOT EXISTS link_unvisited ON link(destination)')
+
curs.close()
conn.commit()
return conn
@@ -102,11 +126,13 @@ def index(conn, data_dir, site_names):
curs.execute('''
INSERT OR IGNORE INTO link (source, destination, visited) VALUES (?, ?, 0)
''', (site,link))
+ conn.commit()
try:
curs.execute('''
INSERT INTO site (id, url) VALUES (?, ?)
''', (str(cuuid), site))
+ conn.commit()
except:
pass
@@ -125,11 +151,14 @@ while True:
SELECT DISTINCT destination FROM link
WHERE visited = 0
AND destination NOT IN (SELECT url FROM site)
+ ORDER BY RANDOM()
LIMIT 10
''')
urls = [row[0] for row in curs.fetchall()]
if not urls:
break
+ conn.commit()
+
index(conn, data_dir, urls)
curs.executemany('UPDATE link SET visited = 1 WHERE destination = ?',
[(u,) for u in urls])