commit 8fedbaa31b0684500cd6fa5b3d18df1497039620
parent 7f411a847b123ae1a847c529c1d2269f1b616a54
Author: Andrew Laack <andrew@laack.co>
Date: Thu, 24 Sep 2026 20:10:47 -0500
Added some crawling stuff
Diffstat:
2 files changed, 141 insertions(+), 0 deletions(-)
diff --git a/python/geminispace/count.py b/python/geminispace/count.py
@@ -0,0 +1,37 @@
+import plotly.express as px
+import os
+import re
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+from sklearn.feature_extraction.text import CountVectorizer
+from sklearn.manifold import TSNE
+from sklearn.cluster import KMeans
+
+documents = os.listdir('data')
+
+documents.remove('idx.db')
+documents.remove('idx.db-journal')
+documents.remove('failures.csv')
+
+doc_text = []
+
+for doc in documents:
+ with open('data/' + doc, 'r') as f:
+ doc_text.append(re.sub(r'[^a-zA-Z ]', '', str(f.readlines())))
+
+vectorizer = CountVectorizer(stop_words='english', max_features=1000)
+X = vectorizer.fit_transform(doc_text)
+print(f"{X.shape[0]} documents, {X.shape[1]} features")
+
+perplexity = min(30, X.shape[0] - 1)
+tsne = TSNE(n_components=2, random_state=0, perplexity=perplexity, init='random')
+tsne_data = tsne.fit_transform(X.toarray())
+
+df = pd.DataFrame(tsne_data, columns=['Dim_1', 'Dim_2'])
+df['doc'] = documents
+
+fig = px.scatter(df, x='Dim_1', y='Dim_2',
+ hover_data=['doc'])
+fig.show()
diff --git a/python/geminispace/search.py b/python/geminispace/search.py
@@ -0,0 +1,104 @@
+import os
+import subprocess
+import sqlite3
+import uuid
+import requests
+import urllib.parse
+
+urllib.parse.uses_relative.append('gemini')
+urllib.parse.uses_netloc.append('gemini')
+
+def download(data_dir, site):
+ cuuid = uuid.uuid4()
+ subprocess.run(['gemget', site, '-o', os.path.join(data_dir, str(cuuid)), "--max-size", "1.0 MB"])
+ return cuuid
+
+def grab_links(prior, gem_file_path):
+ link_lines = []
+ try:
+ with open(gem_file_path, 'r') as f:
+ lines = f.readlines()
+ for line in lines:
+ if(len(line) > 2 and line[0:2] == "=>"):
+ cs = line[2:]
+ while(len(cs) > 0 and (cs[0] == ' ' or cs[0] == '\t')):
+ cs = cs[1:]
+
+ acc = ""
+ while(len(cs) > 0 and cs[0] != ' ' and cs[0] != '\t' and cs[0] != '\n'):
+ acc += cs[0]
+ cs = cs[1:]
+
+ cl = requests.compat.urljoin(prior, acc)
+ link_lines.append(cl)
+ except:
+ with open("data/failures.csv", 'a') as f:
+ f.write(prior + "\n")
+
+ return link_lines
+
+
+def init_db(filename):
+ conn = sqlite3.connect(filename)
+
+ curs = conn.cursor()
+ curs.execute('''
+ CREATE TABLE IF NOT EXISTS site (
+ id TEXT PRIMARY KEY,
+ url TEXT NOT NULL
+ )
+ ''')
+
+ curs.execute('''
+ CREATE TABLE IF NOT EXISTS link (
+ source TEXT NOT NULL,
+ destination TEXT NOT NULL,
+ visited INTEGER NOT NULL,
+ UNIQUE (source, destination)
+ )
+ ''')
+
+ curs.close()
+ conn.commit()
+ return conn
+
+def index(conn, data_dir, site_names):
+ curs = conn.cursor()
+ for site in site_names:
+ cuuid = download(data_dir, site)
+ print(site)
+ links = grab_links(site, 'data/' + str(cuuid))
+ for link in links:
+ if("gemini://" in link):
+ curs.execute('''
+ INSERT OR IGNORE INTO link (source, destination, visited) VALUES (?, ?, 0)
+ ''', (site,link))
+
+ curs.execute('''
+ INSERT INTO site (id, url) VALUES (?, ?)
+ ''', (str(cuuid), site))
+ curs.close()
+ conn.commit()
+
+data_dir = 'data/'
+conn = init_db(data_dir + 'idx.db')
+
+# seed
+index(conn, data_dir, ['gemini://tlgs.one'])
+
+while True:
+ curs = conn.cursor()
+ curs.execute('''
+ SELECT DISTINCT destination FROM link
+ WHERE visited = 0
+ AND destination NOT IN (SELECT url FROM site)
+ LIMIT 10
+ ''')
+ urls = [row[0] for row in curs.fetchall()]
+ if not urls:
+ break
+ index(conn, data_dir, urls)
+ curs.executemany('UPDATE link SET visited = 1 WHERE destination = ?',
+ [(u,) for u in urls])
+ conn.commit()
+ curs.close()