blog

Personal blog
git clone git://git.laack.co/blog.git
Log | Files | Refs

commit 8fedbaa31b0684500cd6fa5b3d18df1497039620
parent 7f411a847b123ae1a847c529c1d2269f1b616a54
Author: Andrew Laack <andrew@laack.co>
Date:   Thu, 24 Sep 2026 20:10:47 -0500

Added some crawling stuff

Diffstat:
Apython/geminispace/count.py | 37+++++++++++++++++++++++++++++++++++++
Apython/geminispace/search.py | 104+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 141 insertions(+), 0 deletions(-)

diff --git a/python/geminispace/count.py b/python/geminispace/count.py @@ -0,0 +1,37 @@ +import plotly.express as px +import os +import re +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns +from sklearn.feature_extraction.text import CountVectorizer +from sklearn.manifold import TSNE +from sklearn.cluster import KMeans + +documents = os.listdir('data') + +documents.remove('idx.db') +documents.remove('idx.db-journal') +documents.remove('failures.csv') + +doc_text = [] + +for doc in documents: + with open('data/' + doc, 'r') as f: + doc_text.append(re.sub(r'[^a-zA-Z ]', '', str(f.readlines()))) + +vectorizer = CountVectorizer(stop_words='english', max_features=1000) +X = vectorizer.fit_transform(doc_text) +print(f"{X.shape[0]} documents, {X.shape[1]} features") + +perplexity = min(30, X.shape[0] - 1) +tsne = TSNE(n_components=2, random_state=0, perplexity=perplexity, init='random') +tsne_data = tsne.fit_transform(X.toarray()) + +df = pd.DataFrame(tsne_data, columns=['Dim_1', 'Dim_2']) +df['doc'] = documents + +fig = px.scatter(df, x='Dim_1', y='Dim_2', + hover_data=['doc']) +fig.show() diff --git a/python/geminispace/search.py b/python/geminispace/search.py @@ -0,0 +1,104 @@ +import os +import subprocess +import sqlite3 +import uuid +import requests +import urllib.parse + +urllib.parse.uses_relative.append('gemini') +urllib.parse.uses_netloc.append('gemini') + +def download(data_dir, site): + cuuid = uuid.uuid4() + subprocess.run(['gemget', site, '-o', os.path.join(data_dir, str(cuuid)), "--max-size", "1.0 MB"]) + return cuuid + +def grab_links(prior, gem_file_path): + link_lines = [] + try: + with open(gem_file_path, 'r') as f: + lines = f.readlines() + for line in lines: + if(len(line) > 2 and line[0:2] == "=>"): + cs = line[2:] + while(len(cs) > 0 and (cs[0] == ' ' or cs[0] == '\t')): + cs = cs[1:] + + acc = "" + while(len(cs) > 0 and cs[0] != ' ' and cs[0] != '\t' and cs[0] != '\n'): + acc += cs[0] + cs = cs[1:] + + cl = requests.compat.urljoin(prior, acc) + link_lines.append(cl) + except: + with open("data/failures.csv", 'a') as f: + f.write(prior + "\n") + + return link_lines + + +def init_db(filename): + conn = sqlite3.connect(filename) + + curs = conn.cursor() + curs.execute(''' + CREATE TABLE IF NOT EXISTS site ( + id TEXT PRIMARY KEY, + url TEXT NOT NULL + ) + ''') + + curs.execute(''' + CREATE TABLE IF NOT EXISTS link ( + source TEXT NOT NULL, + destination TEXT NOT NULL, + visited INTEGER NOT NULL, + UNIQUE (source, destination) + ) + ''') + + curs.close() + conn.commit() + return conn + +def index(conn, data_dir, site_names): + curs = conn.cursor() + for site in site_names: + cuuid = download(data_dir, site) + print(site) + links = grab_links(site, 'data/' + str(cuuid)) + for link in links: + if("gemini://" in link): + curs.execute(''' + INSERT OR IGNORE INTO link (source, destination, visited) VALUES (?, ?, 0) + ''', (site,link)) + + curs.execute(''' + INSERT INTO site (id, url) VALUES (?, ?) + ''', (str(cuuid), site)) + curs.close() + conn.commit() + +data_dir = 'data/' +conn = init_db(data_dir + 'idx.db') + +# seed +index(conn, data_dir, ['gemini://tlgs.one']) + +while True: + curs = conn.cursor() + curs.execute(''' + SELECT DISTINCT destination FROM link + WHERE visited = 0 + AND destination NOT IN (SELECT url FROM site) + LIMIT 10 + ''') + urls = [row[0] for row in curs.fetchall()] + if not urls: + break + index(conn, data_dir, urls) + curs.executemany('UPDATE link SET visited = 1 WHERE destination = ?', + [(u,) for u in urls]) + conn.commit() + curs.close()