blog

Personal blog
git clone git://git.laack.co/blog.git
Log | Files | Refs

commit cd6fd56b769ad22f709faa61d137e21ec0118654
parent 8fedbaa31b0684500cd6fa5b3d18df1497039620
Author: Andrew Laack <andrew@laack.co>
Date:   Fri, 25 Sep 2026 14:20:07 -0500

Updated scripts

Diffstat:
M.gitignore | 1+
Mpython/geminispace/count.py | 18++++++++++++++----
Mpython/geminispace/search.py | 2+-
3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore @@ -4,3 +4,4 @@ *.pdf *.jpg python/youtube/urls/urls.txt +data diff --git a/python/geminispace/count.py b/python/geminispace/count.py @@ -18,10 +18,20 @@ documents.remove('failures.csv') doc_text = [] for doc in documents: - with open('data/' + doc, 'r') as f: - doc_text.append(re.sub(r'[^a-zA-Z ]', '', str(f.readlines()))) + with open('data/'+doc, 'rb') as f: + data = f.read() + try: + text = data.decode('utf-8') + except UnicodeDecodeError: + continue -vectorizer = CountVectorizer(stop_words='english', max_features=1000) + txt = re.sub(r'[^a-zA-Z]+', ' ', text).strip() + if(txt == ""): + continue + + doc_text.append(txt) + +vectorizer = CountVectorizer(stop_words='english', max_features=10000) X = vectorizer.fit_transform(doc_text) print(f"{X.shape[0]} documents, {X.shape[1]} features") @@ -30,7 +40,7 @@ tsne = TSNE(n_components=2, random_state=0, perplexity=perplexity, init='random' tsne_data = tsne.fit_transform(X.toarray()) df = pd.DataFrame(tsne_data, columns=['Dim_1', 'Dim_2']) -df['doc'] = documents +df['doc'] = doc_text fig = px.scatter(df, x='Dim_1', y='Dim_2', hover_data=['doc']) diff --git a/python/geminispace/search.py b/python/geminispace/search.py @@ -10,7 +10,7 @@ urllib.parse.uses_netloc.append('gemini') def download(data_dir, site): cuuid = uuid.uuid4() - subprocess.run(['gemget', site, '-o', os.path.join(data_dir, str(cuuid)), "--max-size", "1.0 MB"]) + subprocess.run(['timeout', '10', 'gemget', site, '-o', os.path.join(data_dir, str(cuuid)), "--max-size", "1.0 MB"]) return cuuid def grab_links(prior, gem_file_path):