commit cd6fd56b769ad22f709faa61d137e21ec0118654
parent 8fedbaa31b0684500cd6fa5b3d18df1497039620
Author: Andrew Laack <andrew@laack.co>
Date: Fri, 25 Sep 2026 14:20:07 -0500
Updated scripts
Diffstat:
3 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -4,3 +4,4 @@
*.pdf
*.jpg
python/youtube/urls/urls.txt
+data
diff --git a/python/geminispace/count.py b/python/geminispace/count.py
@@ -18,10 +18,20 @@ documents.remove('failures.csv')
doc_text = []
for doc in documents:
- with open('data/' + doc, 'r') as f:
- doc_text.append(re.sub(r'[^a-zA-Z ]', '', str(f.readlines())))
+ with open('data/'+doc, 'rb') as f:
+ data = f.read()
+ try:
+ text = data.decode('utf-8')
+ except UnicodeDecodeError:
+ continue
-vectorizer = CountVectorizer(stop_words='english', max_features=1000)
+ txt = re.sub(r'[^a-zA-Z]+', ' ', text).strip()
+ if(txt == ""):
+ continue
+
+ doc_text.append(txt)
+
+vectorizer = CountVectorizer(stop_words='english', max_features=10000)
X = vectorizer.fit_transform(doc_text)
print(f"{X.shape[0]} documents, {X.shape[1]} features")
@@ -30,7 +40,7 @@ tsne = TSNE(n_components=2, random_state=0, perplexity=perplexity, init='random'
tsne_data = tsne.fit_transform(X.toarray())
df = pd.DataFrame(tsne_data, columns=['Dim_1', 'Dim_2'])
-df['doc'] = documents
+df['doc'] = doc_text
fig = px.scatter(df, x='Dim_1', y='Dim_2',
hover_data=['doc'])
diff --git a/python/geminispace/search.py b/python/geminispace/search.py
@@ -10,7 +10,7 @@ urllib.parse.uses_netloc.append('gemini')
def download(data_dir, site):
cuuid = uuid.uuid4()
- subprocess.run(['gemget', site, '-o', os.path.join(data_dir, str(cuuid)), "--max-size", "1.0 MB"])
+ subprocess.run(['timeout', '10', 'gemget', site, '-o', os.path.join(data_dir, str(cuuid)), "--max-size", "1.0 MB"])
return cuuid
def grab_links(prior, gem_file_path):