count.py (1208B)
1 import plotly.express as px 2 import os 3 import re 4 import numpy as np 5 import pandas as pd 6 import matplotlib.pyplot as plt 7 import seaborn as sns 8 from sklearn.feature_extraction.text import CountVectorizer 9 from sklearn.manifold import TSNE 10 from sklearn.cluster import KMeans 11 12 documents = os.listdir('data') 13 14 documents.remove('idx.db') 15 documents.remove('idx.db-journal') 16 documents.remove('failures.csv') 17 18 doc_text = [] 19 20 for doc in documents: 21 with open('data/'+doc, 'rb') as f: 22 data = f.read() 23 try: 24 text = data.decode('utf-8') 25 except UnicodeDecodeError: 26 continue 27 28 txt = re.sub(r'[^a-zA-Z]+', ' ', text).strip() 29 if(txt == ""): 30 continue 31 32 doc_text.append(txt) 33 34 vectorizer = CountVectorizer(stop_words='english', max_features=10000) 35 X = vectorizer.fit_transform(doc_text) 36 print(f"{X.shape[0]} documents, {X.shape[1]} features") 37 38 perplexity = min(30, X.shape[0] - 1) 39 tsne = TSNE(n_components=2, random_state=0, perplexity=perplexity, init='random') 40 tsne_data = tsne.fit_transform(X.toarray()) 41 42 df = pd.DataFrame(tsne_data, columns=['Dim_1', 'Dim_2']) 43 df['doc'] = doc_text 44 45 fig = px.scatter(df, x='Dim_1', y='Dim_2', 46 hover_data=['doc']) 47 fig.show()