blog

Personal blog
git clone git://git.laack.co/blog.git
Log | Files | Refs

count.py (1208B)


      1 import plotly.express as px
      2 import os
      3 import re
      4 import numpy as np
      5 import pandas as pd
      6 import matplotlib.pyplot as plt
      7 import seaborn as sns
      8 from sklearn.feature_extraction.text import CountVectorizer
      9 from sklearn.manifold import TSNE
     10 from sklearn.cluster import KMeans
     11 
     12 documents = os.listdir('data')
     13 
     14 documents.remove('idx.db')
     15 documents.remove('idx.db-journal')
     16 documents.remove('failures.csv')
     17 
     18 doc_text = []
     19 
     20 for doc in documents:
     21     with open('data/'+doc, 'rb') as f:
     22         data = f.read()
     23     try:
     24         text = data.decode('utf-8')
     25     except UnicodeDecodeError:
     26         continue
     27 
     28     txt = re.sub(r'[^a-zA-Z]+', ' ', text).strip()
     29     if(txt == ""):
     30         continue
     31 
     32     doc_text.append(txt)
     33 
     34 vectorizer = CountVectorizer(stop_words='english', max_features=10000)
     35 X = vectorizer.fit_transform(doc_text)
     36 print(f"{X.shape[0]} documents, {X.shape[1]} features")
     37 
     38 perplexity = min(30, X.shape[0] - 1)
     39 tsne = TSNE(n_components=2, random_state=0, perplexity=perplexity, init='random')
     40 tsne_data = tsne.fit_transform(X.toarray())
     41 
     42 df = pd.DataFrame(tsne_data, columns=['Dim_1', 'Dim_2'])
     43 df['doc'] = doc_text
     44 
     45 fig = px.scatter(df, x='Dim_1', y='Dim_2',
     46                  hover_data=['doc'])
     47 fig.show()