gemini-search

main.go (6493B)

---

 package main
 
 import (
 	"database/sql"
 	"errors"
 	"fmt"
 	"geminisearch/gemtextparser"
 	_ "github.com/mattn/go-sqlite3"
 	"os"
 )
 
 // TODO: centralize this?
 var CRAWLED_DIR = "outputs/crawled"
 var INDEXED_DIR = "outputs/indexed"
 var CRAWLED_DB_NAME = "main.db"
 var INDEXED_DB_NAME = "main.db"
 
 // TODO: Probably return error instead of panic...
 func connectToDB(directory string, name string) *sql.DB {
 
 	if _, err := os.Stat(directory + "/" + name); errors.Is(err, os.ErrNotExist) {
 		panic(err)
 	}
 	db, err := sql.Open("sqlite3", directory+"/"+name)
 
 	if err != nil {
 		panic(err)
 	}
 
 	return db
 }
 
 func ensureIndexDB(dir string, filename string) {
 
 	os.MkdirAll(dir, 0755)
 	db, err := sql.Open("sqlite3", "./"+dir+"/"+filename)
 
 	if err != nil {
 		panic(err)
 	}
 
 	defer db.Close()
 
 	// There could be multiple links from the same source to the same destination, though that'd be kinda stupid
 	createLink := "CREATE TABLE IF NOT EXISTS link (source, destination, added_timestamp datetime);"
 	createInvertedIndex := "CREATE TABLE IF NOT EXISTS inverted_index (term, url, count);"
 	createTermIndex := "CREATE INDEX IF NOT EXISTS term_idx ON inverted_index(term);"
 	createDocumentIndex := "CREATE INDEX IF NOT EXISTS url_idx ON inverted_index(url);"
 
 	// TODO: Add filesystem location for this (mv {old} {new} location)
 	createDocument := "CREATE TABLE IF NOT EXISTS document (url UNIQUE, indexed_timestamp datetime);"
 
 	_, err = db.Exec(createLink)
 
 	if err != nil {
 		panic(err)
 	}
 
 	_, err = db.Exec(createInvertedIndex)
 
 	if err != nil {
 		panic(err)
 	}
 
 	_, err = db.Exec(createTermIndex)
 
 	if err != nil {
 		panic(err)
 	}
 
 	_, err = db.Exec(createDocumentIndex)
 
 	if err != nil {
 		panic(err)
 	}
 
 	_, err = db.Exec(createDocument)
 
 	if err != nil {
 		panic(err)
 	}
 
 
 	db.Exec("PRAGMA journal_mode=WAL;")
 	db.Exec("PRAGMA busy_timeout=2000;")
 	db.Exec("PRAGMA synchronous=NORMAL;")
 
 }
 
 func getNextPath(dbCrawl *sql.DB) (string, string) {
 
 	// It's not guaranteed this site will be indexed because it's possible
 	// the process fails during that but is still removed from the db
 	// We could fix this with a locking mechanism, but I don't know how necessary that'd be
 
 	// TODO: Make sure the page from the filesystem is deleted after indexing.
 	popPage := `DELETE FROM page
 						WHERE path = (SELECT path FROM page ORDER BY added_timestamp ASC LIMIT 1)
 						RETURNING url, path`
 
 	selectedUrl := ""
 	selectedPath := ""
 
 	err := dbCrawl.QueryRow(popPage).Scan(&selectedUrl, &selectedPath)
 
 	if err == sql.ErrNoRows {
 		return "", ""
 	} else if err != nil {
 		panic(err)
 	}
 
 	return selectedUrl, selectedPath
 }
 
 func insertLinks(db *sql.DB, source string, destinations []string) error {
 	// createLink := "CREATE TABLE IF NOT EXISTS link (source, destination, added_timestamp datetime);"
 
 	insertLinkQuery := "INSERT INTO link (source, destination, added_timestamp) VALUES (?, ?, CURRENT_TIMESTAMP)"
 	// TODO: Why is it not parsing error return?
 	tx, _ := db.Begin()
 
 	for _, current := range destinations {
 		_, err := tx.Exec(insertLinkQuery, source, current)
 
 		if err != nil {
 			return err
 		}
 	}
 
 	tx.Commit()
 	return nil
 }
 
 
 // document (url, indexed_timestamp datetime)"
 func insertDocument(db *sql.DB, url string) error {
 
 	// createLink := "CREATE TABLE IF NOT EXISTS link (source, destination, added_timestamp datetime);"
 
 	// we don't do on conflict resolution because data sould be deleted before insertion
 	// although... it's maybe possible that two threads could try indexing and re-indexing a site
 	// at the same time, but seems incredibly unlikely.
 
 	insertDocumentQuery := "INSERT INTO document (url, indexed_timestamp) VALUES (?, CURRENT_TIMESTAMP)"
 
 	// TODO: Why is it not parsing error return?
 
 	tx, _ := db.Begin()
 
 	_, err := tx.Exec(insertDocumentQuery, url)
 
 	if err != nil {
 		return err
 	}
 
 	tx.Commit()
 	return nil
 }
 
 func insertTerms(db *sql.DB, url string, terms []string) error {
 
 	insertTerm := "INSERT INTO inverted_index (term, url, count) VALUES (?, ?, ?)"
 
 	termMap := make(map[string]int)
 
 	for _, term := range terms {
 		termMap[term]++
 	}
 
 	tx, _ := db.Begin()
 	for key, value := range termMap {
 		_, err := tx.Exec(insertTerm, key, url, value)
 
 		if err != nil {
 			return err
 		}
 	}
 
 	tx.Commit()
 	return nil
 																					
 }
 
 
 // 	createLink := "CREATE TABLE IF NOT EXISTS link (source, destination, added_timestamp datetime);"
 // 	createInvertedIndex := "CREATE TABLE IF NOT EXISTS inverted_index (term, url, count);"
 // 	createTermIndex := "CREATE INDEX IF NOT EXISTS term_idx ON inverted_index(term);"
 // 	createDocumentIndex := "CREATE INDEX IF NOT EXISTS url_idx ON inverted_index(url);"
 // 	createDocument := "CREATE TABLE IF NOT EXISTS document (url UNIQUE, indexed_timestamp datetime);"
 
 func ensureNotIndexed(url string, db *sql.DB) {
 	deleteDocument := "DELETE FROM document WHERE url = ?"
 	deleteTerm := "DELETE FROM inverted_index WHERE url = ?"
 	deleteLinks := "DELETE FROM link WHERE source = ?"
 
 	// TODO: This should probably be one transaction in case of crashes and such.
 	_, err := db.Exec(deleteDocument, url)
 	if err != nil {
 		panic(err)
 	}
 	_, err = db.Exec(deleteTerm, url)
 	if err != nil {
 		panic(err)
 	}
 	_, err = db.Exec(deleteLinks, url)
 	if err != nil {
 		panic(err)
 	}
 }
 
 func main() {
 
 	dbCrawl := connectToDB(CRAWLED_DIR, CRAWLED_DB_NAME)
 	defer dbCrawl.Close()
 
 	ensureIndexDB(INDEXED_DIR, INDEXED_DB_NAME)
 	dbIndex := connectToDB(INDEXED_DIR, INDEXED_DB_NAME)
 	defer dbIndex.Close()
 
 	for true {
 
 		selectedUrl, selectedPath := getNextPath(dbCrawl)
 		if selectedUrl == "" && selectedPath == "" {
 			fmt.Println("No more documents do index, exiting gracefully.")
 			return
 		}
 		ensureNotIndexed(selectedUrl, dbIndex)
 
 		fmt.Printf("Indexing url: %s\nIndexing path: %s\n", selectedUrl, selectedPath)
 
 		bodyBytes, err := os.ReadFile(selectedPath)
 
 		if err != nil {
 			panic(err)
 		}
 
 		body := string(bodyBytes)
 
 		links := gemtextparser.ParseLinks(body, selectedUrl)
 		fmt.Printf("Links: %v\n", links)
 		err = insertLinks(dbIndex, selectedUrl, links)
 
 
 		if err != nil {
 			panic(err)
 		}
 
 		// TODO: This should be in the same transaction as the link insertions and term insertions
 		err = insertDocument(dbIndex, selectedUrl)
 
 
 		if err != nil {
 			panic(err)
 		}
 
 		words := gemtextparser.StemmedDocument(body)
 		err = insertTerms(dbIndex, selectedUrl, words)
 
 		if err != nil {
 			panic(err)
 		}
 
 
 		for _, word := range words {
 			// TODO: Setup inverted index
 			fmt.Printf("%s,", word)
 		}
 	}
 
 }