gemini-search

main.go (8637B)

---

 package main
 
 import (
 	"database/sql"
 	"errors"
 	"fmt"
 	"geminisearch/gemtextparser"
 	"github.com/makeworld-the-better-one/go-gemini"
 	_ "github.com/mattn/go-sqlite3"
 	"io"
 	"net/url"
 	"os"
 	"strings"
 	"sync"
 	"time"
 )
 
 var OUTPUT_DIR = "outputs/crawled"
 var DB_NAME = "main.db"
 
 var NotGemtextError = errors.New("Meta stated resource is not gemtext.")
 var TooManyRedirects = errors.New("Too many redirects.")
 var InteractiveStatusCode = errors.New("Interactive status code received.")
 var TempFailureStatusCode = errors.New("Temporary failure status code received.")
 var FailureStatusCode = errors.New("Failure status code received.")
 var CertStatusCode = errors.New("Certificate related status code received.")
 
 // TODO: Respect robots.txt (does this go here? Does this go on insertion for querying?)
 // TODO: Limit size returned, could fuck up ram
 
 func fetchGemtext(currentUrl string) (string, string, error) {
 
 	client := &gemini.Client{ConnectTimeout: 5 * time.Second}
 	resp, err := client.Fetch(currentUrl)
 
 	if err != nil {
 		return "", "", err
 	}
 
 	redirects := 0
 
 	for resp.Status/10 == 3 && redirects < 5 {
 		redirects += 1
 
 		fmt.Printf("redirect count: %d, original url: %s\n", redirects, currentUrl)
 
 		base, err := url.Parse(currentUrl)
 
 		if err != nil {
 			return "", "", err
 		}
 
 		redirect, err := url.Parse(resp.Meta)
 
 		if err != nil {
 			return "", "", err
 		}
 
 		currentUrl = base.ResolveReference(redirect).String()
 
 		fmt.Printf("redirect count: %d, resulting url: %s\n", redirects, currentUrl)
 
 		resp, err = client.Fetch(currentUrl)
 
 		if err != nil {
 			return "", "", err
 		}
 	}
 
 	fmt.Printf("Final URL: %s\n", currentUrl)
 
 	if resp.Status/10 == 3 {
 		return "", "", TooManyRedirects
 	}
 
 	// Followed redirects first, then check the rest of the status codes just in case
 	// https://github.com/AyrA/Gemini/blob/master/Protocol.md
 	// 1X are interactive
 
 	if resp.Status/10 == 1 {
 		return "", "", InteractiveStatusCode
 	}
 
 	if resp.Status/10 == 4 {
 		return "", "", TempFailureStatusCode
 	}
 	if resp.Status/10 == 5 {
 		return "", "", FailureStatusCode
 	}
 	if resp.Status/10 == 6 {
 		return "", "", CertStatusCode
 	}
 
 	// Only success (20) should fall through at this point.
 
 	meta := resp.Meta
 
 	fmt.Println(meta)
 
 	if !strings.Contains(meta, "text/gemini") {
 		err = NotGemtextError
 		fmt.Printf("Non-gemtext type: %s\n", meta)
 		return "", "", err
 	}
 
 	bodyBytes, err := io.ReadAll(resp.Body)
 
 	if err != nil {
 
 		// saw this happen with
 		// read tcp {local_ip}:40396->{remote_ip}:1965: read: connection reset by peer
 
 		return "", "", err
 	}
 
 	body := string(bodyBytes)
 
 	// CurrentURL reflects the url after redirects and such applied.
 	return body, currentUrl, nil
 
 }
 
 func setupDB(dir string, filename string) *sql.DB {
 
 	os.MkdirAll(dir, 0755)
 	db, err := sql.Open("sqlite3", "./"+dir+"/"+filename)
 
 	if err != nil {
 		panic(err)
 	}
 
 	createLink := "CREATE TABLE IF NOT EXISTS query_queue (source UNIQUE, added_timestamp datetime);"
 	createPage := "CREATE TABLE IF NOT EXISTS page (url UNIQUE, path, added_timestamp datetime);"
 
 	_, err = db.Exec(createLink)
 
 	if err != nil {
 		panic(err)
 	}
 
 	_, err = db.Exec(createPage)
 
 	if err != nil {
 		panic(err)
 	}
 
 	db.Exec("PRAGMA journal_mode=WAL;")
 	db.Exec("PRAGMA busy_timeout=2000;") // :( is it so soon the migration to postgresql is initiated?
 	db.Exec("PRAGMA synchronous=NORMAL;")
 
 	return db
 }
 
 func missingFilename(path string) bool {
 	return strings.Compare(path, "") == 0 || strings.Compare(path, "/") == 0
 }
 
 // The reason I chose to do {host}/{escaped_path} instead of a UUID or just {escaped_url} is to
 // have a better hierarchy for looking at files that are downloaded.
 
 // I could've done a more deeply nested structure, but cases like the following become annoying:
 // {site}/blog
 // {site}/blog/entry.gmi
 // I could probably assume the first one is /blog/index.gmi, but since the '.' character is valid in urls
 // there doesn't seem to be a reliable way to differentiate between directories and files in paths.
 // This is to say, different server implementation may do weird stuff, and we shouldn't make assumptions about
 // that sort of thing. I also don't want to limit myself to **just** gemtext so this seems good enough for now.
 
 func writeSiteOutput(outputDir string, body string, currentUrl string) (string, error) {
 
 	u, err := url.Parse(currentUrl)
 
 	if err != nil {
 		panic(err)
 	}
 
 	if !u.IsAbs() {
 		return "", errors.New("Must use absolute paths.")
 	}
 
 	// This sometimes seems to include a leading '/' so drop that, no need, directory
 	// hierarchy gives this information to us on the fs.
 
 	filename := u.EscapedPath()
 
 	if len(filename) > 1 && filename[0] == '/' {
 		filename = filename[1:]
 	}
 
 	dirname := u.Host // include subdomains
 
 	if missingFilename(filename) {
 
 		// this is presumptuous, albeit quite standard.
 		// main reason for this is to have uniquely identifying names for files.
 		// unfortunately no way to create a file with an empty filename. 
 		// this can result in ambiguity between {site}/index.gmi and {site}, but we shall assume they are the same.
 
 		filename = "index.gmi"
 	}
 
 	filename = url.PathEscape(filename)
 
 	outputDirectory := outputDir + "/" + "sites" + "/" + dirname + "/"
 	outputPath := outputDirectory + filename
 
 	os.MkdirAll(outputDirectory, 0755)
 
 	// verified correct for ext4, see here for details:
 	// https://serverfault.com/questions/9546/filename-length-limits-on-linux
 
 	if len(filename) <= 255 {
 		err = os.WriteFile(outputPath, []byte(body), 0644)
 	} else {
 		fmt.Println("Skipping file creation; filename too long")
 	}
 
 	if err != nil {
 		panic(err)
 	}
 
 	return outputPath, nil
 }
 
 func getNext(db *sql.DB) string {
 
 	link := ""
 
 	popLinkQuery := `	DELETE FROM query_queue
 						WHERE source = (SELECT source FROM query_queue ORDER BY added_timestamp ASC LIMIT 1)
 						RETURNING source`
 
 	err := db.QueryRow(popLinkQuery).Scan(&link)
 
 	if err == sql.ErrNoRows {
 		return ""
 	} else if err != nil {
 		panic(err)
 	}
 
 	return link
 }
 
 func worker(db *sql.DB, wg *sync.WaitGroup) {
 	defer wg.Done()
 
 	insertPageQuery := "INSERT OR REPLACE INTO page (url, path, added_timestamp) VALUES (?, ?, CURRENT_TIMESTAMP)"
 	insertLinkQuery := "INSERT INTO query_queue (source, added_timestamp) VALUES (?, CURRENT_TIMESTAMP) ON CONFLICT DO UPDATE SET added_timestamp=CURRENT_TIMESTAMP"
 	for true {
 
 		currentUrl := getNext(db)
 
 		if currentUrl == "" {
 			fmt.Printf("No links remaining.\n")
 			return
 		}
 		fmt.Printf("Querying: %s\n", currentUrl)
 
 		body, finalUrl, err := fetchGemtext(currentUrl)
 
 		if err == NotGemtextError {
 			fmt.Println("Not gemtext... Continuing")
 		} else if err != nil {
 			fmt.Printf("Unable to fetch %s: %s\n", currentUrl, err)
 			continue
 		}
 
 		if finalUrl != "" && finalUrl != currentUrl {
 			fmt.Printf("Updating url to reflect redirects: %s => %s\n", currentUrl, finalUrl)
 			currentUrl = finalUrl
 		}
 
 		fmt.Printf("Fetched %s\n", currentUrl)
 
 		forwardGeminiLinks := gemtextparser.ParseLinks(body, currentUrl)
 
 		// TODO: We should check to make sure the filename isn't too long before we get to this point
 		// otherwise we have an annoying db inconsistency. 
 		// I noticed this when performing indexing and it said "file name too long" which means the path
 		// was inserted into the db, but presumably nothing was written to disk because it'd've been too long
 		// also, there is a generally worrying incosistency that occurs when writing to disk and using sqlite
 		// without some sort of locking mechanism. 
 
 		// TODO: When the meta != text/gemini, what happens? I'm seeing a bunch of files that are empty; this 
 		// isn't really what i'd expect, I'd expect some short circuiting.
 
 		outputLocation, err := writeSiteOutput(OUTPUT_DIR, body, currentUrl)
 
 		if err != nil {
 			panic(err)
 		}
 
 		tx, _ := db.Begin()
 
 		_, err = tx.Exec(insertPageQuery, currentUrl, outputLocation)
 
 		if err != nil {
 			panic(err)
 		}
 
 		for _, link := range forwardGeminiLinks {
 
 			_, err := tx.Exec(insertLinkQuery, link)
 
 			if err != nil {
 				panic(err)
 			}
 
 		}
 		tx.Commit()
 	}
 
 }
 
 func main() {
 
 	db := setupDB(OUTPUT_DIR, DB_NAME)
 	defer db.Close()
 
 	base := "gemini://tlgs.one"
 
 	insertLinkQuery := "INSERT INTO query_queue (source, added_timestamp) VALUES (?, CURRENT_TIMESTAMP) ON CONFLICT DO UPDATE SET added_timestamp=CURRENT_TIMESTAMP"
 
 	// bootstrap
 	_, err := db.Exec(insertLinkQuery, base)
 
 	if err != nil {
 		panic(err)
 	}
 
 	var wg sync.WaitGroup
 
 	for i := 0; i < 20; i++ {
 		wg.Add(1)
 		go worker(db, &wg)
 		time.Sleep(time.Second * 5) // This allows the queue to be built up before dispatching to help with fresh runs
 	}
 	wg.Wait()
 
 }