blog

Personal blog
git clone git://git.laack.co/blog.git
Log | Files | Refs

search.py (5281B)


      1 import os
      2 import subprocess
      3 import sqlite3
      4 import uuid
      5 import requests
      6 import urllib.parse
      7 
      8 urllib.parse.uses_relative.append('gemini')
      9 urllib.parse.uses_netloc.append('gemini')
     10 
     11 def parse_robots(text, agents=("*", "researcher")):
     12     rules = []
     13     group_agents = []
     14     in_rules = False
     15     for raw in text.splitlines():
     16         line = raw.split("#", 1)[0].strip()
     17         if not line or ":" not in line:
     18             continue
     19         key, val = line.split(":", 1)
     20         key, val = key.strip().lower(), val.strip()
     21         if key == "user-agent":
     22             if in_rules:
     23                 group_agents = []
     24                 in_rules = False
     25             group_agents.append(val.lower())
     26         elif key == "disallow":
     27             in_rules = True
     28             if val and any(a in agents for a in group_agents):
     29                 rules.append(val)
     30     return rules
     31 
     32 
     33 # these are domains I noticed had lots of stuff. 
     34 # I indexed some of them, but added them once I noticed they were far too outsized.
     35 BLACKLIST = {"taz.de", "tuxmachines.org", "techrights.org", "uploadedlobster.com"}
     36 
     37 def is_blacklisted(host):
     38     host = (host or "").lower().rstrip(".")
     39     return any(host == b or host.endswith("." + b) for b in BLACKLIST)
     40 
     41 def can_fetch(robot_dir, domain, target):
     42     os.makedirs(robot_dir, exist_ok=True)
     43     
     44     host = urllib.parse.urlparse(target).hostname
     45     if is_blacklisted(host):
     46         return False
     47 
     48     pth = os.path.join(robot_dir, domain)
     49     if not os.path.exists(pth):
     50         subprocess.run(['timeout', '10', 'gemget', f"gemini://{domain}/robots.txt",
     51                         '-o', pth, "--max-size", "1.0 MB"])
     52     try:
     53         with open(pth, 'r', errors='replace') as f:
     54             rules = parse_robots(f.read())
     55     except FileNotFoundError:
     56         with open(pth,'w') as f:
     57             f.writelines(["Allow: /"]) # implicit consent?
     58         return True
     59 
     60     path = urllib.parse.urlparse(target).path or "/"
     61     for rule in rules:
     62         if path.startswith(rule):
     63             print("Can't fetch: " + target)
     64             return False
     65     return True
     66 
     67 
     68 
     69 def download(data_dir, site):
     70 
     71     domain = urllib.parse.urlparse(site).netloc
     72     if not can_fetch("robots/", domain,  site):
     73         print("Can't fetch.")
     74         return
     75 
     76 
     77     cuuid = uuid.uuid4()
     78     subprocess.run(['timeout', '10', 'gemget', site, '-o', os.path.join(data_dir, str(cuuid)), "--max-size", "1.0 MB"])
     79     return cuuid
     80 
     81 def grab_links(prior, gem_file_path):
     82     link_lines = []
     83     try:
     84         with open(gem_file_path, 'r') as f:
     85             lines = f.readlines()
     86             for line in lines:
     87                 if(len(line) > 2 and line[0:2] == "=>"):
     88                     cs = line[2:]
     89                     while(len(cs) > 0 and (cs[0] == ' ' or cs[0] == '\t')):
     90                         cs = cs[1:]
     91 
     92                     acc = ""
     93                     while(len(cs) > 0 and cs[0] != ' ' and cs[0] != '\t' and cs[0] != '\n'):
     94                         acc += cs[0]
     95                         cs = cs[1:]
     96 
     97                     cl = requests.compat.urljoin(prior, acc)
     98                     link_lines.append(cl)
     99     except:
    100         with open("data/failures.csv", 'a') as f:
    101             f.write(prior + "\n")
    102 
    103     return link_lines
    104 
    105 
    106 def init_db(filename):
    107     conn = sqlite3.connect(filename, timeout=300)
    108 
    109     curs = conn.cursor()
    110     curs.execute('''
    111     CREATE TABLE IF NOT EXISTS site (
    112         id TEXT PRIMARY KEY,
    113         url TEXT NOT NULL
    114     )
    115     ''')
    116 
    117     curs.execute('''
    118     CREATE TABLE IF NOT EXISTS link (
    119         source TEXT NOT NULL,
    120         destination TEXT NOT NULL,
    121         visited INTEGER NOT NULL,
    122         UNIQUE (source, destination)
    123     )
    124      ''')
    125 
    126     curs.execute('CREATE INDEX IF NOT EXISTS link_dest ON link(destination)')
    127     curs.execute('CREATE INDEX IF NOT EXISTS site_url ON site(url)')
    128     curs.execute('CREATE INDEX IF NOT EXISTS link_unvisited ON link(destination)')
    129 
    130     curs.close()
    131     conn.commit()
    132     return conn
    133 
    134 def index(conn, data_dir, site_names):
    135     curs = conn.cursor()
    136 
    137     for site in site_names:
    138         cuuid = download(data_dir, site)
    139         print(site)
    140         links = grab_links(site, 'data/' + str(cuuid))
    141         for link in links:
    142             if("gemini://" in link):
    143                 curs.execute('''
    144                         INSERT OR IGNORE INTO link (source, destination, visited) VALUES (?, ?, 0)
    145                  ''', (site,link))
    146                 conn.commit()
    147 
    148         try:
    149             curs.execute('''
    150                     INSERT INTO site (id, url) VALUES (?, ?)
    151                          ''', (str(cuuid), site))
    152             conn.commit()
    153         except:
    154             pass
    155 
    156     curs.close()
    157     conn.commit()
    158 
    159 data_dir = 'data/'
    160 conn = init_db(data_dir + 'idx.db')
    161 
    162 # seed
    163 index(conn, data_dir, ['gemini://tlgs.one'])
    164 
    165 while True:
    166     curs = conn.cursor()
    167     curs.execute('''
    168         SELECT DISTINCT destination FROM link
    169         WHERE visited = 0
    170           AND destination NOT IN (SELECT url FROM site)
    171         ORDER BY RANDOM()
    172         LIMIT 10
    173     ''')
    174     urls = [row[0] for row in curs.fetchall()]
    175     if not urls:
    176         break
    177     conn.commit()
    178 
    179     index(conn, data_dir, urls)
    180     curs.executemany('UPDATE link SET visited = 1 WHERE destination = ?',
    181                  [(u,) for u in urls])
    182     conn.commit()
    183     curs.close()