search.py (5281B)
1 import os 2 import subprocess 3 import sqlite3 4 import uuid 5 import requests 6 import urllib.parse 7 8 urllib.parse.uses_relative.append('gemini') 9 urllib.parse.uses_netloc.append('gemini') 10 11 def parse_robots(text, agents=("*", "researcher")): 12 rules = [] 13 group_agents = [] 14 in_rules = False 15 for raw in text.splitlines(): 16 line = raw.split("#", 1)[0].strip() 17 if not line or ":" not in line: 18 continue 19 key, val = line.split(":", 1) 20 key, val = key.strip().lower(), val.strip() 21 if key == "user-agent": 22 if in_rules: 23 group_agents = [] 24 in_rules = False 25 group_agents.append(val.lower()) 26 elif key == "disallow": 27 in_rules = True 28 if val and any(a in agents for a in group_agents): 29 rules.append(val) 30 return rules 31 32 33 # these are domains I noticed had lots of stuff. 34 # I indexed some of them, but added them once I noticed they were far too outsized. 35 BLACKLIST = {"taz.de", "tuxmachines.org", "techrights.org", "uploadedlobster.com"} 36 37 def is_blacklisted(host): 38 host = (host or "").lower().rstrip(".") 39 return any(host == b or host.endswith("." + b) for b in BLACKLIST) 40 41 def can_fetch(robot_dir, domain, target): 42 os.makedirs(robot_dir, exist_ok=True) 43 44 host = urllib.parse.urlparse(target).hostname 45 if is_blacklisted(host): 46 return False 47 48 pth = os.path.join(robot_dir, domain) 49 if not os.path.exists(pth): 50 subprocess.run(['timeout', '10', 'gemget', f"gemini://{domain}/robots.txt", 51 '-o', pth, "--max-size", "1.0 MB"]) 52 try: 53 with open(pth, 'r', errors='replace') as f: 54 rules = parse_robots(f.read()) 55 except FileNotFoundError: 56 with open(pth,'w') as f: 57 f.writelines(["Allow: /"]) # implicit consent? 58 return True 59 60 path = urllib.parse.urlparse(target).path or "/" 61 for rule in rules: 62 if path.startswith(rule): 63 print("Can't fetch: " + target) 64 return False 65 return True 66 67 68 69 def download(data_dir, site): 70 71 domain = urllib.parse.urlparse(site).netloc 72 if not can_fetch("robots/", domain, site): 73 print("Can't fetch.") 74 return 75 76 77 cuuid = uuid.uuid4() 78 subprocess.run(['timeout', '10', 'gemget', site, '-o', os.path.join(data_dir, str(cuuid)), "--max-size", "1.0 MB"]) 79 return cuuid 80 81 def grab_links(prior, gem_file_path): 82 link_lines = [] 83 try: 84 with open(gem_file_path, 'r') as f: 85 lines = f.readlines() 86 for line in lines: 87 if(len(line) > 2 and line[0:2] == "=>"): 88 cs = line[2:] 89 while(len(cs) > 0 and (cs[0] == ' ' or cs[0] == '\t')): 90 cs = cs[1:] 91 92 acc = "" 93 while(len(cs) > 0 and cs[0] != ' ' and cs[0] != '\t' and cs[0] != '\n'): 94 acc += cs[0] 95 cs = cs[1:] 96 97 cl = requests.compat.urljoin(prior, acc) 98 link_lines.append(cl) 99 except: 100 with open("data/failures.csv", 'a') as f: 101 f.write(prior + "\n") 102 103 return link_lines 104 105 106 def init_db(filename): 107 conn = sqlite3.connect(filename, timeout=300) 108 109 curs = conn.cursor() 110 curs.execute(''' 111 CREATE TABLE IF NOT EXISTS site ( 112 id TEXT PRIMARY KEY, 113 url TEXT NOT NULL 114 ) 115 ''') 116 117 curs.execute(''' 118 CREATE TABLE IF NOT EXISTS link ( 119 source TEXT NOT NULL, 120 destination TEXT NOT NULL, 121 visited INTEGER NOT NULL, 122 UNIQUE (source, destination) 123 ) 124 ''') 125 126 curs.execute('CREATE INDEX IF NOT EXISTS link_dest ON link(destination)') 127 curs.execute('CREATE INDEX IF NOT EXISTS site_url ON site(url)') 128 curs.execute('CREATE INDEX IF NOT EXISTS link_unvisited ON link(destination)') 129 130 curs.close() 131 conn.commit() 132 return conn 133 134 def index(conn, data_dir, site_names): 135 curs = conn.cursor() 136 137 for site in site_names: 138 cuuid = download(data_dir, site) 139 print(site) 140 links = grab_links(site, 'data/' + str(cuuid)) 141 for link in links: 142 if("gemini://" in link): 143 curs.execute(''' 144 INSERT OR IGNORE INTO link (source, destination, visited) VALUES (?, ?, 0) 145 ''', (site,link)) 146 conn.commit() 147 148 try: 149 curs.execute(''' 150 INSERT INTO site (id, url) VALUES (?, ?) 151 ''', (str(cuuid), site)) 152 conn.commit() 153 except: 154 pass 155 156 curs.close() 157 conn.commit() 158 159 data_dir = 'data/' 160 conn = init_db(data_dir + 'idx.db') 161 162 # seed 163 index(conn, data_dir, ['gemini://tlgs.one']) 164 165 while True: 166 curs = conn.cursor() 167 curs.execute(''' 168 SELECT DISTINCT destination FROM link 169 WHERE visited = 0 170 AND destination NOT IN (SELECT url FROM site) 171 ORDER BY RANDOM() 172 LIMIT 10 173 ''') 174 urls = [row[0] for row in curs.fetchall()] 175 if not urls: 176 break 177 conn.commit() 178 179 index(conn, data_dir, urls) 180 curs.executemany('UPDATE link SET visited = 1 WHERE destination = ?', 181 [(u,) for u in urls]) 182 conn.commit() 183 curs.close()