scraping

Scripts for scraping sites
git clone git://git.laack.co/scraping.git
Log | Files | Refs

editorials.py (1694B)


      1 import re
      2 import time
      3 import time
      4 from pathlib import Path
      5 from markdownify import markdownify
      6 from playwright.sync_api import sync_playwright
      7 
      8 HERE = Path(__file__).parent
      9 OUT = HERE / "outputs/editorial"
     10 OUT.mkdir(parents=True, exist_ok=True)
     11 
     12 problems = (HERE / "outputs/problems.txt").read_text().split()
     13 contests = dict.fromkeys(re.match(r"\d+", p).group() for p in problems)
     14 
     15 def save(page, contest, out):
     16     page.goto(f"https://codeforces.com/contest/{contest}", timeout=10000)
     17     link = page.locator('a[href*="/blog/entry/"]').filter(has_text=re.compile(r"tutorial|editorial|разбор", re.I))
     18     if not link.count():
     19         return print("missing", contest)
     20     url = "https://codeforces.com" + re.search(r"/blog/entry/\d+", link.first.get_attribute("href")).group()
     21     page.goto(url, timeout=10000)
     22     body = page.locator(".topic .ttypography")
     23     if not body.count():
     24         return print("no post body", contest, url)
     25     html = body.first.inner_html(timeout=5000).replace("</div>", "\n</div>")
     26     md = markdownify(html, heading_style="ATX", escape_underscores=False, escape_asterisks=False)
     27     md = md.replace("$$$$$$", "$$").replace("$$$", "$")
     28     md = re.sub(r"\n{3,}", "\n\n", md).strip()
     29     out.write_text(f"Source: {url}\n\n{md}\n")
     30     print("saved", out)
     31 
     32 with sync_playwright() as p:
     33     page = p.chromium.launch(headless=False).new_page(java_script_enabled=False)
     34     for contest in contests:
     35         out = OUT / f"{contest}.md"
     36         if not out.exists():
     37             try:
     38                 save(page, contest, out)
     39             except Exception as e:
     40                 print("error", contest, type(e).__name__, str(e).splitlines()[0])
     41             time.sleep(5)