scraping

Scripts for scraping sites
git clone git://git.laack.co/scraping.git
Log | Files | Refs

statements.py (1571B)


      1 import re
      2 from pathlib import Path
      3 from markdownify import markdownify
      4 from playwright.sync_api import sync_playwright
      5 
      6 HERE = Path(__file__).parent
      7 OUT = HERE / "outputs/problem-statement"
      8 
      9 with sync_playwright() as p:
     10     page = p.chromium.launch(headless=False).new_page(java_script_enabled=False)
     11     for prob in (HERE / "outputs/problems.txt").read_text().split():
     12         if any(OUT.glob(f"*/{prob}.md")):
     13             continue
     14         contest, idx = re.match(r"(\d+)(.+)", prob).groups()
     15         for url in (f"https://codeforces.com/problemset/problem/{contest}/{idx}",
     16                     f"https://codeforces.com/contest/{contest}/problem/{idx}"):
     17             try:
     18                 page.goto(url, timeout=10000)
     19             except:
     20                 print("Timeout.")
     21                 continue
     22             if page.locator(".problem-statement").count():
     23                 break
     24         else:
     25             print("missing", prob)
     26             continue
     27         tag = page.locator('[title="Difficulty"]')
     28         rating = tag.first.inner_text().strip("* \n") if tag.count() else "unrated"
     29         html = page.inner_html(".problem-statement").replace("</div>", "\n</div>")
     30         md = markdownify(html, heading_style="ATX", escape_underscores=False, escape_asterisks=False)
     31         md = md.replace("$$$$$$", "$$").replace("$$$", "$")
     32         md = re.sub(r"\n{3,}", "\n\n", md).strip()
     33         md = f"Source: {url}\n\n{md}\n"
     34         out = OUT / rating / f"{prob}.md"
     35         out.parent.mkdir(parents=True, exist_ok=True)
     36         out.write_text(md)
     37         print("saved", out)