scraping

Scripts for scraping sites
git clone git://git.laack.co/scraping.git
Log | Files | Refs

commit 7bbd67dd1cebc6084a763994fda4521efa5192a4
parent a66331054a3c5b89036343f04b1cccde4e0f73f1
Author: Andrew Laack <andrew@laack.co>
Date:   Sun,  8 Nov 2026 23:28:51 -0600

Scrape editorials

Diffstat:
Mcodeforces/Makefile | 2+-
Acodeforces/fetch-contests.py | 39+++++++++++++++++++++++++++++++++++++++
Rcodeforces/run.py -> codeforces/statements.py | 0
3 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/codeforces/Makefile b/codeforces/Makefile @@ -1,3 +1,3 @@ scrape: python3 fetch-problems.py > outputs/problems.txt - python3 run.py + python3 statements.py diff --git a/codeforces/fetch-contests.py b/codeforces/fetch-contests.py @@ -0,0 +1,39 @@ +import re +from pathlib import Path +from markdownify import markdownify +from playwright.sync_api import sync_playwright + +HERE = Path(__file__).parent +OUT = HERE / "outputs/editorial" +OUT.mkdir(parents=True, exist_ok=True) + +problems = (HERE / "outputs/problems.txt").read_text().split() +contests = dict.fromkeys(re.match(r"\d+", p).group() for p in problems) + +with sync_playwright() as p: + page = p.chromium.launch(headless=False).new_page(java_script_enabled=False) + for contest in contests: + out = OUT / f"{contest}.md" + if out.exists(): + continue + try: + page.goto(f"https://codeforces.com/contest/{contest}", timeout=10000) + except: + print("Timeout.") + continue + link = page.locator('a[href*="/blog/entry/"]').filter(has_text=re.compile(r"tutorial|editorial|разбор", re.I)) + if not link.count(): + print("missing", contest) + continue + url = "https://codeforces.com" + re.search(r"/blog/entry/\d+", link.first.get_attribute("href")).group() + try: + page.goto(url, timeout=10000) + except: + print("Timeout.") + continue + html = page.locator(".topic .ttypography").first.inner_html().replace("</div>", "\n</div>") + md = markdownify(html, heading_style="ATX", escape_underscores=False, escape_asterisks=False) + md = md.replace("$$$$$$", "$$").replace("$$$", "$") + md = re.sub(r"\n{3,}", "\n\n", md).strip() + out.write_text(f"Source: {url}\n\n{md}\n") + print("saved", out) diff --git a/codeforces/run.py b/codeforces/statements.py