scraping

Scripts for scraping sites
git clone git://git.laack.co/scraping.git
Log | Files | Refs

commit 4d1b5ec1df70334ccdfc01c2cdd432b8d46a318d
Author: Andrew Laack <andrew@laack.co>
Date:   Sun,  8 Nov 2026 23:06:19 -0600

Basic problem scraping codeforces

Diffstat:
A.gitignore | 1+
Acodeforces/Makefile | 3+++
Acodeforces/fetch-problems.py | 23+++++++++++++++++++++++
Acodeforces/run.py | 37+++++++++++++++++++++++++++++++++++++
4 files changed, 64 insertions(+), 0 deletions(-)

diff --git a/.gitignore b/.gitignore @@ -0,0 +1 @@ +codeforces/outputs/* diff --git a/codeforces/Makefile b/codeforces/Makefile @@ -0,0 +1,3 @@ +scrape: + python3 fetch-problems.py > outputs/problems.txt + python3 run.py diff --git a/codeforces/fetch-problems.py b/codeforces/fetch-problems.py @@ -0,0 +1,23 @@ +import json +import urllib.request + +URL = "https://codeforces.com/api/problemset.problems" +LIMIT = None + + +def fetch(limit=LIMIT): + req = urllib.request.Request(URL, headers={"User-Agent": "Mozilla/5.0"}) + with urllib.request.urlopen(req) as r: + data = json.load(r) + if data["status"] != "OK": + raise RuntimeError(data.get("comment", "Codeforces API error")) + + ids = [f"{p['contestId']}{p['index']}" for p in data["result"]["problems"]] + return ids[:limit] if limit else ids + + +if __name__ == "__main__": + for problem in fetch(): + print(problem) +else: + PROBLEMS = fetch() diff --git a/codeforces/run.py b/codeforces/run.py @@ -0,0 +1,37 @@ +import re +from pathlib import Path +from markdownify import markdownify +from playwright.sync_api import sync_playwright + +HERE = Path(__file__).parent +OUT = HERE / "outputs/problem-statement" + +with sync_playwright() as p: + page = p.chromium.launch(headless=False).new_page(java_script_enabled=False) + for prob in (HERE / "outputs/problems.txt").read_text().split(): + if any(OUT.glob(f"*/{prob}.md")): + continue + contest, idx = re.match(r"(\d+)(.+)", prob).groups() + for url in (f"https://codeforces.com/problemset/problem/{contest}/{idx}", + f"https://codeforces.com/contest/{contest}/problem/{idx}"): + try: + page.goto(url, timeout=10000) + except: + print("Timeout.") + continue + if page.locator(".problem-statement").count(): + break + else: + print("missing", prob) + continue + tag = page.locator('[title="Difficulty"]') + rating = tag.first.inner_text().strip("* \n") if tag.count() else "unrated" + html = page.inner_html(".problem-statement").replace("</div>", "\n</div>") + md = markdownify(html, heading_style="ATX", escape_underscores=False, escape_asterisks=False) + md = md.replace("$$$$$$", "$$").replace("$$$", "$") + md = re.sub(r"\n{3,}", "\n\n", md).strip() + md = f"Source: {url}\n\n{md}\n" + out = OUT / rating / f"{prob}.md" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(md) + print("saved", out)