commit 4d1b5ec1df70334ccdfc01c2cdd432b8d46a318d
Author: Andrew Laack <andrew@laack.co>
Date: Sun, 8 Nov 2026 23:06:19 -0600
Basic problem scraping codeforces
Diffstat:
4 files changed, 64 insertions(+), 0 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1 @@
+codeforces/outputs/*
diff --git a/codeforces/Makefile b/codeforces/Makefile
@@ -0,0 +1,3 @@
+scrape:
+ python3 fetch-problems.py > outputs/problems.txt
+ python3 run.py
diff --git a/codeforces/fetch-problems.py b/codeforces/fetch-problems.py
@@ -0,0 +1,23 @@
+import json
+import urllib.request
+
+URL = "https://codeforces.com/api/problemset.problems"
+LIMIT = None
+
+
+def fetch(limit=LIMIT):
+ req = urllib.request.Request(URL, headers={"User-Agent": "Mozilla/5.0"})
+ with urllib.request.urlopen(req) as r:
+ data = json.load(r)
+ if data["status"] != "OK":
+ raise RuntimeError(data.get("comment", "Codeforces API error"))
+
+ ids = [f"{p['contestId']}{p['index']}" for p in data["result"]["problems"]]
+ return ids[:limit] if limit else ids
+
+
+if __name__ == "__main__":
+ for problem in fetch():
+ print(problem)
+else:
+ PROBLEMS = fetch()
diff --git a/codeforces/run.py b/codeforces/run.py
@@ -0,0 +1,37 @@
+import re
+from pathlib import Path
+from markdownify import markdownify
+from playwright.sync_api import sync_playwright
+
+HERE = Path(__file__).parent
+OUT = HERE / "outputs/problem-statement"
+
+with sync_playwright() as p:
+ page = p.chromium.launch(headless=False).new_page(java_script_enabled=False)
+ for prob in (HERE / "outputs/problems.txt").read_text().split():
+ if any(OUT.glob(f"*/{prob}.md")):
+ continue
+ contest, idx = re.match(r"(\d+)(.+)", prob).groups()
+ for url in (f"https://codeforces.com/problemset/problem/{contest}/{idx}",
+ f"https://codeforces.com/contest/{contest}/problem/{idx}"):
+ try:
+ page.goto(url, timeout=10000)
+ except:
+ print("Timeout.")
+ continue
+ if page.locator(".problem-statement").count():
+ break
+ else:
+ print("missing", prob)
+ continue
+ tag = page.locator('[title="Difficulty"]')
+ rating = tag.first.inner_text().strip("* \n") if tag.count() else "unrated"
+ html = page.inner_html(".problem-statement").replace("</div>", "\n</div>")
+ md = markdownify(html, heading_style="ATX", escape_underscores=False, escape_asterisks=False)
+ md = md.replace("$$$$$$", "$$").replace("$$$", "$")
+ md = re.sub(r"\n{3,}", "\n\n", md).strip()
+ md = f"Source: {url}\n\n{md}\n"
+ out = OUT / rating / f"{prob}.md"
+ out.parent.mkdir(parents=True, exist_ok=True)
+ out.write_text(md)
+ print("saved", out)