scraping

Scripts for scraping sites
git clone git://git.laack.co/scraping.git
Log | Files | Refs

commit 76c1b01ded51246be571e8820c48932ea4be7573
parent 7bbd67dd1cebc6084a763994fda4521efa5192a4
Author: Andrew Laack <andrew@laack.co>
Date:   Sun,  8 Nov 2026 23:46:34 -0600

Refactor

Diffstat:
Mcodeforces/Makefile | 7++++++-
Acodeforces/editorials.py | 38++++++++++++++++++++++++++++++++++++++
Dcodeforces/fetch-contests.py | 39---------------------------------------
3 files changed, 44 insertions(+), 40 deletions(-)

diff --git a/codeforces/Makefile b/codeforces/Makefile @@ -1,3 +1,8 @@ -scrape: +all: python3 fetch-problems.py > outputs/problems.txt python3 statements.py + python3 editorials.py +editorial: + python3 fetch-problems.py > outputs/problems.txt + python3 editorials.py + diff --git a/codeforces/editorials.py b/codeforces/editorials.py @@ -0,0 +1,38 @@ +import re +from pathlib import Path +from markdownify import markdownify +from playwright.sync_api import sync_playwright + +HERE = Path(__file__).parent +OUT = HERE / "outputs/editorial" +OUT.mkdir(parents=True, exist_ok=True) + +problems = (HERE / "outputs/problems.txt").read_text().split() +contests = dict.fromkeys(re.match(r"\d+", p).group() for p in problems) + +def save(page, contest, out): + page.goto(f"https://codeforces.com/contest/{contest}", timeout=10000) + link = page.locator('a[href*="/blog/entry/"]').filter(has_text=re.compile(r"tutorial|editorial|разбор", re.I)) + if not link.count(): + return print("missing", contest) + url = "https://codeforces.com" + re.search(r"/blog/entry/\d+", link.first.get_attribute("href")).group() + page.goto(url, timeout=10000) + body = page.locator(".topic .ttypography") + if not body.count(): + return print("no post body", contest, url) + html = body.first.inner_html(timeout=5000).replace("</div>", "\n</div>") + md = markdownify(html, heading_style="ATX", escape_underscores=False, escape_asterisks=False) + md = md.replace("$$$$$$", "$$").replace("$$$", "$") + md = re.sub(r"\n{3,}", "\n\n", md).strip() + out.write_text(f"Source: {url}\n\n{md}\n") + print("saved", out) + +with sync_playwright() as p: + page = p.chromium.launch(headless=False).new_page(java_script_enabled=False) + for contest in contests: + out = OUT / f"{contest}.md" + if not out.exists(): + try: + save(page, contest, out) + except Exception as e: + print("error", contest, type(e).__name__, str(e).splitlines()[0]) diff --git a/codeforces/fetch-contests.py b/codeforces/fetch-contests.py @@ -1,39 +0,0 @@ -import re -from pathlib import Path -from markdownify import markdownify -from playwright.sync_api import sync_playwright - -HERE = Path(__file__).parent -OUT = HERE / "outputs/editorial" -OUT.mkdir(parents=True, exist_ok=True) - -problems = (HERE / "outputs/problems.txt").read_text().split() -contests = dict.fromkeys(re.match(r"\d+", p).group() for p in problems) - -with sync_playwright() as p: - page = p.chromium.launch(headless=False).new_page(java_script_enabled=False) - for contest in contests: - out = OUT / f"{contest}.md" - if out.exists(): - continue - try: - page.goto(f"https://codeforces.com/contest/{contest}", timeout=10000) - except: - print("Timeout.") - continue - link = page.locator('a[href*="/blog/entry/"]').filter(has_text=re.compile(r"tutorial|editorial|разбор", re.I)) - if not link.count(): - print("missing", contest) - continue - url = "https://codeforces.com" + re.search(r"/blog/entry/\d+", link.first.get_attribute("href")).group() - try: - page.goto(url, timeout=10000) - except: - print("Timeout.") - continue - html = page.locator(".topic .ttypography").first.inner_html().replace("</div>", "\n</div>") - md = markdownify(html, heading_style="ATX", escape_underscores=False, escape_asterisks=False) - md = md.replace("$$$$$$", "$$").replace("$$$", "$") - md = re.sub(r"\n{3,}", "\n\n", md).strip() - out.write_text(f"Source: {url}\n\n{md}\n") - print("saved", out)