Improve scrapers content extraction

pyproject.toml

@@ -11,11 +11,11 @@ dependencies = [
     "pgvector",
     "alembic",
     "httpx",
-    "trafilatura",
     "langchain-text-splitters",
     "celery",
     "redis",
     "discord-py>=2.7.1",
+    "beautifulsoup4>=4.15.0",
 ]
 
 [dependency-groups]

src/ingestion/scrapers/html_scraper.py

@@ -1,43 +1,103 @@
-import httpx
-import trafilatura
-
-from src.config.logger import get_logger
-
-log = get_logger(__name__)
-
-
-def scrape(url: str) -> tuple[str, str | None]:
-    """Fetch a URL and extract clean text + title.
-
-    Returns (text, title). Title may be None if not found. Text may be empty
-    if trafilatura couldn't extract anything; the caller should check and skip.
-    """
-    log.info("scrape_started", url=url)
-    with httpx.Client(timeout=30, follow_redirects=True) as client:
-        response = client.get(url)
-        response.raise_for_status()
-
-    # Known limitations (tracked as a separate ticket):
-    #  - Misses content inside aria-hidden="true" accordions (common on FAQ pages)
-    #  - Misses question text but gets answer text on other FAQ pages, we should get both
-    #  - Link formatting is markdown-style and may need cleanup downstream
-    # Improving extraction quality is a tuning ticket, not a blocker for development.
-    text = (
-        trafilatura.extract(
-            response.text,
-            favor_recall=True,
-            include_links=True,
-            include_tables=True,
-            include_formatting=False,
-            include_comments=False,
-            deduplicate=True,
-            output_format="txt",
-        )
-        or ""
-    )
-
-    metadata = trafilatura.extract_metadata(response.text)
-    title = metadata.title if metadata else None
-
-    log.info("scrape_complete", url=url, chars=len(text), title=title)
-    return text, title
+import re
+from urllib.parse import urlsplit
+
+import httpx
+from bs4 import BeautifulSoup
+
+from src.config.logger import get_logger
+
+log = get_logger(__name__)
+
+_BOILERPLATE_TAGS = ["header", "footer", "nav", "script", "style", "noscript"]
+_BOILERPLATE_SELECTORS = (
+    ".navbar-container, .navbar, .footer, .global-nav, "
+    ".navigation, .topbar, .content__meta, .visuallyhidden, "
+    ".resource__contributors-list, #on-this-page, .resource__sources__heading"
+)
+_BLOCK_TAGS = [
+    "p",
+    "div",
+    "li",
+    "h1",
+    "h2",
+    "h3",
+    "h4",
+    "h5",
+    "h6",
+    "tr",
+    "section",
+    "article",
+    "ul",
+    "ol",
+    "table",
+    "br",
+    "blockquote",
+]
+
+
+def _extract_from_html(html: str) -> tuple[str, str | None]:
+    log.info("html_extract_start", html_len=len(html))
+
+    soup = BeautifulSoup(html, "html.parser")
+
+    root = soup.select_one("main, [role=main], #main-content")
+    if root is None:  # CCSS-style pages have no <main>
+        root = soup.body or soup
+
+    # strip boilerplate from within the extracted region
+    for t in root(_BOILERPLATE_TAGS):
+        t.decompose()
+    for el in root.select(_BOILERPLATE_SELECTORS):
+        el.decompose()
+
+    log.info("boilerplate_removed")
+
+    # format links after removing boilerplate
+    for link in root.find_all("a"):
+        href = (link.get("href") or "").strip()
+        if href and not href.startswith("#"):
+            text = link.get_text(" ", strip=True)
+            is_pdf = urlsplit(href).path.lower().endswith(".pdf")
+            marker = "PDF" if is_pdf else "Link"
+            link.replace_with(f"{text} [{marker}: {href}] ")
+
+    log.info("links_parsed")
+
+    for tag in root.find_all(_BLOCK_TAGS):
+        tag.append("\n")
+
+    text = root.get_text(" ", strip=False)
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r" *\n *", "\n", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    text = text.strip()
+
+    title = soup.title.get_text(strip=True) if soup.title else None
+
+    log.info("html_extract_complete", text_len=len(text), has_title=title is not None)
+    return text, title
+
+
+def scrape(url: str) -> tuple[str, str | None]:
+    """Fetch a URL and extract clean text + title.
+
+    Returns (text, title). Title may be None if not found.
+    """
+    log.info("scrape_called", url=url)
+
+    log.info("http_request_start", url=url)
+    with httpx.Client(timeout=30, follow_redirects=True) as client:
+        response = client.get(url)
+        response.raise_for_status()
+
+    log.info(
+        "http_request_success",
+        url=url,
+        status_code=response.status_code,
+        bytes=len(response.text),
+    )
+
+    log.info("html_parsing_start", url=url)
+    text, title = _extract_from_html(response.text)
+    log.info("scrape_complete", url=url, chars=len(text), title=title)
+    return text, title

tests/ingestion/fixtures/CCSS_ArticleTest.html

@@ -0,0 +1,25 @@
+<!doctype html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><link rel=stylesheet href=https://ccss.carleton.ca/css/styles.6e91bbb8770e829f7718dc3011c02037c6c550a9338b13c8b1f092ca356325adabdb3a1ded197ef6fa835569b88930d412d5581f7c630e13a1d0c39679202e43.css crossorigin=anonymous integrity="sha512-bpG7uHcOgp93GNwwEcAgN8bFUKkzixPIsfCSyjVjJa2r2zod7Rl+9vqDVWm4iTDUEtVYH3xjDhOh0MOWeSAuQw=="><link rel=stylesheet href=https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.4/css/all.min.css integrity="sha512-1ycn6IcaQQ40/MKBW2W4Rhis/DbILU74C1vSrLJxCq57o941Ym01SwNsOMqvEBFlcgUa6xLiPY/NS5R+E6ztJQ==" crossorigin=anonymous referrerpolicy=no-referrer><link rel=preconnect href=https://fonts.gstatic.com><link href="https://fonts.googleapis.com/css2?family=Nunito:wght@600&display=swap" rel=stylesheet><link href=https://ccss.carleton.ca/favicon.ico rel=icon /><title>Carleton Computer Science Society | Making a Balanced Course Schedule</title>
+<meta name=description content="The Carleton Computer Science Society is a student led organization that represents all computer science students attending Carleton University."><meta property="og:type" content="website"><meta property="og:url" content="https://ccss.carleton.ca"><meta property="og:title" content="Carleton Computer Science Society | Making a Balanced Course Schedule"><meta property="og:description" content="The Carleton Computer Science Society is a student led organization that represents all computer science students attending Carleton University."><meta property="og:image" content="https://ccss.carleton.ca/images/orientation2018-min.jpeg"><meta property="twitter:card" content="summary_large_image"><meta property="twitter:url" content="https://ccss.carleton.ca"><meta property="twitter:title" content="Carleton Computer Science Society | Making a Balanced Course Schedule"><meta property="twitter:description" content="The Carleton Computer Science Society is a student led organization that represents all computer science students attending Carleton University."><meta property="twitter:image" content="https://ccss.carleton.ca/images/orientation2018-min.jpeg"><meta http-equiv=Cache-control content="no-cache"><link rel=apple-touch-icon href=images/ccss-logo-ios.png></head><body><div class=navbar-container><div class=navbar-wrapper><div class=navbar id=primary_nav_wrap><a href=https://ccss.carleton.ca/ class="nav-brand nav_item"><img src=/images/ccss-logo-2022.png class=navbar-logo alt="The logo of the Carleton Computer Science Society"></a><ul class=navbar-items><li><a href=/about>ABOUT</a><ul><li><a href=/about/team>THE TEAM</a></li><li><a href=/about/governance>GOVERNANCE</a></li><li><a href=/partnerships>PARTNER WITH US</a></li><li><a href=https://docs.google.com/spreadsheets/d/171SP2PWsn6LUza4QzX0nNi2-qwFZnKN-2Q8ERwGkV6o target=_blank>BUDGET</a></li></ul></li><li><a href=/events>EVENTS</a><ul><li><a href=/jobuary>JOBUARY</a></li><li><a href=/hackthetunnels>HACK THE TUNNELS</a></li><li><a href=/events/weekofawesome>WEEK OF AWESOME</a></li></ul></li><li><a href=/news>NEWS</a></li><li><a href=#>COMMUNITY</a><ul><li><a href=/blog>BLOG</a></li><li><a href=/community/organizations>ORGANIZATIONS</a></li></ul></li><li><a href=/volunteer>VOLUNTEER</a></li><li><a href=/resources/>RESOURCES</a></li></ul><div class=mobile-hamburger><i class="fas fa-bars"></i></div></div></div></div><header class=header><a href=https://ccss.carleton.ca/ class="nav-brand nav_item"><img width=180px src=/images/ccss-logo-2022.png alt="The logo of the Carleton Computer Science Society">
+</a><input class=menu-btn type=checkbox id=menu-btn>
+<label class=menu-icon for=menu-btn><span class=navicon></span></label><ul class=menu><li class=mobile-dropdown><div class=mobile-dropdown-header><a href=/about>ABOUT</a>
+<button class=mobile-dropdown-toggle aria-label="Toggle submenu">
+<i class="fa fa-chevron-down"></i></button></div><ul class=mobile-submenu><li><a href=/about/team>THE TEAM</a></li><li><a href=/about/governance>GOVERNANCE</a></li><li><a href=/partnerships>PARTNER WITH US</a></li><li><a href=https://docs.google.com/spreadsheets/d/171SP2PWsn6LUza4QzX0nNi2-qwFZnKN-2Q8ERwGkV6o target=_blank>BUDGET</a></li></ul></li><li class=mobile-dropdown><div class=mobile-dropdown-header><a href=/events>EVENTS</a>
+<button class=mobile-dropdown-toggle aria-label="Toggle submenu">
+<i class="fa fa-chevron-down"></i></button></div><ul class=mobile-submenu><li><a href=/jobuary>JOBUARY</a></li><li><a href=/hackthetunnels>HACK THE TUNNELS</a></li><li><a href=/events/weekofawesome>WEEK OF AWESOME</a></li></ul></li><li><a href=/news>NEWS</a></li><li class=mobile-dropdown><div class=mobile-dropdown-header><a href=#>COMMUNITY</a>
+<button class=mobile-dropdown-toggle aria-label="Toggle submenu">
+<i class="fa fa-chevron-down"></i></button></div><ul class=mobile-submenu><li><a href=/blog>BLOG</a></li><li><a href=/community/organizations>ORGANIZATIONS</a></li></ul></li><li><a href=/volunteer>VOLUNTEER</a></li><li><a href=/resources/>RESOURCES</a></li></ul></header><script>document.querySelectorAll(".mobile-dropdown-toggle").forEach(function(e){e.addEventListener("click",function(){e.closest(".mobile-dropdown").classList.toggle("open")})})</script><div class=article-question__page><div class=article-question__content><h1 class=article-question__title>Making a Balanced Course Schedule</h1><div class=article-question__summary><i>Careful planning can help you create a timetable that supports your academic goals, fits your lifestyle, and keeps your workload manageable.</i></div><div class=resource__updated-at><i class="far fa-clock" aria-hidden=true></i>
+Updated:
+June 27, 2026</div><hr class=resource__separator style=margin-bottom:-.7rem><div class=article-question__body><h3 id=consider-all-the-possibilities>Consider all the Possibilities</h3><p>When you are initially building your schedule, you might find it helpful to <strong>start by adding in all possible sections</strong> of your mandatory classes, and do the same for any elective classes you absolutely want to take.</p><p>While this may look a little scary at first, you will have a visual representation of classes that may cause timing conflicts in your schedule - which makes planning your classes a bit easier.</p><p>If you are going into your first year, classes that you will (likely) need to add are:</p><ul><li>COMP 1405</li><li>COMP 1406</li><li>COMP 1805</li><li>MATH 1007</li><li>MATH 1104</li></ul><p>You can visit <a href=https://carleton.ca/registration/course-selection-guide/bcs/>this</a> SCS page to view suggested course sequences for first-year students, if you’re curious about when students typically take the courses mentioned above.</p><p><strong>NOTE:</strong> <em>The courses listed above are for <strong>Bachelor of Computer Science</strong> students. If you&rsquo;re a <strong>Bachelor of Cybersecurity</strong> student, refer to your <a href="https://carleton.ca/scs/current-students/bachelor-of-cybersecurity/bcyber-courses-and-registration/#:~:text=a%20payment.-,Course%20Patterns,-It%20is%20recommended">course pattern</a> instead.</em></p><br><h3 id=class-timing>Class Timing</h3><p>An important thing to consider when planning your schedule is class timing, as it can impact your class attendance, or how successful you are in the course.</p><p><strong>Are you a morning person?</strong> If yes, maybe you want to consider taking early morning classes to free up the rest of your day.</p><p><strong>Are you a night owl who wakes up late?</strong> If this is the case, maybe willingly adding an 8:30 am class into your scheduling is a bad idea.</p><p>Do you <strong>have obligations during the evening, or just like hanging out at night?</strong> If so, you might want to consider taking classes earlier in the day, or taking “unscheduled” courses that are flexible as to when classwork needs to be completed.</p><p>If you’re a <strong>commuter student</strong> - you might want to consider trying to <strong>limit your in-person classes to certain days</strong>, in order to save you some travel time.</p><br><h3 id=lunch-time>Lunch Time</h3><p>When adding classes, you may be tempted to add all of your classes consecutively to consolidate your classes to two or three days of your schedule.</p><p>While you might think you’ll be okay - consider planning a gap in your schedule in order to <strong>have time to eat your lunch.</strong> It likely won’t be possible to align it to a certain time frame, but having time to refuel in order to be able to focus in class is critical!</p><br><h3 id=other-factors-outside-of-timing-to-consider>Other Factors (Outside of Timing) to Consider</h3><p>Outside of class timing, there are a couple of other factors that you should take into consideration when planning your classes.</p><p>Here are some guiding questions:</p><ul><li>What kind of course is it? Is it a mandatory class? Is it a prerequisite for other classes?</li><li>Is it online-friendly? Are there live lectures or is it unscheduled?</li><li>Do you need to sprint across campus between blocks?</li></ul><br><h3 id=time-ticket-madness>Time-Ticket Madness</h3><p>If you are not one of the lucky few to have an early registration <a href=https://carleton.ca/registration/dates/timetickets/>time ticket</a>, the initial schedule that you made may not work out, as <strong>some classes may fill up before your time ticket.</strong> In order to prevent this being an issue for you - you can do a couple of things to prepare.</p><p>First off, before your time ticket, you can <strong>create multiple different versions of your schedule</strong> with different sections of the same classes. However, this isn’t fail-proof, and one of the classes that you were interested in taking might be entirely full. Since there is always a probability of this happening, be prepared to revise your schedule right before your timeslot, in order to not have to rush to adjust courses when it opens.</p><p>If there is a course that you have your heart set on, but it fills up before you’re able to register - you might be able to join a <a href=https://carleton.ca/registration/waitlisting/>waitlist</a> for the course. Oftentimes, a couple of students will drop any given course, meaning that there is a chance that you’d be able to register in it closer to/during the term.</p><br><h3 id=rate-my-prof>Rate My Prof</h3><p>While you may be tempted to solely base your professor choices on their RateMyProf ratings - it is best to <strong>take them with a grain of salt.</strong> However, if most of the reviews are positive - then the rating is more likely to be credible, as negative reviews are sometimes written in reaction to getting a bad grade in the class (which can be a result of lack of engagement in the course on the student’s part).</p><p>If you are seeking to get insight on which professor you might want to choose, you can <strong>ask upper year students for advice!</strong></p><p>You can also check out <a href=https://outline.scs.carleton.ca/>past course outlines</a> to see how specific instructors have strucutured their course in the past.</p><p><strong>NOTE:</strong> <em>Course outlines can change each year. Use past outlines as a reference only, and don’t rely on them as a guarantee of what the course will be like this year.</em></p><br><h3 id=no-schedule-is-perfect>No Schedule is Perfect</h3><p>Even if your schedule doesn’t turn out perfectly - whether due to factors beyond your control or decisions you later wish you’d made differently - it’s not the end of the world. Take note of what didn’t work and use that insight to improve your planning the next time course registration comes around.</p><p>If you ever want to make changes to your courses down the line, <strong>you are able to drop and add courses until the respective deadlines</strong>, which you can find in the <a href=https://students.carleton.ca/academic-dates/>Academic Calendar</a>.</p></div><hr class=resource__separator><div style=height:.5rem></div><div class=resource__contributors><span><i class="far fa-user" aria-hidden=true></i></span>
+<span class=resource__contributors-list><span class=resource__contributor-pill>Veronica Mordvinova</span></span></div><div style=height:.2rem></div><div class=article-question__related-content><h3>Related Content:</h3><div class=resources-topic__questions><a class=resources-topic__question href=/resources/faqs/first-year-courses-order/><div class=faq-content><h3>Do I have to take first year courses in a specific order?</h3><p>COMP 1405 and 1406 should be taken in Fall and Winter respectively. Other first-year core courses can be taken in either term.</p></div><i class="fas fa-arrow-right faq-arrow"></i>
+</a><a class=resources-topic__question href=/resources/articles/reducing-second-year-workload/><div class=faq-content><h3>Ways to Reduce Your Second Year Workload</h3><p>Strategically selecting courses in your first year can help reduce your second-year workload and set you up for academic success.</p></div><i class="fas fa-arrow-right faq-arrow"></i></a></div></div><div style=height:.2rem></div><div class=resource__sources><div><i class="fas fa-external-link-alt" aria-hidden=true></i>
+<span class=resource__sources__heading>Links and Sources</span></div><p>(1)
+<a href=https://carleton.ca/registration/course-selection-guide/bcs/>BCS first year course selection guide</a></p><p>(2)
+<a href=https://carleton.ca/registration/dates/timetickets/>Time Tickets</a></p><p>(3)
+<a href=https://carleton.ca/registration/waitlisting/>Waitlists</a></p><p>(4)
+<a href=https://outline.scs.carleton.ca/>School of Computer Science old course outlines</a></p><p>(5)
+<a href=https://students.carleton.ca/academic-dates/>Academic Dates and Deadlines</a></p></div></div></div><noscript><img src=https://shynet-mpb4.onrender.com/ingress/7b07c3d5-c87b-4c27-a3c0-6d7c060d4d17/pixel.gif></noscript><script defer src=https://shynet-mpb4.onrender.com/ingress/7b07c3d5-c87b-4c27-a3c0-6d7c060d4d17/script.js></script></body><div class=footer><div class=footer-text><p><b>Carleton Computer Science Society</b><br>4135 Herzberg Laboratories<br>1125 Colonel By Drive,<br>Ottawa, ON K1S 5B6<br><br>email: <a href=mailto:bod@ccss.carleton.ca>bod@ccss.carleton.ca</a></p></div><div class=footer-socials><a href=http://discord.carletoncomputersciencesociety.ca><i class="fab fa-discord"></i></a>
+<a href=https://www.instagram.com/carletoncss/><i class="fab fa-instagram"></i></a>
+<a href=https://www.facebook.com/CarletonComputerScienceSociety><i class="fab fa-facebook"></i></a>
+<a href=https://ca.linkedin.com/company/carleton-computer-science-society><i class="fab fa-linkedin"></i></a>
+<a href=https://www.youtube.com/@carletoncomputersciencesoc305><i class="fab fa-youtube"></i></a>
+<a href=https://github.com/CarletonComputerScienceSociety><i class="fab fa-github"></i></a></div><div class=footer-copyright>© 2026 Carleton Computer Science Society</div></div><script src=https://code.jquery.com/jquery-1.9.1.min.js integrity="sha256-wS9gmOZBqsqWxgIVgA8Y9WcQOa7PgSIX+rPA0VL2rbQ=" crossorigin=anonymous></script><script type=text/javascript src=https://ccss.carleton.ca/js/bundle.a84941d9e5d05ec6b591e8f61e5cd392217520701e85eb735060ee6be204d7c359d4b381f91840b289361e49c7a908afc7ec5a92c6a72b3add687ffa3f0f3ac9.js integrity="sha512-qElB2eXQXsa1kej2HlzTkiF1IHAehetzUGDua+IE18NZ1LOB+RhAsok2HknHqQivx+xaksanKzrdaH/6Pw86yQ==" crossorigin=anonymous></script></html>