diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index a6f8fd3..4aa758d 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -1,27 +1,29 @@ -# This workflow will run tests using node and then publish a package to GitHub Packages when a release is created -# For more information see: https://docs.github.com/en/actions/publishing-packages/publishing-nodejs-packages - name: npm package on: release: types: [created] +permissions: + id-token: write # Required for OIDC + contents: read + jobs: publish-npm: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - run: echo "PACKAGE_JSON=$(jq -c . < package.json)" >> $GITHUB_ENV - uses: pnpm/action-setup@v4 with: - version: 10.7.0 - - uses: actions/setup-node@v3 + version: ${{ fromJson(env.PACKAGE_JSON).volta.pnpm }} + - uses: actions/setup-node@v4 with: - node-version: 22.14.0 - registry-url: https://registry.npmjs.org/ - - run: pnpm i --frozen-lockfile + node-version: ${{ fromJson(env.PACKAGE_JSON).volta.node }} + registry-url: https://registry.npmjs.org/ + - name: Upgrade npm for OCID support + run: npm i -g npm@latest + - run: pnpm install --frozen-lockfile - run: pnpm clean - run: pnpm build - - run: pnpm -r publish --no-git-checks --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + - run: pnpm -r publish --provenance --no-git-checks \ No newline at end of file diff --git a/package.json b/package.json index acb0d3f..7e4d5ca 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,6 @@ "devDependencies": { "@biomejs/biome": "1.9.4", "rimraf": "^6.0.1", - "syncpack": "^13.0.2", "typescript": "5.5.4" }, "scripts": { diff --git a/packages/scraper-strategies/package.json b/packages/scraper-strategies/package.json index 132e826..89309a8 100644 --- a/packages/scraper-strategies/package.json +++ b/packages/scraper-strategies/package.json @@ -1,12 +1,12 @@ { "name": "@qnaplus/scraper-strategies", - "version": "3.1.1", + "version": "3.1.2", "description": "Request stategies for @qnaplus/scraper", "main": "dist/index.js", "repository": { "url": "git+ssh://git@github.com/qnaplus/scraper.git" }, - "author": "Battlesquid <25509915+Battlesquid@users.noreply.github.com>", + "author": "battlesquid <25509915+battlesquid@users.noreply.github.com>", "license": "GPL-3.0", "scripts": { "build": "tsc", diff --git a/packages/scraper/README.md b/packages/scraper/README.md index 8ba6c15..7bae7f0 100644 --- a/packages/scraper/README.md +++ b/packages/scraper/README.md @@ -1,6 +1,6 @@ # scraper -A set of utilities focused on scraping the [VEX Robotics Q&A](https://www.robotevents.com/V5RC/2024-2025/QA/). +A set of utilities focused on scraping the [VEX Robotics Q&A](https://events.vex.com/V5RC/2024-2025/QA/). ## Usage diff --git a/packages/scraper/package.json b/packages/scraper/package.json index b128ec3..1f2793e 100644 --- a/packages/scraper/package.json +++ b/packages/scraper/package.json @@ -1,12 +1,12 @@ { "name": "@qnaplus/scraper", - "version": "3.1.1", + "version": "3.1.2", "description": "Utility package to scrape questions from the VEX Robotics Q&A.", "main": "dist/index.js", "repository": { "url": "git+ssh://git@github.com/qnaplus/scraper.git" }, - "author": "Battlesquid <25509915+Battlesquid@users.noreply.github.com>", + "author": "battlesquid <25509915+battlesquid@users.noreply.github.com>", "license": "GPL-3.0", "scripts": { "build": "tsc", diff --git a/packages/scraper/src/modules/parsing.ts b/packages/scraper/src/modules/parsing.ts index e2d1d22..35cf398 100644 --- a/packages/scraper/src/modules/parsing.ts +++ b/packages/scraper/src/modules/parsing.ts @@ -11,13 +11,13 @@ interface QnaUrlPageParams extends BaseQnaUrlParams { page: number; } -export type QnaHomeUrl = `https://www.robotevents.com/${string}/${string}/QA`; +export type QnaHomeUrl = `https://events.vex.com/${string}/${string}/QA`; export type QnaPageUrl = `${QnaHomeUrl}?page=${string}`; export type QnaIdUrl = `${QnaHomeUrl}/${string}`; export const validateQnaUrl = (url: string): Record => { const regex = - /^https:\/\/www\.robotevents\.com\/(?\w+)\/(?\d{4}-\d{4})\/QA(?:\/(?\d+))?(?:\?page=(?\d+))?$/; + /^https:\/\/events\.vex\.com\/(?\w+)\/(?\d{4}-\d{4})\/QA(?:\/(?\d+))?(?:\?page=(?\d+))?$/; const match = url.match(regex); if (!match?.groups) { throw Error(`${url} in unrecognized format.`); @@ -45,15 +45,15 @@ export const parseQnaUrlWithPage = (url: QnaPageUrl): QnaUrlPageParams => { export const buildHomeQnaUrl = (params: BaseQnaUrlParams): QnaHomeUrl => { const { program, season } = params; - return `https://www.robotevents.com/${program}/${season}/QA`; + return `https://events.vex.com/${program}/${season}/QA`; }; export const buildQnaUrlWithId = (params: QnaUrlIdParams): QnaIdUrl => { const { program, season, id } = params; - return `https://www.robotevents.com/${program}/${season}/QA/${id}`; + return `https://events.vex.com/${program}/${season}/QA/${id}`; }; export const buildQnaUrlWithPage = (params: QnaUrlPageParams): QnaPageUrl => { const { program, season, page } = params; - return `https://www.robotevents.com/${program}/${season}/QA?page=${page}`; + return `https://events.vex.com/${program}/${season}/QA?page=${page}`; }; diff --git a/packages/scraper/tests/data/judging_home_page.ts b/packages/scraper/tests/data/judging_home_page.ts index d1fdce1..908fd5d 100644 --- a/packages/scraper/tests/data/judging_home_page.ts +++ b/packages/scraper/tests/data/judging_home_page.ts @@ -2,7 +2,7 @@ import type { ScrapedPage } from "../../src/modules/extractors"; import type { QnaHomeUrl } from "../../src/modules/parsing"; export default { - url: "https://www.robotevents.com/judging/2023-2024/QA", + url: "https://events.vex.com/judging/2023-2024/QA", html: ` @@ -44,14 +44,14 @@ export default { - +
@@ -67,7 +67,7 @@ export default {
@@ -76,8 +76,8 @@ export default {
- Ask a Question @@ -396,7 +396,7 @@ export default {

-
@@ -404,7 +404,7 @@ export default { 5 - @@ -419,7 +419,7 @@ export default {

- + Excellence Eligibility criteria @@ -473,7 +473,7 @@ export default {


- + Optional Materials During Interview @@ -500,7 +500,7 @@ export default {


- + Excellence Award Minimum Criteria within VEXU and small blended Tournaments @@ -546,20 +546,20 @@ export default { 1519 INTERSTATE HIGHWAY 30 W
Greenville, TX 75402
© 2023 Robotics Education & Competition Foundation.
All rights reserved. All other names/trademarks are the property of their respective owners.
- Privacy Policy + Privacy Policy | - Terms of Service + Terms of Service | - Acceptable Use Policy + Acceptable Use Policy | - Refund Policy
- State Nonprofit Disclosures
+ Refund Policy
+ State Nonprofit Disclosures
@@ -567,8 +567,8 @@ export default { @@ -586,7 +586,7 @@ export default { - + `, } satisfies ScrapedPage; diff --git a/packages/scraper/tests/data/test_questions.ts b/packages/scraper/tests/data/test_questions.ts index fe5d3d8..81e770b 100644 --- a/packages/scraper/tests/data/test_questions.ts +++ b/packages/scraper/tests/data/test_questions.ts @@ -6,7 +6,7 @@ export const TEST_SEASON = "2024-2025"; export const OLDEST_TEST_QUESTION_UNIQUE_DATES: Question = { id: "2059", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2059", + url: "https://events.vex.com/V5RC/2024-2025/QA/2059", program: "V5RC", season: "2024-2025", author: "", @@ -26,7 +26,7 @@ export const OLDEST_TEST_QUESTION_UNIQUE_DATES: Question = { export const TEST_QUESTIONS_UNIQUE_DATES: Question[] = [ { id: "2057", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2057", + url: "https://events.vex.com/V5RC/2024-2025/QA/2057", program: "V5RC", season: "2024-2025", author: "", @@ -44,7 +44,7 @@ export const TEST_QUESTIONS_UNIQUE_DATES: Question[] = [ }, { id: "2058", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2058", + url: "https://events.vex.com/V5RC/2024-2025/QA/2058", program: "V5RC", season: "2024-2025", author: "", @@ -62,7 +62,7 @@ export const TEST_QUESTIONS_UNIQUE_DATES: Question[] = [ }, { id: "2059", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2059", + url: "https://events.vex.com/V5RC/2024-2025/QA/2059", program: "V5RC", season: "2024-2025", author: "", @@ -80,7 +80,7 @@ export const TEST_QUESTIONS_UNIQUE_DATES: Question[] = [ }, { id: "2060", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2060", + url: "https://events.vex.com/V5RC/2024-2025/QA/2060", program: "V5RC", season: "2024-2025", author: "", @@ -98,7 +98,7 @@ export const TEST_QUESTIONS_UNIQUE_DATES: Question[] = [ }, { id: "2061", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2061", + url: "https://events.vex.com/V5RC/2024-2025/QA/2061", program: "V5RC", season: "2024-2025", author: "", @@ -116,7 +116,7 @@ export const TEST_QUESTIONS_UNIQUE_DATES: Question[] = [ }, { id: "2062", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2062", + url: "https://events.vex.com/V5RC/2024-2025/QA/2062", program: "V5RC", season: "2024-2025", author: "", @@ -134,7 +134,7 @@ export const TEST_QUESTIONS_UNIQUE_DATES: Question[] = [ }, { id: "2063", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2063", + url: "https://events.vex.com/V5RC/2024-2025/QA/2063", program: "V5RC", season: "2024-2025", author: "", @@ -154,7 +154,7 @@ export const TEST_QUESTIONS_UNIQUE_DATES: Question[] = [ export const OLDEST_TEST_QUESTIONS_OVERLAPPING_DATES: Question = { id: "2058", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2058", + url: "https://events.vex.com/V5RC/2024-2025/QA/2058", program: "V5RC", season: "2024-2025", author: "", @@ -174,7 +174,7 @@ export const OLDEST_TEST_QUESTIONS_OVERLAPPING_DATES: Question = { export const TEST_QUESTIONS_OVERLAPPING_DATES: Question[] = [ { id: "2057", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2057", + url: "https://events.vex.com/V5RC/2024-2025/QA/2057", program: "V5RC", season: "2024-2025", author: "", @@ -192,7 +192,7 @@ export const TEST_QUESTIONS_OVERLAPPING_DATES: Question[] = [ }, { id: "2060", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2060", + url: "https://events.vex.com/V5RC/2024-2025/QA/2060", program: "V5RC", season: "2024-2025", author: "", @@ -210,7 +210,7 @@ export const TEST_QUESTIONS_OVERLAPPING_DATES: Question[] = [ }, { id: "2061", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2061", + url: "https://events.vex.com/V5RC/2024-2025/QA/2061", program: "V5RC", season: "2024-2025", author: "", @@ -228,7 +228,7 @@ export const TEST_QUESTIONS_OVERLAPPING_DATES: Question[] = [ }, { id: "2059", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2059", + url: "https://events.vex.com/V5RC/2024-2025/QA/2059", program: "V5RC", season: "2024-2025", author: "", @@ -246,7 +246,7 @@ export const TEST_QUESTIONS_OVERLAPPING_DATES: Question[] = [ }, { id: "2058", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2058", + url: "https://events.vex.com/V5RC/2024-2025/QA/2058", program: "V5RC", season: "2024-2025", author: "", @@ -264,7 +264,7 @@ export const TEST_QUESTIONS_OVERLAPPING_DATES: Question[] = [ }, { id: "2062", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2062", + url: "https://events.vex.com/V5RC/2024-2025/QA/2062", program: "V5RC", season: "2024-2025", author: "", @@ -282,7 +282,7 @@ export const TEST_QUESTIONS_OVERLAPPING_DATES: Question[] = [ }, { id: "2063", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2063", + url: "https://events.vex.com/V5RC/2024-2025/QA/2063", program: "V5RC", season: "2024-2025", author: "", @@ -302,7 +302,7 @@ export const TEST_QUESTIONS_OVERLAPPING_DATES: Question[] = [ export const OLDEST_TEST_QUESTIONS_MIXED_OVERLAPPING_DATES: Question = { id: "2061", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2061", + url: "https://events.vex.com/V5RC/2024-2025/QA/2061", program: "V5RC", season: "2024-2025", author: "", @@ -322,7 +322,7 @@ export const OLDEST_TEST_QUESTIONS_MIXED_OVERLAPPING_DATES: Question = { export const TEST_QUESTIONS_MIXED_OVERLAPPING_DATES: Question[] = [ { id: "2059", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2059", + url: "https://events.vex.com/V5RC/2024-2025/QA/2059", program: "V5RC", season: "2024-2025", author: "", @@ -340,7 +340,7 @@ export const TEST_QUESTIONS_MIXED_OVERLAPPING_DATES: Question[] = [ }, { id: "2058", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2058", + url: "https://events.vex.com/V5RC/2024-2025/QA/2058", program: "V5RC", season: "2024-2025", author: "", @@ -358,7 +358,7 @@ export const TEST_QUESTIONS_MIXED_OVERLAPPING_DATES: Question[] = [ }, { id: "2063", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2063", + url: "https://events.vex.com/V5RC/2024-2025/QA/2063", program: "V5RC", season: "2024-2025", author: "", @@ -376,7 +376,7 @@ export const TEST_QUESTIONS_MIXED_OVERLAPPING_DATES: Question[] = [ }, { id: "2060", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2060", + url: "https://events.vex.com/V5RC/2024-2025/QA/2060", program: "V5RC", season: "2024-2025", author: "", @@ -394,7 +394,7 @@ export const TEST_QUESTIONS_MIXED_OVERLAPPING_DATES: Question[] = [ }, { id: "2062", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2062", + url: "https://events.vex.com/V5RC/2024-2025/QA/2062", program: "V5RC", season: "2024-2025", author: "", @@ -412,7 +412,7 @@ export const TEST_QUESTIONS_MIXED_OVERLAPPING_DATES: Question[] = [ }, { id: "2057", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2057", + url: "https://events.vex.com/V5RC/2024-2025/QA/2057", program: "V5RC", season: "2024-2025", author: "", @@ -430,7 +430,7 @@ export const TEST_QUESTIONS_MIXED_OVERLAPPING_DATES: Question[] = [ }, { id: "2061", - url: "https://www.robotevents.com/V5RC/2024-2025/QA/2061", + url: "https://events.vex.com/V5RC/2024-2025/QA/2061", program: "V5RC", season: "2024-2025", author: "", diff --git a/packages/scraper/tests/data/v5rc_home_page.ts b/packages/scraper/tests/data/v5rc_home_page.ts index 61fc06d..44d8005 100644 --- a/packages/scraper/tests/data/v5rc_home_page.ts +++ b/packages/scraper/tests/data/v5rc_home_page.ts @@ -2,7 +2,7 @@ import type { ScrapedPage } from "../../src/modules/extractors"; import type { QnaHomeUrl } from "../../src/modules/parsing"; export default { - url: "https://www.robotevents.com/V5RC/2023-2024/QA", + url: "https://events.vex.com/V5RC/2023-2024/QA", html: ` @@ -30,7 +30,7 @@ export default { } }; - @@ -41,7 +41,7 @@ export default { body:before {background:url("/images/bg/bg2.svg") repeat;display:block;content:"";position:absolute;z-index:-1;top:165px;left:0;right:0;height:80%;opacity:0.75} - + - + - + - +