From 77af40379cd956aa8c66252521110267ed5eb9e3 Mon Sep 17 00:00:00 2001 From: ujiro99 Date: Sun, 13 Sep 2026 14:31:50 +0900 Subject: [PATCH 1/3] Add: A/B test framework with hub-hosted allocation config Adds the experiment plumbing for the onboarding A/B test (#455): - packages/hub/public/data/experiments.json serves the allocation ratio, so the split can be changed - or the test stopped - by deploying the hub, with no Chrome Web Store release. - services/experiments assigns a variant once per install and persists it in chrome.storage.local. On a fetch failure it still assigns, using the build-time allocation, and records config_source so a hub outage can be separated out during analysis. - The assignment runs in onInstalled right before the onboarding tab is created, so the page renders its first frame from storage. - Every onboarding_* event now carries a `variant` param. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VeTTSrCxM9ifQatx6XjCxo --- AGENTS.md | 5 +- packages/extension/src/background_script.ts | 10 ++ .../onboarding/onboardingAnalytics.test.ts | 44 +++++- .../onboarding/onboardingAnalytics.ts | 18 ++- .../experiments/experimentConfig.test.ts | 90 ++++++++++++ .../services/experiments/experimentConfig.ts | 91 ++++++++++++ .../src/services/experiments/index.ts | 3 + .../experiments/onboardingExperiment.test.ts | 129 ++++++++++++++++++ .../experiments/onboardingExperiment.ts | 97 +++++++++++++ .../src/services/experiments/types.ts | 33 +++++ .../extension/src/services/storage/const.ts | 1 + .../extension/src/services/storage/index.ts | 1 + packages/hub/AGENTS.md | 22 ++- packages/hub/public/data/experiments.json | 6 + 14 files changed, 539 insertions(+), 11 deletions(-) create mode 100644 packages/extension/src/services/experiments/experimentConfig.test.ts create mode 100644 packages/extension/src/services/experiments/experimentConfig.ts create mode 100644 packages/extension/src/services/experiments/index.ts create mode 100644 packages/extension/src/services/experiments/onboardingExperiment.test.ts create mode 100644 packages/extension/src/services/experiments/onboardingExperiment.ts create mode 100644 packages/extension/src/services/experiments/types.ts create mode 100644 packages/hub/public/data/experiments.json diff --git a/AGENTS.md b/AGENTS.md index adb5b0f3..87878202 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -118,7 +118,8 @@ yarn dev # watch モード 1. **型システム共有**: SharedパッケージでBaseCommandなどの基本型を定義し、ExtensionとHubで拡張 2. **AIサービス定義**: `packages/hub/public/data/ai-services.json` を Extension がビルド時/実行時の両方で参照 -3. **e2eテスト**: `packages/hub` がデプロイする `/en/test` ページを Extension の Playwright テストが利用 +3. **ABテスト配分設定**: `packages/hub/public/data/experiments.json` を Extension が実行時に参照し、オンボーディング等のABテストの配分比率を制御 +4. **e2eテスト**: `packages/hub` がデプロイする `/en/test` ページを Extension の Playwright テストが利用 ### Chrome拡張機能の構造 (packages/extension) @@ -188,7 +189,7 @@ interface PageActionOption { **Hub開発:** - Hub は縮小版のため新機能は追加しない。コマンド共有プラットフォームとしての機能拡張は新リポジトリ（selection-command-hub）側で行う -- `ai-services.json` とテストページの変更時は Extension 側への影響を確認すること +- `ai-services.json` / `experiments.json` とテストページの変更時は Extension 側への影響を確認すること **テスト:** diff --git a/packages/extension/src/background_script.ts b/packages/extension/src/background_script.ts index 2f0c52b1..b704f655 100644 --- a/packages/extension/src/background_script.ts +++ b/packages/extension/src/background_script.ts @@ -29,6 +29,7 @@ import { getOrCreateClientId, } from "@/services/analytics" import * as HubBackground from "@/services/hub/background" +import { ensureOnboardingAssignment } from "@/services/experiments" import { importIf } from "@import-if" importIf("production", "./lib/sentry/initialize") @@ -471,6 +472,15 @@ chrome.runtime.onInstalled.addListener(async (details) => { if (details.reason === chrome.runtime.OnInstalledReason.INSTALL) { await Settings.reset() sendEvent(ANALYTICS_EVENTS.INSTALLED, {}, SCREEN.SERVICE_WORKER) + // Assign the onboarding A/B variant before the tab is created, so the + // page can render its first frame from storage without fetching the + // remote config itself. A failure here must never block onboarding - + // the page assigns on its own if no assignment is stored yet. + try { + await ensureOnboardingAssignment() + } catch (error) { + console.error("Failed to assign onboarding variant:", error) + } chrome.tabs.create({ url: ONBOARDING_PAGE_PATH }) } diff --git a/packages/extension/src/components/onboarding/onboardingAnalytics.test.ts b/packages/extension/src/components/onboarding/onboardingAnalytics.test.ts index c6257baf..3c147262 100644 --- a/packages/extension/src/components/onboarding/onboardingAnalytics.test.ts +++ b/packages/extension/src/components/onboarding/onboardingAnalytics.test.ts @@ -1,6 +1,10 @@ -import { describe, it, expect, vi } from "vitest" +import { describe, it, expect, vi, beforeEach } from "vitest" import { OnboardingStep } from "@/types/onboarding" import { SCREEN } from "@/const" +import { + overrideOnboardingAssignment, + resetOnboardingAssignmentCache, +} from "@/services/experiments" const sendEvent = vi.fn() vi.mock("@/services/analytics", async () => { @@ -11,10 +15,16 @@ vi.mock("@/services/analytics", async () => { } }) -const { sendOnboardingEvent } = await import("./onboardingAnalytics") +const { sendOnboardingEvent, UNKNOWN_VARIANT } = + await import("./onboardingAnalytics") const { ANALYTICS_EVENTS } = await import("@/services/analytics") describe("sendOnboardingEvent", () => { + beforeEach(() => { + sendEvent.mockClear() + resetOnboardingAssignmentCache() + }) + it("normalizes a numeric OnboardingStep param to its stable string label", () => { sendOnboardingEvent(ANALYTICS_EVENTS.ONBOARDING_VALUE_REACHED, { step: OnboardingStep.AI_PROMPT, @@ -22,7 +32,7 @@ describe("sendOnboardingEvent", () => { expect(sendEvent).toHaveBeenCalledWith( ANALYTICS_EVENTS.ONBOARDING_VALUE_REACHED, - { step: "ai_prompt" }, + { step: "ai_prompt", variant: UNKNOWN_VARIANT }, SCREEN.ONBOARDING, ) }) @@ -34,7 +44,33 @@ describe("sendOnboardingEvent", () => { expect(sendEvent).toHaveBeenCalledWith( ANALYTICS_EVENTS.ONBOARDING_START, - { locale: "en" }, + { locale: "en", variant: UNKNOWN_VARIANT }, + SCREEN.ONBOARDING, + ) + }) + + it("tags the event with the assigned A/B variant", () => { + overrideOnboardingAssignment("B") + + sendOnboardingEvent(ANALYTICS_EVENTS.ONBOARDING_SKIP, { + step: OnboardingStep.SEARCH, + }) + + expect(sendEvent).toHaveBeenCalledWith( + ANALYTICS_EVENTS.ONBOARDING_SKIP, + { step: "search", variant: "B" }, + SCREEN.ONBOARDING, + ) + }) + + it("falls back to the unknown variant when no assignment is resolved yet", () => { + sendOnboardingEvent(ANALYTICS_EVENTS.ONBOARDING_COMPLETE, { + completion_time_sec: 12.3, + }) + + expect(sendEvent).toHaveBeenCalledWith( + ANALYTICS_EVENTS.ONBOARDING_COMPLETE, + { completion_time_sec: 12.3, variant: "unknown" }, SCREEN.ONBOARDING, ) }) diff --git a/packages/extension/src/components/onboarding/onboardingAnalytics.ts b/packages/extension/src/components/onboarding/onboardingAnalytics.ts index 1c60af23..f161551c 100644 --- a/packages/extension/src/components/onboarding/onboardingAnalytics.ts +++ b/packages/extension/src/components/onboarding/onboardingAnalytics.ts @@ -1,6 +1,7 @@ import { sendEvent, ANALYTICS_EVENTS } from "@/services/analytics" import { SCREEN } from "@/const" import { OnboardingStep } from "@/types/onboarding" +import { getOnboardingVariantSync } from "@/services/experiments" // Stable string labels for GA4's `step` param, so recorded events keep their // meaning even if OnboardingStep's numeric enum values ever change. @@ -13,10 +14,16 @@ const STEP_LABELS: Record = { [OnboardingStep.COMPLETE]: "complete", } +// Reported as the `variant` param when an event somehow fires before the +// A/B assignment has been resolved - those events stay in the funnel but +// can be excluded from the comparison. +export const UNKNOWN_VARIANT = "unknown" + // Thin wrapper so step components never call sendEvent directly - keeps the // SCREEN.ONBOARDING tag and event names centralized in one place. Also // normalizes a `step` param (passed as the OnboardingStep enum) to its -// stable string label before sending. +// stable string label, and tags every onboarding_* event with the A/B +// variant so the whole funnel can be compared arm by arm. export function sendOnboardingEvent( name: (typeof ANALYTICS_EVENTS)[keyof typeof ANALYTICS_EVENTS], // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -26,5 +33,12 @@ export function sendOnboardingEvent( params.step !== undefined ? { ...params, step: STEP_LABELS[params.step as OnboardingStep] } : params - sendEvent(name, normalizedParams, SCREEN.ONBOARDING) + sendEvent( + name, + { + ...normalizedParams, + variant: getOnboardingVariantSync() ?? UNKNOWN_VARIANT, + }, + SCREEN.ONBOARDING, + ) } diff --git a/packages/extension/src/services/experiments/experimentConfig.test.ts b/packages/extension/src/services/experiments/experimentConfig.test.ts new file mode 100644 index 00000000..25e53b37 --- /dev/null +++ b/packages/extension/src/services/experiments/experimentConfig.test.ts @@ -0,0 +1,90 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest" +import { + normalizeConfig, + getDefaultConfig, + fetchExperimentConfig, +} from "./experimentConfig" + +const EXPERIMENT_ID = "onboarding_v2" + +const mockFetchOnce = (body: unknown, ok = true, status = 200) => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue({ + ok, + status, + json: async () => body, + }), + ) +} + +describe("normalizeConfig", () => { + it("keeps a well-formed config as-is", () => { + expect( + normalizeConfig({ enabled: true, allocation: 0.3 }, EXPERIMENT_ID), + ).toEqual({ enabled: true, allocation: 0.3 }) + }) + + it("clamps an out-of-range allocation into 0..1", () => { + expect( + normalizeConfig({ enabled: true, allocation: 3 }, EXPERIMENT_ID), + ).toEqual({ enabled: true, allocation: 1 }) + expect( + normalizeConfig({ enabled: true, allocation: -1 }, EXPERIMENT_ID), + ).toEqual({ enabled: true, allocation: 0 }) + }) + + it("falls back field by field for a malformed payload", () => { + const fallback = getDefaultConfig(EXPERIMENT_ID) + expect( + normalizeConfig({ enabled: "yes", allocation: "half" }, EXPERIMENT_ID), + ).toEqual(fallback) + expect(normalizeConfig(null, EXPERIMENT_ID)).toEqual(fallback) + }) +}) + +describe("fetchExperimentConfig", () => { + beforeEach(() => { + vi.spyOn(console, "warn").mockImplementation(() => {}) + }) + afterEach(() => { + vi.unstubAllGlobals() + vi.restoreAllMocks() + }) + + it("returns the remote config for the requested experiment", async () => { + mockFetchOnce({ [EXPERIMENT_ID]: { enabled: true, allocation: 0.25 } }) + + await expect(fetchExperimentConfig(EXPERIMENT_ID)).resolves.toEqual({ + config: { enabled: true, allocation: 0.25 }, + source: "remote", + }) + }) + + it("treats a removed experiment as finished", async () => { + mockFetchOnce({ other_experiment: { enabled: true, allocation: 1 } }) + + await expect(fetchExperimentConfig(EXPERIMENT_ID)).resolves.toEqual({ + config: { enabled: false, allocation: 0 }, + source: "remote", + }) + }) + + it("falls back to the build-time config on an HTTP error", async () => { + mockFetchOnce({}, false, 503) + + await expect(fetchExperimentConfig(EXPERIMENT_ID)).resolves.toEqual({ + config: getDefaultConfig(EXPERIMENT_ID), + source: "fallback", + }) + }) + + it("falls back to the build-time config when the request fails", async () => { + vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("offline"))) + + await expect(fetchExperimentConfig(EXPERIMENT_ID)).resolves.toEqual({ + config: getDefaultConfig(EXPERIMENT_ID), + source: "fallback", + }) + }) +}) diff --git a/packages/extension/src/services/experiments/experimentConfig.ts b/packages/extension/src/services/experiments/experimentConfig.ts new file mode 100644 index 00000000..cad60ae4 --- /dev/null +++ b/packages/extension/src/services/experiments/experimentConfig.ts @@ -0,0 +1,91 @@ +import { HUB_URL } from "@/const" +import type { ExperimentConfig, ExperimentConfigSource } from "./types" + +/** External endpoint serving the experiment allocation config. */ +const EXPERIMENTS_URL = `${HUB_URL}/data/experiments.json` + +/** + * Hard timeout for the config fetch. The onboarding tab opens right after + * this resolves, so a slow hub must never keep the user staring at a blank + * tab - we fall back to the build-time defaults instead. + */ +const FETCH_TIMEOUT_MS = 3000 + +/** + * Build-time defaults, used when the hub is unreachable or returns something + * unusable. Assignment still happens (rather than forcing everyone to the + * control), so a hub outage doesn't skew one arm towards users who happened + * to have connectivity problems at install time. + */ +export const DEFAULT_CONFIGS: Record = { + onboarding_v2: { enabled: true, allocation: 0.5 }, +} + +const DEFAULT_CONFIG: ExperimentConfig = { enabled: false, allocation: 0 } + +export const getDefaultConfig = (experimentId: string): ExperimentConfig => + DEFAULT_CONFIGS[experimentId] ?? DEFAULT_CONFIG + +const clamp01 = (value: number): number => Math.min(1, Math.max(0, value)) + +/** + * Coerce one entry of the remote JSON into a usable config, falling back + * field by field so a partially broken payload still yields safe values. + */ +export const normalizeConfig = ( + raw: unknown, + experimentId: string, +): ExperimentConfig => { + const fallback = getDefaultConfig(experimentId) + if (raw == null || typeof raw !== "object") return fallback + + const { enabled, allocation } = raw as Partial + return { + enabled: typeof enabled === "boolean" ? enabled : fallback.enabled, + allocation: + typeof allocation === "number" && Number.isFinite(allocation) + ? clamp01(allocation) + : fallback.allocation, + } +} + +export type FetchedConfig = { + config: ExperimentConfig + source: ExperimentConfigSource +} + +/** + * Fetch the allocation config for a single experiment. + * Deliberately uncached: an experiment assignment happens once per install, + * so there is nothing to amortize, and the ratio must be changeable from the + * hub without waiting for a cache to expire. + */ +export const fetchExperimentConfig = async ( + experimentId: string, +): Promise => { + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS) + try { + const res = await fetch(EXPERIMENTS_URL, { signal: controller.signal }) + if (!res.ok) { + throw new Error( + `Failed to fetch experiments from ${EXPERIMENTS_URL}: HTTP ${res.status}`, + ) + } + const raw = await res.json() + if (raw == null || typeof raw !== "object") { + throw new Error(`Unexpected experiments response from ${EXPERIMENTS_URL}`) + } + const entry = (raw as Record)[experimentId] + if (entry === undefined) { + // The experiment was removed from the config: treat it as finished. + return { config: { enabled: false, allocation: 0 }, source: "remote" } + } + return { config: normalizeConfig(entry, experimentId), source: "remote" } + } catch (error) { + console.warn("Failed to fetch experiment config:", error) + return { config: getDefaultConfig(experimentId), source: "fallback" } + } finally { + clearTimeout(timer) + } +} diff --git a/packages/extension/src/services/experiments/index.ts b/packages/extension/src/services/experiments/index.ts new file mode 100644 index 00000000..10766082 --- /dev/null +++ b/packages/extension/src/services/experiments/index.ts @@ -0,0 +1,3 @@ +export * from "./types" +export * from "./experimentConfig" +export * from "./onboardingExperiment" diff --git a/packages/extension/src/services/experiments/onboardingExperiment.test.ts b/packages/extension/src/services/experiments/onboardingExperiment.test.ts new file mode 100644 index 00000000..f409db85 --- /dev/null +++ b/packages/extension/src/services/experiments/onboardingExperiment.test.ts @@ -0,0 +1,129 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest" +import { Storage, LOCAL_STORAGE_KEY } from "@/services/storage" +import { setupStorageMocks } from "@/test/setup" +import type { Experiments } from "./types" + +const fetchExperimentConfig = vi.fn() +vi.mock("./experimentConfig", async () => { + const actual = + await vi.importActual( + "./experimentConfig", + ) + return { + ...actual, + fetchExperimentConfig: (...args: unknown[]) => + fetchExperimentConfig(...args), + } +}) + +const { + ensureOnboardingAssignment, + getOnboardingVariantSync, + resetOnboardingAssignmentCache, + ONBOARDING_EXPERIMENT_ID, +} = await import("./onboardingExperiment") + +const readStored = () => Storage.get(LOCAL_STORAGE_KEY.EXPERIMENTS) + +describe("ensureOnboardingAssignment", () => { + beforeEach(async () => { + setupStorageMocks("realistic") + resetOnboardingAssignmentCache() + fetchExperimentConfig.mockReset() + await Storage.set(LOCAL_STORAGE_KEY.EXPERIMENTS, {}) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + it("assigns variant B when the random draw lands below the allocation", async () => { + fetchExperimentConfig.mockResolvedValue({ + config: { enabled: true, allocation: 0.5 }, + source: "remote", + }) + vi.spyOn(Math, "random").mockReturnValue(0.49) + + const assignment = await ensureOnboardingAssignment() + + expect(assignment.variant).toBe("B") + expect(assignment.configSource).toBe("remote") + expect(getOnboardingVariantSync()).toBe("B") + expect((await readStored())[ONBOARDING_EXPERIMENT_ID].variant).toBe("B") + }) + + it("assigns the control when the draw lands above the allocation", async () => { + fetchExperimentConfig.mockResolvedValue({ + config: { enabled: true, allocation: 0.5 }, + source: "remote", + }) + vi.spyOn(Math, "random").mockReturnValue(0.5) + + expect((await ensureOnboardingAssignment()).variant).toBe("A") + }) + + it("assigns everyone to the control while the experiment is disabled", async () => { + fetchExperimentConfig.mockResolvedValue({ + config: { enabled: false, allocation: 1 }, + source: "remote", + }) + vi.spyOn(Math, "random").mockReturnValue(0) + + expect((await ensureOnboardingAssignment()).variant).toBe("A") + }) + + it("still assigns randomly when the config could not be fetched", async () => { + fetchExperimentConfig.mockResolvedValue({ + config: { enabled: true, allocation: 0.5 }, + source: "fallback", + }) + vi.spyOn(Math, "random").mockReturnValue(0.1) + + const assignment = await ensureOnboardingAssignment() + + expect(assignment.variant).toBe("B") + expect(assignment.configSource).toBe("fallback") + }) + + it("reuses a stored assignment instead of drawing again", async () => { + fetchExperimentConfig.mockResolvedValue({ + config: { enabled: true, allocation: 1 }, + source: "remote", + }) + await Storage.set(LOCAL_STORAGE_KEY.EXPERIMENTS, { + [ONBOARDING_EXPERIMENT_ID]: { + variant: "A", + allocation: 0.5, + configSource: "remote", + assignedAt: 1, + }, + }) + + const assignment = await ensureOnboardingAssignment() + + expect(assignment.variant).toBe("A") + expect(assignment.assignedAt).toBe(1) + expect(fetchExperimentConfig).not.toHaveBeenCalled() + }) + + it("keeps other experiments' assignments when persisting a new one", async () => { + fetchExperimentConfig.mockResolvedValue({ + config: { enabled: true, allocation: 1 }, + source: "remote", + }) + await Storage.set(LOCAL_STORAGE_KEY.EXPERIMENTS, { + other_experiment: { + variant: "B", + allocation: 1, + configSource: "remote", + assignedAt: 1, + }, + }) + + await ensureOnboardingAssignment() + + const stored = await readStored() + expect(stored.other_experiment.variant).toBe("B") + expect(stored[ONBOARDING_EXPERIMENT_ID].variant).toBe("B") + }) +}) diff --git a/packages/extension/src/services/experiments/onboardingExperiment.ts b/packages/extension/src/services/experiments/onboardingExperiment.ts new file mode 100644 index 00000000..6e59ef60 --- /dev/null +++ b/packages/extension/src/services/experiments/onboardingExperiment.ts @@ -0,0 +1,97 @@ +import { Storage, LOCAL_STORAGE_KEY } from "@/services/storage" +import { fetchExperimentConfig } from "./experimentConfig" +import type { + ExperimentAssignment, + ExperimentVariant, + Experiments, +} from "./types" + +/** Experiment id reported to GA4 as `experiment_id`. */ +export const ONBOARDING_EXPERIMENT_ID = "onboarding_v2" + +/** + * Last resolved assignment, kept in module memory so analytics helpers can + * read the variant synchronously (chrome.storage is async, and the event + * wrapper must not become async for every caller). + */ +let cachedAssignment: ExperimentAssignment | null = null + +export const getOnboardingAssignmentSync = (): ExperimentAssignment | null => + cachedAssignment + +export const getOnboardingVariantSync = (): ExperimentVariant | null => + cachedAssignment?.variant ?? null + +/** + * dev/e2e only: force a variant for the rest of this page's lifetime without + * fetching or persisting anything, so `?variant=B` can be used to review a + * screen directly. See useOnboardingVariant(). + */ +export const overrideOnboardingAssignment = ( + variant: ExperimentVariant, +): ExperimentAssignment => { + cachedAssignment = { + variant, + allocation: variant === "B" ? 1 : 0, + configSource: "fallback", + assignedAt: Date.now(), + } + return cachedAssignment +} + +/** Test-only: drop the in-memory cache between cases. */ +export const resetOnboardingAssignmentCache = () => { + cachedAssignment = null +} + +const readAssignment = async (): Promise => { + const experiments = await Storage.get( + LOCAL_STORAGE_KEY.EXPERIMENTS, + ) + return experiments?.[ONBOARDING_EXPERIMENT_ID] ?? null +} + +/** + * Resolve this user's onboarding variant, assigning (and persisting) one on + * first call. Normally called from the background script's onInstalled + * handler before the onboarding tab is opened, so the page itself only has + * to read the stored value; calling it again is a no-op that returns the + * same assignment, which keeps every onboarding_* event of a given user on + * one variant even if the page is reopened. + */ +export const ensureOnboardingAssignment = + async (): Promise => { + const existing = await readAssignment() + if (existing) { + cachedAssignment = existing + return existing + } + + const { config, source } = await fetchExperimentConfig( + ONBOARDING_EXPERIMENT_ID, + ) + const variant: ExperimentVariant = + config.enabled && Math.random() < config.allocation ? "B" : "A" + const assignment: ExperimentAssignment = { + variant, + allocation: config.allocation, + configSource: source, + assignedAt: Date.now(), + } + + // Re-read inside the update so a concurrent assignment (e.g. the + // onboarding page racing the background script) wins consistently + // instead of being overwritten with a second coin flip. + let stored = assignment + await Storage.update(LOCAL_STORAGE_KEY.EXPERIMENTS, (cur) => { + const current = cur?.[ONBOARDING_EXPERIMENT_ID] + if (current) { + stored = current + return cur + } + return { ...cur, [ONBOARDING_EXPERIMENT_ID]: assignment } + }) + + cachedAssignment = stored + return stored + } diff --git a/packages/extension/src/services/experiments/types.ts b/packages/extension/src/services/experiments/types.ts new file mode 100644 index 00000000..96cf665d --- /dev/null +++ b/packages/extension/src/services/experiments/types.ts @@ -0,0 +1,33 @@ +/** + * Variant a user is assigned to for an A/B test. + * "A" is always the control (the behavior that shipped before the test). + */ +export type ExperimentVariant = "A" | "B" + +/** Remote configuration of a single experiment, served by the hub. */ +export type ExperimentConfig = { + /** Kill switch: when false, everyone is assigned to the control. */ + enabled: boolean + /** Ratio of users assigned to variant B, clamped to 0..1. */ + allocation: number +} + +/** + * Whether the config that drove an assignment came from the hub or from the + * build-time defaults (hub unreachable / malformed response). Reported to + * analytics so a failure spike can be separated out during analysis. + */ +export type ExperimentConfigSource = "remote" | "fallback" + +/** A user's assignment for one experiment, persisted in chrome.storage.local. */ +export type ExperimentAssignment = { + variant: ExperimentVariant + /** The allocation in effect when the assignment was made. */ + allocation: number + configSource: ExperimentConfigSource + /** Unix ms, for debugging and for expiring stale experiments later. */ + assignedAt: number +} + +/** Shape of the LOCAL_STORAGE_KEY.EXPERIMENTS record, keyed by experiment id. */ +export type Experiments = Record diff --git a/packages/extension/src/services/storage/const.ts b/packages/extension/src/services/storage/const.ts index 2a59c8bb..368596d1 100644 --- a/packages/extension/src/services/storage/const.ts +++ b/packages/extension/src/services/storage/const.ts @@ -19,6 +19,7 @@ export enum LOCAL_STORAGE_KEY { HUB_USER = "hubUser", HUB_SHARED_AT = "hubSharedAt", HUB_REGISTERED = "hubRegistered", + EXPERIMENTS = "experiments", } export enum SESSION_STORAGE_KEY { diff --git a/packages/extension/src/services/storage/index.ts b/packages/extension/src/services/storage/index.ts index 1e3c4e87..b8e9b6a8 100644 --- a/packages/extension/src/services/storage/index.ts +++ b/packages/extension/src/services/storage/index.ts @@ -77,6 +77,7 @@ const DEFAULTS = { [LOCAL_STORAGE_KEY.GLOBAL_COMMAND_METADATA]: null, [LOCAL_STORAGE_KEY.HUB_USER]: null, [LOCAL_STORAGE_KEY.HUB_REGISTERED]: false, + [LOCAL_STORAGE_KEY.EXPERIMENTS]: {}, [SESSION_STORAGE_KEY.BG]: {}, [SESSION_STORAGE_KEY.SESSION_DATA]: null, [SESSION_STORAGE_KEY.MESSAGE_QUEUE]: [], diff --git a/packages/hub/AGENTS.md b/packages/hub/AGENTS.md index 4f8b7335..91bd1369 100644 --- a/packages/hub/AGENTS.md +++ b/packages/hub/AGENTS.md @@ -18,7 +18,7 @@ This file provides guidance to AI Agent when working with code in this repositor （Next.js 15 フル機能アプリ）でしたが、そのアプリケーションは新しいリポジトリ （https://github.com/ujiro99/selection-command-hub、selection-command.com へデプロイ）に移行しました。 -このパッケージに残っているのは、以下の2つの責務のみです。 +このパッケージに残っているのは、以下の3つの責務のみです。 1. **ai-services.json の静的ホスティング** - `public/data/ai-services.json` を GitHub Pages で静的配信する @@ -30,7 +30,22 @@ This file provides guidance to AI Agent when working with code in this repositor - このファイルを変更・削除する際は、必ず Extension 側のこれら2箇所への 影響を確認すること -2. **e2e テスト用ページの配信** +2. **experiments.json の静的ホスティング（ABテスト配信制御）** + - `public/data/experiments.json` を GitHub Pages で静的配信する + - Extension が実行時に `${HUB_URL}/data/experiments.json` へ fetch し、 + ABテストの variant 配分比率を決定する + （`packages/extension/src/services/experiments/experimentConfig.ts`） + - このファイルを編集して hub をデプロイするだけで、Chrome Web Store への + 再申請なしに配分比率の変更・実験の停止（`enabled: false`）ができる + - 形式: + ```json + { + "": { "enabled": true, "allocation": 0.5 } + } + ``` + `allocation` は variant B に割り当てるユーザーの比率（0..1） + +3. **e2e テスト用ページの配信** - `src/app/[lang]/test/page.tsx`（`/en/test` など）は、Extension の Playwright e2e テストが実際にデプロイされたページにアクセスして 動作確認するためのテストページ @@ -49,7 +64,8 @@ This file provides guidance to AI Agent when working with code in this repositor ## プロジェクト構造（縮小後） -- `public/data/ai-services.json` - Extension が参照する唯一のデータファイル +- `public/data/ai-services.json` - Extension が参照するデータファイル +- `public/data/experiments.json` - Extension のABテスト配分設定 - `src/app/[lang]/test/` - e2e テストページ本体（`page.tsx`, `QuillWrapper.tsx`） - `src/app/[lang]/layout.tsx`, `src/app/layout.tsx` - App Router の必須レイアウト（Header/Footer/LanguageProvider を保持） - `src/features/locale/` - 14言語分の辞書ファイル（layout.tsx が diff --git a/packages/hub/public/data/experiments.json b/packages/hub/public/data/experiments.json new file mode 100644 index 00000000..ea07f503 --- /dev/null +++ b/packages/hub/public/data/experiments.json @@ -0,0 +1,6 @@ +{ + "onboarding_v2": { + "enabled": true, + "allocation": 0.5 + } +} From 97f5f52ed129f018f4539d85f406908f08a203fa Mon Sep 17 00:00:00 2001 From: ujiro99 Date: Sun, 13 Sep 2026 14:31:58 +0900 Subject: [PATCH 2/3] Add: Onboarding variant B - welcome overlay instead of the intro step The B arm of the onboarding A/B test (#455), aimed at the 60% drop-off on the intro screen: - Drops the INTRO step and opens directly on the search command step. - Shows the logo and a welcome message first, in a new WELCOME phase. It advances after 2s, or immediately on click - there is nothing to read there, so the timer should not be a second thing to sit through. - Enter and exit use a blur + fade transition (Linear-style), added as an `effect` option on OnboardingFadeIn so variant A keeps its current rise animation untouched. - `?variant=A|B` lands on either arm in dev/e2e builds, and the screenshot spec captures the welcome overlay. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VeTTSrCxM9ifQatx6XjCxo --- .../extension/e2e/onboarding-shots.spec.ts | 12 +++- .../public/_locales/de/messages.json | 6 ++ .../public/_locales/en/messages.json | 8 +++ .../public/_locales/es/messages.json | 6 ++ .../public/_locales/fr/messages.json | 6 ++ .../public/_locales/hi/messages.json | 6 ++ .../public/_locales/id/messages.json | 6 ++ .../public/_locales/it/messages.json | 6 ++ .../public/_locales/ja/messages.json | 6 ++ .../public/_locales/ko/messages.json | 6 ++ .../public/_locales/ms/messages.json | 6 ++ .../public/_locales/pt_BR/messages.json | 6 ++ .../public/_locales/pt_PT/messages.json | 6 ++ .../public/_locales/ru/messages.json | 6 ++ .../public/_locales/zh_CN/messages.json | 6 ++ .../onboarding/OnboardingFadeIn.tsx | 21 +++++- .../components/onboarding/OnboardingPage.tsx | 69 +++++++++++++----- .../onboarding/OnboardingWelcome.test.tsx | 59 +++++++++++++++ .../onboarding/OnboardingWelcome.tsx | 72 +++++++++++++++++++ .../onboarding/useOnboardingState.test.tsx | 41 +++++++++++ .../onboarding/useOnboardingState.ts | 32 +++++++-- .../onboarding/useOnboardingVariant.ts | 49 +++++++++++++ packages/extension/src/types/onboarding.ts | 6 +- packages/extension/tailwind.config.js | 18 +++++ 24 files changed, 435 insertions(+), 30 deletions(-) create mode 100644 packages/extension/src/components/onboarding/OnboardingWelcome.test.tsx create mode 100644 packages/extension/src/components/onboarding/OnboardingWelcome.tsx create mode 100644 packages/extension/src/components/onboarding/useOnboardingState.test.tsx create mode 100644 packages/extension/src/components/onboarding/useOnboardingVariant.ts diff --git a/packages/extension/e2e/onboarding-shots.spec.ts b/packages/extension/e2e/onboarding-shots.spec.ts index 3311533a..7c068a35 100644 --- a/packages/extension/e2e/onboarding-shots.spec.ts +++ b/packages/extension/e2e/onboarding-shots.spec.ts @@ -19,14 +19,20 @@ const pathToExtension = path.join(__dirname, "../dist") type Shot = { name: string - step: keyof typeof OnboardingStep + // Omitted for shots that should open on whichever screen the variant + // starts at (e.g. variant B's welcome overlay). + step?: keyof typeof OnboardingStep phase?: StepPhase + variant?: "A" | "B" } // One shot per visually distinct screen - not one per StepPhase transition; // e.g. Step2's WAIT_EXECUTE looks like Step1's, so only Step1 gets it. const SHOTS: Shot[] = [ { name: "00-intro", step: "INTRO" }, + // Variant B of the onboarding A/B test (see services/experiments): no + // INTRO step, a brief welcome overlay in front of the first step instead. + { name: "00b-welcome-variant-b", variant: "B" }, { name: "01-search-explain", step: "SEARCH", phase: StepPhase.EXPLAIN }, { name: "02-search-wait-execute", @@ -90,8 +96,10 @@ test.describe("onboarding screenshots (design review)", () => { fs.mkdirSync(outDir, { recursive: true }) for (const shot of SHOTS) { - const params = new URLSearchParams({ step: shot.step }) + const params = new URLSearchParams() + if (shot.step) params.set("step", shot.step) if (shot.phase) params.set("phase", shot.phase) + if (shot.variant) params.set("variant", shot.variant) await page.goto( `chrome-extension://${extensionId}/src/onboarding_page.html?${params}`, ) diff --git a/packages/extension/public/_locales/de/messages.json b/packages/extension/public/_locales/de/messages.json index 10e20548..bd544e6d 100644 --- a/packages/extension/public/_locales/de/messages.json +++ b/packages/extension/public/_locales/de/messages.json @@ -1314,6 +1314,12 @@ "onboarding_skipButton": { "message": "Überspringen" }, + "onboarding_welcomeMessage": { + "message": "Willkommen!" + }, + "onboarding_welcomeContinue": { + "message": "Zum Fortfahren klicken" + }, "onboarding_step1Explain": { "message": "Markiere den folgenden Text, um sofort eine Suche auszuprobieren." }, diff --git a/packages/extension/public/_locales/en/messages.json b/packages/extension/public/_locales/en/messages.json index 77b80e6d..146a1745 100644 --- a/packages/extension/public/_locales/en/messages.json +++ b/packages/extension/public/_locales/en/messages.json @@ -1336,6 +1336,14 @@ "message": "Skip", "description": "Button shown on every onboarding step to end the onboarding early." }, + "onboarding_welcomeMessage": { + "message": "Welcome!", + "description": "Onboarding variant B welcome overlay, shown for a moment before the first step." + }, + "onboarding_welcomeContinue": { + "message": "Click to continue", + "description": "Accessible label for the onboarding variant B welcome overlay, which advances on click." + }, "onboarding_step1Explain": { "message": "Select the text below to try a search right away.", "description": "Onboarding Step1 (Search command) instruction shown before the user selects the sample text." diff --git a/packages/extension/public/_locales/es/messages.json b/packages/extension/public/_locales/es/messages.json index 4dc78aef..8ea4cf95 100644 --- a/packages/extension/public/_locales/es/messages.json +++ b/packages/extension/public/_locales/es/messages.json @@ -1314,6 +1314,12 @@ "onboarding_skipButton": { "message": "Omitir" }, + "onboarding_welcomeMessage": { + "message": "¡Bienvenido!" + }, + "onboarding_welcomeContinue": { + "message": "Haz clic para continuar" + }, "onboarding_step1Explain": { "message": "Selecciona el siguiente texto para probar una búsqueda al instante." }, diff --git a/packages/extension/public/_locales/fr/messages.json b/packages/extension/public/_locales/fr/messages.json index cd716827..535d92ff 100644 --- a/packages/extension/public/_locales/fr/messages.json +++ b/packages/extension/public/_locales/fr/messages.json @@ -1314,6 +1314,12 @@ "onboarding_skipButton": { "message": "Passer" }, + "onboarding_welcomeMessage": { + "message": "Bienvenue !" + }, + "onboarding_welcomeContinue": { + "message": "Cliquez pour continuer" + }, "onboarding_step1Explain": { "message": "Sélectionnez le texte ci-dessous pour essayer une recherche immédiatement." }, diff --git a/packages/extension/public/_locales/hi/messages.json b/packages/extension/public/_locales/hi/messages.json index 2ec54606..c4a4cbaa 100644 --- a/packages/extension/public/_locales/hi/messages.json +++ b/packages/extension/public/_locales/hi/messages.json @@ -1314,6 +1314,12 @@ "onboarding_skipButton": { "message": "छोड़ें" }, + "onboarding_welcomeMessage": { + "message": "स्वागत है!" + }, + "onboarding_welcomeContinue": { + "message": "जारी रखने के लिए क्लिक करें" + }, "onboarding_step1Explain": { "message": "तुरंत सर्च आज़माने के लिए नीचे दिए गए टेक्स्ट को चुनें।" }, diff --git a/packages/extension/public/_locales/id/messages.json b/packages/extension/public/_locales/id/messages.json index 6cd36074..cc0bcbc4 100644 --- a/packages/extension/public/_locales/id/messages.json +++ b/packages/extension/public/_locales/id/messages.json @@ -1317,6 +1317,12 @@ "onboarding_skipButton": { "message": "Lewati" }, + "onboarding_welcomeMessage": { + "message": "Selamat datang!" + }, + "onboarding_welcomeContinue": { + "message": "Klik untuk melanjutkan" + }, "onboarding_step1Explain": { "message": "Pilih teks di bawah ini untuk langsung mencoba pencarian." }, diff --git a/packages/extension/public/_locales/it/messages.json b/packages/extension/public/_locales/it/messages.json index f5baecca..b5ce0c49 100644 --- a/packages/extension/public/_locales/it/messages.json +++ b/packages/extension/public/_locales/it/messages.json @@ -1314,6 +1314,12 @@ "onboarding_skipButton": { "message": "Salta" }, + "onboarding_welcomeMessage": { + "message": "Benvenuto!" + }, + "onboarding_welcomeContinue": { + "message": "Clicca per continuare" + }, "onboarding_step1Explain": { "message": "Seleziona il testo qui sotto per provare subito una ricerca." }, diff --git a/packages/extension/public/_locales/ja/messages.json b/packages/extension/public/_locales/ja/messages.json index ad47d4c2..32d1adfe 100644 --- a/packages/extension/public/_locales/ja/messages.json +++ b/packages/extension/public/_locales/ja/messages.json @@ -1311,6 +1311,12 @@ "onboarding_skipButton": { "message": "スキップ" }, + "onboarding_welcomeMessage": { + "message": "ようこそ！" + }, + "onboarding_welcomeContinue": { + "message": "クリックして続ける" + }, "onboarding_step1Explain": { "message": "気になるテキストを選択して、すぐに検索してみましょう。" }, diff --git a/packages/extension/public/_locales/ko/messages.json b/packages/extension/public/_locales/ko/messages.json index bbd9cd83..6cd9cb34 100644 --- a/packages/extension/public/_locales/ko/messages.json +++ b/packages/extension/public/_locales/ko/messages.json @@ -1314,6 +1314,12 @@ "onboarding_skipButton": { "message": "건너뛰기" }, + "onboarding_welcomeMessage": { + "message": "환영합니다!" + }, + "onboarding_welcomeContinue": { + "message": "클릭하여 계속하기" + }, "onboarding_step1Explain": { "message": "아래 텍스트를 선택해서 바로 검색해 보세요." }, diff --git a/packages/extension/public/_locales/ms/messages.json b/packages/extension/public/_locales/ms/messages.json index fc1bc922..7ec23b71 100644 --- a/packages/extension/public/_locales/ms/messages.json +++ b/packages/extension/public/_locales/ms/messages.json @@ -1317,6 +1317,12 @@ "onboarding_skipButton": { "message": "Langkau" }, + "onboarding_welcomeMessage": { + "message": "Selamat datang!" + }, + "onboarding_welcomeContinue": { + "message": "Klik untuk teruskan" + }, "onboarding_step1Explain": { "message": "Pilih teks di bawah untuk cuba carian dengan segera." }, diff --git a/packages/extension/public/_locales/pt_BR/messages.json b/packages/extension/public/_locales/pt_BR/messages.json index 8768173a..60dfca09 100644 --- a/packages/extension/public/_locales/pt_BR/messages.json +++ b/packages/extension/public/_locales/pt_BR/messages.json @@ -1317,6 +1317,12 @@ "onboarding_skipButton": { "message": "Pular" }, + "onboarding_welcomeMessage": { + "message": "Bem-vindo!" + }, + "onboarding_welcomeContinue": { + "message": "Clique para continuar" + }, "onboarding_step1Explain": { "message": "Selecione o texto abaixo para testar uma pesquisa agora mesmo." }, diff --git a/packages/extension/public/_locales/pt_PT/messages.json b/packages/extension/public/_locales/pt_PT/messages.json index 0f45b75f..6a3ceb5a 100644 --- a/packages/extension/public/_locales/pt_PT/messages.json +++ b/packages/extension/public/_locales/pt_PT/messages.json @@ -1317,6 +1317,12 @@ "onboarding_skipButton": { "message": "Ignorar" }, + "onboarding_welcomeMessage": { + "message": "Bem-vindo!" + }, + "onboarding_welcomeContinue": { + "message": "Clique para continuar" + }, "onboarding_step1Explain": { "message": "Selecione o texto abaixo para experimentar uma pesquisa de imediato." }, diff --git a/packages/extension/public/_locales/ru/messages.json b/packages/extension/public/_locales/ru/messages.json index 742199b4..40182d76 100644 --- a/packages/extension/public/_locales/ru/messages.json +++ b/packages/extension/public/_locales/ru/messages.json @@ -1314,6 +1314,12 @@ "onboarding_skipButton": { "message": "Пропустить" }, + "onboarding_welcomeMessage": { + "message": "Добро пожаловать!" + }, + "onboarding_welcomeContinue": { + "message": "Нажмите, чтобы продолжить" + }, "onboarding_step1Explain": { "message": "Выделите текст ниже, чтобы сразу попробовать поиск." }, diff --git a/packages/extension/public/_locales/zh_CN/messages.json b/packages/extension/public/_locales/zh_CN/messages.json index b0a41661..4393614a 100644 --- a/packages/extension/public/_locales/zh_CN/messages.json +++ b/packages/extension/public/_locales/zh_CN/messages.json @@ -1314,6 +1314,12 @@ "onboarding_skipButton": { "message": "跳过" }, + "onboarding_welcomeMessage": { + "message": "欢迎！" + }, + "onboarding_welcomeContinue": { + "message": "点击继续" + }, "onboarding_step1Explain": { "message": "选择下面的文本，立即试试搜索功能。" }, diff --git a/packages/extension/src/components/onboarding/OnboardingFadeIn.tsx b/packages/extension/src/components/onboarding/OnboardingFadeIn.tsx index b48a30a7..6e173aff 100644 --- a/packages/extension/src/components/onboarding/OnboardingFadeIn.tsx +++ b/packages/extension/src/components/onboarding/OnboardingFadeIn.tsx @@ -1,21 +1,38 @@ import { ReactNode } from "react" import clsx from "clsx" +// `rise` is the default onboarding entrance (fade + slight upward slide). +// `blur` additionally resolves the content out of a blur, used by variant +// B's welcome overlay for a softer, Linear-style transition. +export type FadeInEffect = "rise" | "blur" + type Props = { children: ReactNode className?: string delay?: number // in milliseconds; optional delay before the fade-in starts + effect?: FadeInEffect +} + +const EFFECT_CLASS: Record = { + rise: "animate-onboarding-rise", + blur: "animate-onboarding-blur-in", } // Wraps a block of onboarding explanation text so it fades in from slightly // below, per the PRD's "説明テキストは少し下からフェードイン" rule. Uses a // `key`-less CSS animation (re-triggered by React remounting the element, // e.g. when the step/phase changes) rather than a JS animation library. -export function OnboardingFadeIn({ children, className, delay }: Props) { +export function OnboardingFadeIn({ + children, + className, + delay, + effect = "rise", +}: Props) { return (
+ } + return +} + +function OnboardingFlow({ variant }: { variant: ExperimentVariant }) { + const onboarding = useOnboardingState(variant) const [positionElm, setPositionElm] = useState(null) + // Variant B opens on a welcome overlay instead of the INTRO step. The step + // below it stays unmounted meanwhile, so its own entrance animations start + // only once the greeting is out of the way. + const isWelcome = onboarding.phase === StepPhase.WELCOME + return ( - {onboarding.step === OnboardingStep.INTRO && ( - - )} - {onboarding.step === OnboardingStep.SEARCH && ( - - )} - {onboarding.step === OnboardingStep.AI_PROMPT && ( - - )} - {onboarding.step === OnboardingStep.LINK_PREVIEW && ( - - )} - {onboarding.step === OnboardingStep.CUSTOMIZE && ( - - )} - {onboarding.step === OnboardingStep.COMPLETE && ( - + {!isWelcome && ( + <> + {onboarding.step === OnboardingStep.INTRO && ( + + )} + {onboarding.step === OnboardingStep.SEARCH && ( + + )} + {onboarding.step === OnboardingStep.AI_PROMPT && ( + + )} + {onboarding.step === OnboardingStep.LINK_PREVIEW && ( + + )} + {onboarding.step === OnboardingStep.CUSTOMIZE && ( + + )} + {onboarding.step === OnboardingStep.COMPLETE && ( + + )} + )} + {isWelcome && ( + onboarding.setPhase(StepPhase.EXPLAIN)} + /> + )} + diff --git a/packages/extension/src/components/onboarding/OnboardingWelcome.test.tsx b/packages/extension/src/components/onboarding/OnboardingWelcome.test.tsx new file mode 100644 index 00000000..6353dbeb --- /dev/null +++ b/packages/extension/src/components/onboarding/OnboardingWelcome.test.tsx @@ -0,0 +1,59 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest" +import { render, screen, act } from "@testing-library/react" +import { OnboardingWelcome } from "./OnboardingWelcome" + +// Keep in sync with OnboardingWelcome's own timings. +const WELCOME_MS = 2000 +const EXIT_MS = 320 + +describe("OnboardingWelcome", () => { + beforeEach(() => { + vi.useFakeTimers() + }) + afterEach(() => { + vi.useRealTimers() + }) + + const advance = async (ms: number) => { + await act(async () => { + vi.advanceTimersByTime(ms) + }) + } + + it("advances on its own after the welcome duration plus its exit animation", async () => { + const onDone = vi.fn() + render() + + await advance(WELCOME_MS - 1) + expect(onDone).not.toHaveBeenCalled() + + await advance(1 + EXIT_MS) + expect(onDone).toHaveBeenCalledTimes(1) + }) + + it("advances immediately when clicked, without waiting out the timer", async () => { + const onDone = vi.fn() + render() + + await act(async () => { + screen.getByTestId("onboarding-welcome").click() + }) + await advance(EXIT_MS) + + expect(onDone).toHaveBeenCalledTimes(1) + + // The pending auto-advance timer must not fire a second time. + await advance(WELCOME_MS + EXIT_MS) + expect(onDone).toHaveBeenCalledTimes(1) + }) + + it("does not call back after unmounting", async () => { + const onDone = vi.fn() + const { unmount } = render() + + unmount() + await advance(WELCOME_MS + EXIT_MS) + + expect(onDone).not.toHaveBeenCalled() + }) +}) diff --git a/packages/extension/src/components/onboarding/OnboardingWelcome.tsx b/packages/extension/src/components/onboarding/OnboardingWelcome.tsx new file mode 100644 index 00000000..3ea9853c --- /dev/null +++ b/packages/extension/src/components/onboarding/OnboardingWelcome.tsx @@ -0,0 +1,72 @@ +import { useCallback, useEffect, useRef, useState } from "react" +import clsx from "clsx" +import { t } from "@/services/i18n" +import { OnboardingFadeIn } from "./OnboardingFadeIn" + +const ICON_URL = chrome.runtime.getURL("SelectionCommandLogo.png") + +/** How long the overlay stays up before advancing on its own. */ +export const WELCOME_DURATION_MS = 2000 + +/** Keep in sync with the onboarding-blur-out duration in tailwind.config.js. */ +const EXIT_DURATION_MS = 320 + +type Props = { + onDone: () => void +} + +// Variant B's opening beat, replacing variant A's INTRO step: the brand mark +// and a short greeting, then straight into the first step. Rendered as an +// opaque full-screen overlay so the layout chrome (progress pills, Skip) +// stays hidden until the greeting is done, matching how INTRO looks in +// variant A. Advances on its own after WELCOME_DURATION_MS, or immediately +// when the user clicks - there is nothing to read here, so making people +// wait out the timer would just be a second thing to sit through. +export function OnboardingWelcome({ onDone }: Props) { + const [exiting, setExiting] = useState(false) + const timersRef = useRef[]>([]) + const finishedRef = useRef(false) + + const finish = useCallback(() => { + if (finishedRef.current) return + finishedRef.current = true + setExiting(true) + timersRef.current.push(setTimeout(onDone, EXIT_DURATION_MS)) + }, [onDone]) + + useEffect(() => { + timersRef.current.push(setTimeout(finish, WELCOME_DURATION_MS)) + return () => { + timersRef.current.forEach(clearTimeout) + timersRef.current = [] + } + }, [finish]) + + return ( + + ) +} diff --git a/packages/extension/src/components/onboarding/useOnboardingState.test.tsx b/packages/extension/src/components/onboarding/useOnboardingState.test.tsx new file mode 100644 index 00000000..f15e984a --- /dev/null +++ b/packages/extension/src/components/onboarding/useOnboardingState.test.tsx @@ -0,0 +1,41 @@ +import { describe, it, expect, vi, beforeEach } from "vitest" +import { renderHook } from "@testing-library/react" +import { OnboardingStep, StepPhase } from "@/types/onboarding" +import { ANALYTICS_EVENTS } from "@/services/analytics" +import { ONBOARDING_EXPERIMENT_ID } from "@/services/experiments" + +const sendOnboardingEvent = vi.fn() +vi.mock("./onboardingAnalytics", () => ({ + sendOnboardingEvent: (...args: unknown[]) => sendOnboardingEvent(...args), +})) + +const { useOnboardingState } = await import("./useOnboardingState") + +describe("useOnboardingState", () => { + beforeEach(() => { + sendOnboardingEvent.mockClear() + }) + + it("opens on the intro step for the control variant", () => { + const { result } = renderHook(() => useOnboardingState("A")) + + expect(result.current.step).toBe(OnboardingStep.INTRO) + expect(result.current.phase).toBe(StepPhase.EXPLAIN) + }) + + it("skips the intro and opens on the welcome overlay for variant B", () => { + const { result } = renderHook(() => useOnboardingState("B")) + + expect(result.current.step).toBe(OnboardingStep.SEARCH) + expect(result.current.phase).toBe(StepPhase.WELCOME) + }) + + it("reports the experiment id on the start event", () => { + renderHook(() => useOnboardingState("B")) + + expect(sendOnboardingEvent).toHaveBeenCalledWith( + ANALYTICS_EVENTS.ONBOARDING_START, + expect.objectContaining({ experiment_id: ONBOARDING_EXPERIMENT_ID }), + ) + }) +}) diff --git a/packages/extension/src/components/onboarding/useOnboardingState.ts b/packages/extension/src/components/onboarding/useOnboardingState.ts index 90365d08..5e22b824 100644 --- a/packages/extension/src/components/onboarding/useOnboardingState.ts +++ b/packages/extension/src/components/onboarding/useOnboardingState.ts @@ -6,6 +6,11 @@ import { Settings } from "@/services/settings/settings" import { getCurrentLocale } from "@/services/i18n" import { VERSION } from "@/const" import { closeOnboardingTab } from "./onboardingWindow" +import { + ONBOARDING_EXPERIMENT_ID, + getOnboardingAssignmentSync, +} from "@/services/experiments" +import type { ExperimentVariant } from "@/services/experiments" export type UseOnboardingState = ReturnType @@ -34,13 +39,24 @@ function readStepAndPhaseOverride(): { return { step, phase } } -export function useOnboardingState() { - const [step, setStep] = useState( - () => readStepAndPhaseOverride()?.step ?? OnboardingStep.INTRO, - ) - const [phase, _setPhase] = useState( - () => readStepAndPhaseOverride()?.phase ?? StepPhase.EXPLAIN, +// Where the flow opens, per A/B variant: the control keeps the INTRO step, +// while variant B drops it and starts on the first real step behind a short +// welcome overlay (see OnboardingWelcome). +function initialStepAndPhase(variant: ExperimentVariant): { + step: OnboardingStep + phase: StepPhase +} { + return variant === "B" + ? { step: OnboardingStep.SEARCH, phase: StepPhase.WELCOME } + : { step: OnboardingStep.INTRO, phase: StepPhase.EXPLAIN } +} + +export function useOnboardingState(variant: ExperimentVariant) { + const [initial] = useState( + () => readStepAndPhaseOverride() ?? initialStepAndPhase(variant), ) + const [step, setStep] = useState(initial.step) + const [phase, _setPhase] = useState(initial.phase) const startedAtRef = useRef(Date.now()) const seenValueStepsRef = useRef>(new Set()) const seenSelectionStepsRef = useRef>(new Set()) @@ -62,6 +78,10 @@ export function useOnboardingState() { sendOnboardingEvent(ANALYTICS_EVENTS.ONBOARDING_START, { locale: getCurrentLocale(), extension_version: VERSION, + experiment_id: ONBOARDING_EXPERIMENT_ID, + // Lets a hub outage (which forces the build-time allocation) be + // separated out when the results are analyzed. + config_source: getOnboardingAssignmentSync()?.configSource ?? "unknown", }) }, []) diff --git a/packages/extension/src/components/onboarding/useOnboardingVariant.ts b/packages/extension/src/components/onboarding/useOnboardingVariant.ts new file mode 100644 index 00000000..3c235041 --- /dev/null +++ b/packages/extension/src/components/onboarding/useOnboardingVariant.ts @@ -0,0 +1,49 @@ +import { useEffect, useState } from "react" +import { + ensureOnboardingAssignment, + overrideOnboardingAssignment, +} from "@/services/experiments" +import type { ExperimentVariant } from "@/services/experiments" + +// development or e2e build-only escape hatch (`?variant=B`), mirroring +// useOnboardingState's `?step=/?phase=` override so screenshot and manual QA +// tooling can land on either arm of the experiment on demand. +function readVariantOverride(): ExperimentVariant | null { + if (!["e2e", "development"].includes(import.meta.env.MODE)) return null + const param = new URLSearchParams(window.location.search).get("variant") + if (param !== "A" && param !== "B") return null + return overrideOnboardingAssignment(param).variant +} + +/** + * Resolve which onboarding variant to render, returning null until it is + * known. The background script assigns the variant on install before opening + * this tab (see background_script.ts), so this is normally just a single + * chrome.storage.local read; the fetch path only runs if that assignment is + * missing (page reopened manually, or the install-time assignment failed). + */ +export function useOnboardingVariant(): ExperimentVariant | null { + const [variant, setVariant] = useState( + readVariantOverride, + ) + + useEffect(() => { + if (variant != null) return + let cancelled = false + ensureOnboardingAssignment() + .then((assignment) => { + if (!cancelled) setVariant(assignment.variant) + }) + .catch((error) => { + console.error("Failed to resolve onboarding variant:", error) + // Never leave the user on a blank tab: fall back to the control. + if (!cancelled) setVariant("A") + }) + return () => { + cancelled = true + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + return variant +} diff --git a/packages/extension/src/types/onboarding.ts b/packages/extension/src/types/onboarding.ts index 5fe7a1e3..cc0d3347 100644 --- a/packages/extension/src/types/onboarding.ts +++ b/packages/extension/src/types/onboarding.ts @@ -11,8 +11,12 @@ export enum OnboardingStep { // user to select text, waiting for them to run the command, waiting for // them to come back from the popup/tab it opened, then showing the value // message. Not every step uses every phase (e.g. INTRO/CUSTOMIZE/COMPLETE -// only ever use EXPLAIN). +// only ever use EXPLAIN, and WELCOME only ever precedes SEARCH's EXPLAIN in +// variant B). export enum StepPhase { + // Variant B only: the welcome overlay shown before the first step's + // explanation (variant A shows the INTRO step instead). + WELCOME = "welcome", EXPLAIN = "explain", WAIT_SELECTION = "wait_selection", WAIT_EXECUTE = "wait_execute", diff --git a/packages/extension/tailwind.config.js b/packages/extension/tailwind.config.js index 32a0efc6..d931644c 100644 --- a/packages/extension/tailwind.config.js +++ b/packages/extension/tailwind.config.js @@ -47,6 +47,20 @@ module.exports = { from: { opacity: "0", transform: "translateY(10px)" }, to: { opacity: "1", transform: "translateY(0)" }, }, + // Linear-style enter/exit used by the variant B welcome overlay: + // content resolves out of a blur instead of just sliding up. + "onboarding-blur-in": { + from: { + opacity: "0", + filter: "blur(8px)", + transform: "translateY(6px)", + }, + to: { opacity: "1", filter: "blur(0px)", transform: "translateY(0)" }, + }, + "onboarding-blur-out": { + from: { opacity: "1", filter: "blur(0px)" }, + to: { opacity: "0", filter: "blur(8px)" }, + }, "onboarding-ring": { "0%, 100%": { boxShadow: "0 0 0 0 var(--onboarding-ring-color)" }, "50%": { boxShadow: "0 0 0 6px var(--onboarding-ring-color)" }, @@ -94,6 +108,10 @@ module.exports = { popup: "popup 0.3s cubic-bezier(0.34, 1.56, 0.64, 1) both", "onboarding-rise": "onboarding-rise 0.7s cubic-bezier(0.16, 1, 0.3, 1) both", + "onboarding-blur-in": + "onboarding-blur-in 0.8s cubic-bezier(0.16, 1, 0.3, 1) both", + "onboarding-blur-out": + "onboarding-blur-out 0.32s cubic-bezier(0.4, 0, 1, 1) both", "onboarding-ring": "onboarding-ring 2.4s ease-in-out 0.6s infinite", "onboarding-blink": "onboarding-blink 1.5s ease-in-out 0.3s infinite", "onboarding-highlight": "onboarding-highlight 4s ease-in-out infinite", From d7b20ebea10d629a12d050d44e3bc0e3af6ca8de Mon Sep 17 00:00:00 2001 From: ujiro99 Date: Sun, 13 Sep 2026 15:04:51 +0900 Subject: [PATCH 3/3] Fix: Address review findings on the onboarding welcome overlay - Hold onDone in a ref so the 2s auto-advance timer survives parent re-renders. Callers pass an inline callback, so depending on it directly restarted the pending timer on every render of the flow. - Withhold the progress indicator and the Skip button during the WELCOME phase: the overlay only hid the button visually, leaving it reachable by keyboard. - Correct the concurrency comment in ensureOnboardingAssignment - Storage.update is a plain get/set, so the re-read is best-effort, not atomic. - Note that the build-time fallback allocation deliberately duplicates the hub's experiments.json, and has to be updated alongside it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VeTTSrCxM9ifQatx6XjCxo --- .../onboarding/OnboardingLayout.tsx | 9 +++-- .../components/onboarding/OnboardingPage.tsx | 16 ++++---- .../onboarding/OnboardingWelcome.test.tsx | 13 ++++++ .../onboarding/OnboardingWelcome.tsx | 16 +++++++- .../onboarding/onboardingProgress.test.ts | 40 ++++++++++++------- .../onboarding/onboardingProgress.ts | 20 +++++++--- .../services/experiments/experimentConfig.ts | 5 +++ .../experiments/onboardingExperiment.ts | 10 +++-- packages/hub/AGENTS.md | 4 ++ 9 files changed, 97 insertions(+), 36 deletions(-) diff --git a/packages/extension/src/components/onboarding/OnboardingLayout.tsx b/packages/extension/src/components/onboarding/OnboardingLayout.tsx index 26b29743..1187784a 100644 --- a/packages/extension/src/components/onboarding/OnboardingLayout.tsx +++ b/packages/extension/src/components/onboarding/OnboardingLayout.tsx @@ -1,12 +1,13 @@ import { ReactNode } from "react" import { t } from "@/services/i18n" import { getProgress, showsSkip } from "./onboardingProgress" -import type { OnboardingStep } from "@/types/onboarding" +import type { OnboardingStep, StepPhase } from "@/types/onboarding" const ICON_URL = chrome.runtime.getURL("SelectionCommandLogo.png") type Props = { step: OnboardingStep + phase: StepPhase onSkip: () => void children: ReactNode } @@ -17,8 +18,8 @@ type Props = { // a header with the brand mark and a 4-segment progress indicator, and a // Skip button fixed to the bottom-right corner. Individual step components // now render only their own content. -export function OnboardingLayout({ step, onSkip, children }: Props) { - const progress = getProgress(step) +export function OnboardingLayout({ step, phase, onSkip, children }: Props) { + const progress = getProgress(step, phase) return (
@@ -61,7 +62,7 @@ export function OnboardingLayout({ step, onSkip, children }: Props) { {children}
- {showsSkip(step) && ( + {showsSkip(step, phase) && (