diff --git a/pkg/api/componentreadiness/component_report.go b/pkg/api/componentreadiness/component_report.go index cf2f2ca99..36e457923 100644 --- a/pkg/api/componentreadiness/component_report.go +++ b/pkg/api/componentreadiness/component_report.go @@ -327,104 +327,49 @@ func (c *ComponentReportGenerator) GenerateReport(ctx context.Context) (crtype.C return crtype.ComponentReport{}, errs } report.GeneratedAt = componentReportTestStatus.GeneratedAt - log.Infof("GenerateReport completed in %s with %d sample results and %d base results from db", time.Since(before), sampleLen, len(componentReportTestStatus.BaseStatus)) + log.WithField("duration", time.Since(before).String()). + WithField("sampleResults", sampleLen). + WithField("baseResults", len(componentReportTestStatus.BaseStatus)). + Info("GenerateReport completed") return report, nil } -// getTestStatus orchestrates the actual fetching of junit test run data for both basis and sample. -// goroutines are used to concurrently request the data for basis, sample, and various other edge cases. func (c *ComponentReportGenerator) getTestStatus(ctx context.Context) (crstatus.ReportTestStatus, []error) { before := time.Now() - fLog := log.WithField("func", "getTestStatus") - var baseStatus, sampleStatus map[string]crstatus.TestStatus - baseStatusCh := make(chan map[string]crstatus.TestStatus) // TODO: not hooked up yet, just in place for the interface for now - var baseErrs, sampleErrs []error wg := &sync.WaitGroup{} - - // channels for status as we may collect status from multiple queries run in separate goroutines - sampleStatusCh := make(chan map[string]crstatus.TestStatus) errCh := make(chan error) - statusDoneCh := make(chan struct{}) // To signal when all processing is done - statusErrsDoneCh := make(chan struct{}) // To signal when all processing is done - - // generate inputs to the channels - c.middlewares.Query(ctx, wg, baseStatusCh, sampleStatusCh, errCh) - goInterruptible(ctx, wg, func() { baseStatus, baseErrs = c.dataProvider.QueryBaseTestStatus(ctx, c.ReqOptions) }) - goInterruptible(ctx, wg, func() { - fLog.Infof("running sample query with includeVariants: %+v", c.ReqOptions.VariantOption.IncludeVariants) - status, errs := c.dataProvider.QuerySampleTestStatus(ctx, c.ReqOptions, c.ReqOptions.VariantOption.IncludeVariants, c.ReqOptions.SampleRelease.Start, c.ReqOptions.SampleRelease.End) - fLog.Infof("received %d test statuses and %d errors from sample query", len(status), len(errs)) - sampleStatusCh <- status - for _, err := range errs { + + var baseStatus, sampleStatus map[string]crstatus.TestStatus + wg.Go(func() { + var queryErrs []error + baseStatus, sampleStatus, queryErrs = c.dataProvider.QueryTestStatus(ctx, c.ReqOptions) + for _, err := range queryErrs { errCh <- err } }) - // clean up channels after all queries are done + c.middlewares.Query(ctx, wg, errCh) + go func() { wg.Wait() - close(baseStatusCh) - close(sampleStatusCh) close(errCh) }() - // manage output from the channels - go func() { - for status := range sampleStatusCh { - fLog.Infof("received %d test statuses over channel", len(status)) - for k, v := range status { - if sampleStatus == nil { - fLog.Warnf("initializing sampleStatus map") - sampleStatus = make(map[string]crstatus.TestStatus) - } - if v2, ok := sampleStatus[k]; ok { - fLog.Warnf("sampleStatus already had key: %+v", k) - fLog.Warnf("sampleStatus new value: %+v", v) - fLog.Warnf("sampleStatus old value: %+v", v2) - } - sampleStatus[k] = v - } - } - close(statusDoneCh) - }() - - go func() { - for err := range errCh { - sampleErrs = append(sampleErrs, err) - } - close(statusErrsDoneCh) - }() - - <-statusDoneCh - <-statusErrsDoneCh - fLog.Infof("total test statuses: %d", len(sampleStatus)) - var errs []error - if len(baseErrs) != 0 || len(sampleErrs) != 0 { - errs = append(errs, baseErrs...) - errs = append(errs, sampleErrs...) + for err := range errCh { + errs = append(errs, err) } - log.Infof("getTestStatus completed in %s with %d sample results and %d base results", - time.Since(before), len(sampleStatus), len(baseStatus)) + + log.WithField("duration", time.Since(before)). + WithField("sampleResults", len(sampleStatus)). + WithField("baseResults", len(baseStatus)). + Info("getTestStatus completed") now := time.Now() return crstatus.ReportTestStatus{BaseStatus: baseStatus, SampleStatus: sampleStatus, GeneratedAt: &now}, errs } -func goInterruptible(ctx context.Context, wg *sync.WaitGroup, closure func()) { - wg.Add(1) - go func() { - defer wg.Done() - select { - case <-ctx.Done(): - return - default: - closure() - } - }() -} - var componentAndCapabilityGetter func(stats crstatus.TestStatus) (string, []string) func testToComponentAndCapability(stats crstatus.TestStatus) (string, []string) { diff --git a/pkg/api/componentreadiness/dataprovider/bigquery/provider.go b/pkg/api/componentreadiness/dataprovider/bigquery/provider.go index ae902429d..46aeca374 100644 --- a/pkg/api/componentreadiness/dataprovider/bigquery/provider.go +++ b/pkg/api/componentreadiness/dataprovider/bigquery/provider.go @@ -7,6 +7,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "cloud.google.com/go/bigquery" @@ -67,15 +68,15 @@ func (p *BigQueryProvider) QueryBaseTestStatus(ctx context.Context, reqOptions r return result.BaseStatus, nil } -func (p *BigQueryProvider) QuerySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, - includeVariants map[string][]string, - start, end time.Time) (map[string]crstatus.TestStatus, []error) { +func (p *BigQueryProvider) querySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions) (map[string]crstatus.TestStatus, []error) { allJobVariants, errs := p.QueryJobVariants(ctx, reqOptions) if len(errs) > 0 { return nil, errs } - generator := NewSampleQueryGenerator(p.client, reqOptions, allJobVariants, includeVariants, start, end) + generator := NewSampleQueryGenerator(p.client, reqOptions, allJobVariants, + reqOptions.VariantOption.IncludeVariants, + reqOptions.SampleRelease.Start, reqOptions.SampleRelease.End) result, errs := apiPkg.GetDataFromCacheOrGenerate[crstatus.ReportTestStatus]( ctx, p.client.Cache, reqOptions.CacheOption, apiPkg.NewCacheSpec(generator, "SampleTestStatus~", &reqOptions.SampleRelease.End), @@ -86,6 +87,24 @@ func (p *BigQueryProvider) QuerySampleTestStatus(ctx context.Context, reqOptions return result.SampleStatus, nil } +func (p *BigQueryProvider) QueryTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions) (baseStatus, sampleStatus map[string]crstatus.TestStatus, errs []error) { + var baseErrs, sampleErrs []error + var wg sync.WaitGroup + wg.Go(func() { + baseStatus, baseErrs = p.QueryBaseTestStatus(ctx, reqOptions) + }) + wg.Go(func() { + sampleStatus, sampleErrs = p.querySampleTestStatus(ctx, reqOptions) + }) + wg.Wait() + errs = append(errs, baseErrs...) + errs = append(errs, sampleErrs...) + if len(errs) > 0 { + return nil, nil, errs + } + return baseStatus, sampleStatus, nil +} + // --- TestDetailsQuerier --- func (p *BigQueryProvider) QueryBaseJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions) (map[string][]crstatus.TestDetailsSummary, []error) { diff --git a/pkg/api/componentreadiness/dataprovider/interface.go b/pkg/api/componentreadiness/dataprovider/interface.go index 2adfdf404..e7dd2e162 100644 --- a/pkg/api/componentreadiness/dataprovider/interface.go +++ b/pkg/api/componentreadiness/dataprovider/interface.go @@ -16,10 +16,10 @@ type TestStatusQuerier interface { // QueryBaseTestStatus returns test status for the basis release. QueryBaseTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions) (map[string]crstatus.TestStatus, []error) - // QuerySampleTestStatus returns test status for the sample release. - QuerySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, - includeVariants map[string][]string, - start, end time.Time) (map[string]crstatus.TestStatus, []error) + // QueryTestStatus returns both base and sample test status. + // Providers may execute this as a single optimized query or by + // delegating to QueryBaseTestStatus and a provider-internal sample query. + QueryTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions) (baseStatus, sampleStatus map[string]crstatus.TestStatus, errs []error) } // TestDetailsQuerier fetches per-job test breakdowns used for test details reports. diff --git a/pkg/api/componentreadiness/dataprovider/mixed/provider.go b/pkg/api/componentreadiness/dataprovider/mixed/provider.go index 580b8e844..28e66e96b 100644 --- a/pkg/api/componentreadiness/dataprovider/mixed/provider.go +++ b/pkg/api/componentreadiness/dataprovider/mixed/provider.go @@ -64,8 +64,8 @@ func (p *MixedProvider) QueryBaseTestStatus(ctx context.Context, reqOptions reqo return p.providerFor(reqOptions).QueryBaseTestStatus(ctx, reqOptions) } -func (p *MixedProvider) QuerySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, includeVariants map[string][]string, start, end time.Time) (map[string]crstatus.TestStatus, []error) { - return p.providerFor(reqOptions).QuerySampleTestStatus(ctx, reqOptions, includeVariants, start, end) +func (p *MixedProvider) QueryTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions) (baseStatus, sampleStatus map[string]crstatus.TestStatus, errs []error) { + return p.providerFor(reqOptions).QueryTestStatus(ctx, reqOptions) } func (p *MixedProvider) QueryBaseJobRunTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions) (map[string][]crstatus.TestDetailsSummary, []error) { diff --git a/pkg/api/componentreadiness/dataprovider/postgres/cr_queries.go b/pkg/api/componentreadiness/dataprovider/postgres/cr_queries.go index 00bb3e723..c2e5324f6 100644 --- a/pkg/api/componentreadiness/dataprovider/postgres/cr_queries.go +++ b/pkg/api/componentreadiness/dataprovider/postgres/cr_queries.go @@ -4,8 +4,11 @@ import ( "context" "database/sql" "fmt" - "sync" + "maps" + "strings" + "time" + "cloud.google.com/go/civil" "github.com/lib/pq" "gorm.io/gorm" "k8s.io/apimachinery/pkg/util/sets" @@ -28,6 +31,8 @@ type variantQuerySetup struct { minimumFailure int } +const queryPlannerHints = "SET LOCAL max_parallel_workers_per_gather = 4; SET LOCAL parallel_setup_cost = 0; SET LOCAL parallel_tuple_cost = 0; SET LOCAL enable_nestloop = off; SET LOCAL enable_sort = off" + func prepareVariantQuery( ctx context.Context, dbc *db.DB, @@ -35,31 +40,18 @@ func prepareVariantQuery( dbGroupBy sets.Set[string], minimumFailure int, ) (*variantQuerySetup, error) { - if includeVariants == nil { - includeVariants = map[string][]string{} - } - - variantLookup, err := lookupVariantValues(ctx, dbc, includeVariants, dbGroupBy) + vf, err := resolveVariantFilter(ctx, dbc, includeVariants, dbGroupBy) if err != nil { return nil, err } - if len(variantLookup) == 0 { + if len(vf.lookup) == 0 { return nil, nil } - groupMapping := buildVariantGroupMapping(variantLookup) - - filterClause, filterArgs := buildVariantFilterClause(includeVariants) - - variantSubquery := "SELECT vc.id FROM variant_combinations vc" - if filterClause != "" { - variantSubquery += " WHERE " + filterClause - } - return &variantQuerySetup{ - groupMapping: groupMapping, - filterArgs: filterArgs, - variantSubquery: variantSubquery, + groupMapping: buildVariantGroupMapping(vf.lookup), + filterArgs: vf.filterArgs, + variantSubquery: vf.variantSubquery, minimumFailure: minimumFailure, }, nil } @@ -107,32 +99,120 @@ func buildDrilldownFilters(reqOptions reqopts.RequestOptions) drilldownFilters { return f } -// testStatusSpec parameterizes the shared query structure used by both the -// prefix-sum and GA query paths. The two paths differ only in their source -// table, aggregation expressions, and date/window filter. +// testStatusSpec parameterizes the inner aggregation query used by both the +// prefix-sum and GA query paths. Each spec produces an inner SELECT with +// columns: test_id, suite_id, variant_group_id, total_count, success_count, +// flake_count, last_failure. The outer CTE wrapper (test_ownerships join, +// column group mapping) is identical for all specs and handled by buildStatusCTE. type testStatusSpec struct { - fromTemplate string // FROM clause template with two %s for variantSubquery and groupMapping - preJoinArgs []any // args bound in the FROM clause before filterArgs (e.g. lookupStart) - totalExpr string // SQL expression for total runs - successExpr string // SQL expression for successes - flakeExpr string // SQL expression for flakes - lastFailureExpr string // SQL expression for last failure timestamp (e.g. "MAX(e.prefix_max_last_failure)" or "NULL::timestamptz") - whereFilter string // WHERE fragment like "e.release = ? AND e.date = ?" - whereArgs []any // args for whereFilter - lifecycles []string // when non-empty, filter e.lifecycle to these values (sample-only, matching BQ) + fromTemplate string // FROM clause template with one %s for the formatted prowJobJoin + fromArgs []any // args for FROM clause (before filterArgs) + selectCols string // the 4 aggregation column expressions (total_count through last_failure) + selectArgs []any // args for placeholders within selectCols (e.g., lookupEnd/lookupStart for CASE WHEN) + whereFilter string // WHERE fragment like "e.release = ? AND e.date IN (?, ?)" + whereArgs []any // args for whereFilter + havingClause string // optional HAVING clause (e.g., "\nHAVING SUM(e.runs) > 0") + lifecycles []string } -// queryTestStatus builds and executes the failure + placeholder query pair -// that both queryTestStatusPrefixSum and queryBaseTestStatusGA share. +// buildInnerAggregation constructs the inner SELECT ... GROUP BY from a +// testStatusSpec and a pre-formatted prow job join clause. The result produces +// columns: test_id, suite_id, variant_group_id, total_count, success_count, +// flake_count, last_failure. +func buildInnerAggregation(spec testStatusSpec, prowJobJoin string, filterArgs []any, filters drilldownFilters) (string, []any) { + fromClause := fmt.Sprintf(spec.fromTemplate, prowJobJoin) + + lifecycleClause := "" + var lifecycleArgs []any + if len(spec.lifecycles) > 0 { + lifecycleClause = "\n AND e.lifecycle = ANY(?)" + lifecycleArgs = []any{pq.Array(spec.lifecycles)} + } + + innerSQL := fmt.Sprintf(` + SELECT e.test_id, e.suite_id, vg.group_id AS variant_group_id, + %s + %s + WHERE %s%s%s + GROUP BY e.test_id, e.suite_id, vg.group_id%s`, + spec.selectCols, + fromClause, + spec.whereFilter, lifecycleClause, filters.innerClause, + spec.havingClause) + + var args []any + args = append(args, spec.selectArgs...) + args = append(args, spec.fromArgs...) + args = append(args, filterArgs...) + args = append(args, spec.whereArgs...) + args = append(args, lifecycleArgs...) + args = append(args, filters.innerArgs...) + return innerSQL, args +} + +// buildStatusCTE wraps an inner aggregation query in a materialized CTE that +// joins test_ownerships (using COALESCE for suite_id NULL handling) and the +// column group mapping CTE. The resulting CTE has columns: test_id, suite_id, +// variant_group_id, total_count, success_count, flake_count, last_failure, +// col_group_id, unique_id, component, capabilities. // -// Two queries run in parallel: -// - Failure query: tests with >= MinimumFailure failures (regression candidates) -// - Placeholder query: (component, col_group_id) pairs with any runs (for grid gating) +// colMappingCTE is the name of a CTE (defined earlier in the WITH clause) +// with columns (group_id, col_group_id). +func buildStatusCTE( + cteName string, + innerSQL string, + innerArgs []any, + colMappingCTE string, + filters drilldownFilters, +) (string, []any) { + cteSQL := fmt.Sprintf(`%s AS MATERIALIZED ( + SELECT agg.*, cm.col_group_id, tow.unique_id, tow.component, tow.capabilities FROM ( + %s + ) agg + JOIN %s cm ON cm.group_id = agg.variant_group_id + JOIN test_ownerships tow ON tow.test_id = agg.test_id + AND COALESCE(tow.suite_id, 0) = agg.suite_id + AND tow.staff_approved_obsolete = false + WHERE agg.total_count > 0%s + )`, cteName, innerSQL, colMappingCTE, filters.outerClause) + + var args []any + args = append(args, innerArgs...) + args = append(args, filters.outerArgs...) + return cteSQL, args +} + +// failureBranchTemplate is the UNION ALL branch that selects tests meeting the +// minimum failure threshold from a status CTE. The first %s is an optional +// column prefix (e.g. "? AS source, " for the combined query, or "" for +// standalone), and the second %s is the CTE name. +const failureBranchTemplate = `SELECT + %spa.unique_id AS test_id, t.name AS test_name, + COALESCE(su.name, '') AS test_suite, pa.component, pa.capabilities, + pa.variant_group_id, pa.total_count, pa.success_count, pa.flake_count, pa.last_failure + FROM %s pa + JOIN tests t ON t.id = pa.test_id + LEFT JOIN suites su ON su.id = pa.suite_id + WHERE pa.total_count - pa.success_count - pa.flake_count >= ?` + +// placeholderBranchTemplate is the UNION ALL branch that produces grid +// placeholder entries (one per component + col_group_id) from a status CTE. +// The first %s is an optional column prefix, the second %s is the CTE name. +const placeholderBranchTemplate = `SELECT + %s'grid:' || pa.component AS test_id, '' AS test_name, '' AS test_suite, + pa.component, pa.capabilities, + pa.col_group_id AS variant_group_id, + 1 AS total_count, 1 AS success_count, 0 AS flake_count, NULL::timestamptz AS last_failure + FROM %s pa + GROUP BY pa.component, pa.capabilities, pa.col_group_id` + +// queryTestStatusCTE builds and executes a single CTE-based query that produces +// both failure results (tests with >= MinimumFailure failures) and grid +// placeholder entries (component-level cells confirming data exists). // -// Grid placeholders are only injected for cells where data confirms -// tests actually ran, so that cells without data on one side correctly show -// MissingSample / MissingBasis instead of NotSignificant. -func (p *PostgresProvider) queryTestStatus( +// Grid placeholders ensure that cells without failures on one side correctly +// show MissingSample / MissingBasis instead of NotSignificant. +func (p *PostgresProvider) queryTestStatusCTE( ctx context.Context, reqOptions reqopts.RequestOptions, includeVariants map[string][]string, @@ -150,75 +230,49 @@ func (p *PostgresProvider) queryTestStatus( return map[string]crstatus.TestStatus{}, nil } - if len(spec.lifecycles) > 0 { - spec.whereFilter += " AND e.lifecycle = ANY(?)" - spec.whereArgs = append(spec.whereArgs, pq.Array(spec.lifecycles)) - } + prowJobJoin := prowJobVariantJoin(setup.variantSubquery) - fromClause := fmt.Sprintf(spec.fromTemplate, setup.variantSubquery, setup.groupMapping.valuesClause) - - joinArgs := make([]any, 0, len(spec.preJoinArgs)+len(setup.filterArgs)+len(spec.whereArgs)) - joinArgs = append(joinArgs, spec.preJoinArgs...) - joinArgs = append(joinArgs, setup.filterArgs...) - joinArgs = append(joinArgs, spec.whereArgs...) - - failureInner := fmt.Sprintf(` - SELECT - e.test_id, e.suite_id, vg.group_id AS variant_group_id, - SUM(%s) AS total_count, - SUM(%s) AS success_count, - SUM(%s) AS flake_count, - %s AS last_failure - %s - WHERE %s`+filters.innerClause+` - GROUP BY e.test_id, e.suite_id, vg.group_id - HAVING SUM(%s) > 0 - AND SUM(%s) - SUM(%s) - SUM(%s) >= ?`, - spec.totalExpr, spec.successExpr, spec.flakeExpr, - spec.lastFailureExpr, - fromClause, - spec.whereFilter, - spec.totalExpr, spec.totalExpr, spec.successExpr, spec.flakeExpr) - - failureArgs := make([]any, len(joinArgs)) - copy(failureArgs, joinArgs) - failureArgs = append(failureArgs, filters.innerArgs...) - failureArgs = append(failureArgs, setup.minimumFailure) + innerSQL, innerArgs := buildInnerAggregation(spec, prowJobJoin, setup.filterArgs, filters) colMapping := buildColumnGroupMapping(setup.groupMapping.groupToVariants, reqOptions.VariantOption.ColumnGroupBy) + cteSQL, cteArgs := buildStatusCTE("status_agg", innerSQL, innerArgs, "cm", filters) - placeholderQuery := fmt.Sprintf(` - SELECT - 'grid:' || tow.component AS test_id, - '' AS test_name, - '' AS test_suite, - tow.component, - tow.capabilities, - cm.col_group_id AS variant_group_id, - 1 AS total_count, - 1 AS success_count, - 0 AS flake_count, - NULL::timestamptz AS last_failure - %s - JOIN test_ownerships tow ON tow.test_id = e.test_id - AND (tow.suite_id = e.suite_id OR (tow.suite_id IS NULL AND e.suite_id = 0)) - AND tow.staff_approved_obsolete = false - JOIN (%s) AS cm(group_id, col_group_id) ON cm.group_id = vg.group_id - WHERE %s`+filters.outerClause+` - GROUP BY tow.component, tow.capabilities, cm.col_group_id - HAVING SUM(%s) > 0`, - fromClause, colMapping.valuesClause, spec.whereFilter, spec.totalExpr) - - placeholderArgs := make([]any, len(joinArgs)) - copy(placeholderArgs, joinArgs) - placeholderArgs = append(placeholderArgs, filters.outerArgs...) - - return p.runFailureAndPlaceholder(ctx, failureInner, failureArgs, placeholderQuery, placeholderArgs, - setup.groupMapping, filters) + fullSQL := fmt.Sprintf("WITH vg(vcid, group_id) AS (%s),\ncm(group_id, col_group_id) AS (%s),\n%s\n%s\nUNION ALL\n%s", + setup.groupMapping.valuesClause, colMapping.valuesClause, + cteSQL, + fmt.Sprintf(failureBranchTemplate, "", "status_agg"), + fmt.Sprintf(placeholderBranchTemplate, "", "status_agg")) + + var allArgs []any + allArgs = append(allArgs, cteArgs...) + allArgs = append(allArgs, setup.minimumFailure) + + return p.scanWithParallelHints(ctx, fullSQL, allArgs, setup.groupMapping) +} + +// prefixSumSpec returns a testStatusSpec for querying test_cumulative_summaries +// using CASE WHEN on prefix sums to compute aggregated counts for a date range. +func prefixSumSpec(release string, lookupEnd, lookupStart civil.Date, lifecycles []string) testStatusSpec { + return testStatusSpec{ + fromTemplate: ` + FROM test_cumulative_summaries e + %s`, + selectCols: `SUM(CASE WHEN e.date = ? THEN e.prefix_sum_runs ELSE 0 END) + - SUM(CASE WHEN e.date = ? THEN e.prefix_sum_runs ELSE 0 END) AS total_count, + SUM(CASE WHEN e.date = ? THEN e.prefix_sum_successes ELSE 0 END) + - SUM(CASE WHEN e.date = ? THEN e.prefix_sum_successes ELSE 0 END) AS success_count, + SUM(CASE WHEN e.date = ? THEN e.prefix_sum_flakes ELSE 0 END) + - SUM(CASE WHEN e.date = ? THEN e.prefix_sum_flakes ELSE 0 END) AS flake_count, + MAX(CASE WHEN e.date = ? THEN e.prefix_max_last_failure END) AS last_failure`, + selectArgs: []any{lookupEnd, lookupStart, lookupEnd, lookupStart, lookupEnd, lookupStart, lookupEnd}, + whereFilter: "e.release = ? AND e.date IN (?, ?)", + whereArgs: []any{release, lookupEnd, lookupStart}, + lifecycles: lifecycles, + } } -// queryTestStatusPrefixSum queries test_cumulative_summaries using a -// 2-way self-join on prefix sums to compute aggregated counts for a date range. +// queryTestStatusPrefixSum queries test_cumulative_summaries using CASE WHEN +// on prefix sums to compute aggregated counts for a date range. func (p *PostgresProvider) queryTestStatusPrefixSum( ctx context.Context, reqOptions reqopts.RequestOptions, @@ -234,25 +288,25 @@ func (p *PostgresProvider) queryTestStatusPrefixSum( lookupEnd := dateRange.End.AddDays(-1) lookupStart := dateRange.Start.AddDays(-1) - return p.queryTestStatus(ctx, reqOptions, includeVariants, testStatusSpec{ + return p.queryTestStatusCTE(ctx, reqOptions, includeVariants, + prefixSumSpec(release, lookupEnd, lookupStart, lifecycles)) +} + +// gaSpec returns a testStatusSpec for querying prow_ga_raw_test_data to compute +// aggregated base test status for GA releases. +func gaSpec(release string, windowDays int) testStatusSpec { + return testStatusSpec{ fromTemplate: ` - FROM test_cumulative_summaries e - LEFT JOIN test_cumulative_summaries s - ON s.release = e.release AND s.test_id = e.test_id - AND s.prow_job_id = e.prow_job_id AND s.suite_id = e.suite_id - AND s.lifecycle = e.lifecycle AND s.date = ? - JOIN prow_jobs pj ON pj.id = e.prow_job_id AND pj.deleted_at IS NULL - AND pj.variant_combination_id IN (%s) - JOIN (%s) AS vg(vcid, group_id) ON vg.vcid = pj.variant_combination_id`, - preJoinArgs: []any{lookupStart}, - totalExpr: "e.prefix_sum_runs - COALESCE(s.prefix_sum_runs, 0)", - successExpr: "e.prefix_sum_successes - COALESCE(s.prefix_sum_successes, 0)", - flakeExpr: "e.prefix_sum_flakes - COALESCE(s.prefix_sum_flakes, 0)", - lastFailureExpr: "MAX(e.prefix_max_last_failure)", - whereFilter: "e.release = ? AND e.date = ?", - whereArgs: []any{release, lookupEnd}, - lifecycles: lifecycles, - }) + FROM prow_ga_raw_test_data e + %s`, + selectCols: `SUM(e.runs) AS total_count, + SUM(e.passes) AS success_count, + SUM(e.flakes) AS flake_count, + NULL::timestamptz AS last_failure`, + whereFilter: "e.release = ? AND e.window_days = ?", + whereArgs: []any{release, windowDays}, + havingClause: "\n HAVING SUM(e.runs) > 0", + } } // queryBaseTestStatusGA queries prow_ga_raw_test_data to compute aggregated @@ -266,119 +320,8 @@ func (p *PostgresProvider) queryBaseTestStatusGA( release := reqOptions.BaseRelease.Name windowDays := baseRange.End.AddDays(-1).DaysSince(baseRange.Start) - return p.queryTestStatus(ctx, reqOptions, reqOptions.VariantOption.IncludeVariants, testStatusSpec{ - fromTemplate: ` - FROM prow_ga_raw_test_data e - JOIN prow_jobs pj ON pj.id = e.prow_job_id AND pj.deleted_at IS NULL - AND pj.variant_combination_id IN (%s) - JOIN (%s) AS vg(vcid, group_id) ON vg.vcid = pj.variant_combination_id`, - totalExpr: "e.runs", - successExpr: "e.passes", - flakeExpr: "e.flakes", - lastFailureExpr: "NULL::timestamptz", - whereFilter: "e.release = ? AND e.window_days = ?", - whereArgs: []any{release, windowDays}, - }) -} - -// runFailureAndPlaceholder runs the failure query and a placeholder query in -// parallel. The placeholder query groups by (component, col_group_id) to -// identify grid cells that have data, returning rows in the same 10-column -// format as the failure query. After both complete, placeholder entries are -// merged into the failure map for cells that have data but no failures. -func (p *PostgresProvider) runFailureAndPlaceholder( - ctx context.Context, - failureInner string, - failureArgs []any, - placeholderQuery string, - placeholderArgs []any, - groupMapping variantGroupMapping, - filters drilldownFilters, -) (map[string]crstatus.TestStatus, []error) { - - var failureResult map[string]crstatus.TestStatus - var failureErrs []error - var placeholderResult map[string]crstatus.TestStatus - var placeholderErrs []error - - var wg sync.WaitGroup - wg.Go(func() { - failureResult, failureErrs = p.queryAndScan(ctx, failureInner, failureArgs, groupMapping, filters) - }) - wg.Go(func() { - placeholderResult, placeholderErrs = p.scanGroupedResults(ctx, placeholderQuery, placeholderArgs, groupMapping) - }) - wg.Wait() - - if len(failureErrs) > 0 || len(placeholderErrs) > 0 { - var errs []error - errs = append(errs, failureErrs...) - errs = append(errs, placeholderErrs...) - return nil, errs - } - - merged := 0 - for k, v := range placeholderResult { - if _, exists := failureResult[k]; !exists { - failureResult[k] = v - merged++ - } - } - log.WithField("placeholders", len(placeholderResult)). - WithField("merged", merged). - WithField("failures", len(failureResult)-merged). - WithField("total", len(failureResult)). - Info("placeholder query complete") - return failureResult, nil -} - -// outerQuery wraps an inner aggregation subquery with the shared outer SELECT -// that joins tests, test_ownerships, and suites to produce the final result -// columns. Both sample and base queries use the same outer structure. -const outerQuery = `SELECT - tow.unique_id AS test_id, - t.name AS test_name, - COALESCE(su.name, '') AS test_suite, - tow.component, - tow.capabilities, - pa.variant_group_id, - pa.total_count, - pa.success_count, - pa.flake_count, - pa.last_failure -FROM (%s) pa -JOIN tests t ON t.id = pa.test_id -JOIN test_ownerships tow ON tow.test_id = pa.test_id - AND (tow.suite_id = pa.suite_id OR (tow.suite_id IS NULL AND pa.suite_id = 0)) -LEFT JOIN suites su ON su.id = pa.suite_id -WHERE tow.staff_approved_obsolete = false` - -// queryAndScan wraps an inner aggregation subquery with the shared outer query, -// appends any drill-down filter clauses, and executes with parallel worker hints. -func (p *PostgresProvider) queryAndScan( - ctx context.Context, - innerQuery string, - innerArgs []any, - groupMapping variantGroupMapping, - filters drilldownFilters, -) (map[string]crstatus.TestStatus, []error) { - fullQuery := fmt.Sprintf(outerQuery, innerQuery) + filters.outerClause - allArgs := make([]any, 0, len(innerArgs)+len(filters.outerArgs)) - allArgs = append(allArgs, innerArgs...) - allArgs = append(allArgs, filters.outerArgs...) - return p.scanWithParallelHints(ctx, fullQuery, allArgs, groupMapping) -} - -// scanGroupedResults executes a query that is already grouped by -// variant_group_id (not variant_combination_id), mapping each group ID back -// to dimension values via the group mapping. -func (p *PostgresProvider) scanGroupedResults( - ctx context.Context, - sqlQuery string, - args []any, - groupMapping variantGroupMapping, -) (map[string]crstatus.TestStatus, []error) { - return p.scanWithParallelHints(ctx, sqlQuery, args, groupMapping) + return p.queryTestStatusCTE(ctx, reqOptions, reqOptions.VariantOption.IncludeVariants, + gaSpec(release, windowDays)) } // scanWithParallelHints runs the query inside a transaction that enables @@ -391,7 +334,7 @@ func (p *PostgresProvider) scanWithParallelHints( ) (map[string]crstatus.TestStatus, []error) { var result map[string]crstatus.TestStatus txErr := p.dbc.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - if err := tx.Exec("SET LOCAL max_parallel_workers_per_gather = 4; SET LOCAL parallel_setup_cost = 0; SET LOCAL parallel_tuple_cost = 0").Error; err != nil { + if err := tx.Exec(queryPlannerHints).Error; err != nil { return fmt.Errorf("setting parallel query hints: %w", err) } var err error @@ -404,6 +347,220 @@ func (p *PostgresProvider) scanWithParallelHints( return result, nil } +// queryCombinedTestStatus executes a single SQL statement that folds sample +// and base queries into two materialized CTEs, then reads failure and +// placeholder results from each via UNION ALL. This eliminates the concurrent +// partition scans that cause buffer cache contention when sample and base +// queries run in parallel. +func (p *PostgresProvider) queryCombinedTestStatus( + ctx context.Context, + reqOptions reqopts.RequestOptions, +) (baseStatus, sampleStatus map[string]crstatus.TestStatus, errs []error) { + + sampleIncludeVariants := mergeRequestedVariants( + mergeCompareVariants(reqOptions, reqOptions.VariantOption.IncludeVariants), reqOptions) + baseIncludeVariants := mergeRequestedVariants(reqOptions.VariantOption.IncludeVariants, reqOptions) + + sampleRelease := reqOptions.SampleRelease.Name + sampleRange := query.DateRange{ + Start: civil.DateOf(reqOptions.SampleRelease.Start), + End: civil.DateOf(reqOptions.SampleRelease.End).AddDays(1), + } + if err := query.ResolveDateRanges(p.dbc, sampleRelease, &sampleRange); err != nil { + return nil, nil, []error{err} + } + sampleLookupEnd := sampleRange.End.AddDays(-1) + sampleLookupStart := sampleRange.Start.AddDays(-1) + + baseRelease := reqOptions.BaseRelease.Name + baseRange := query.DateRange{ + Start: civil.DateOf(reqOptions.BaseRelease.Start), + End: civil.DateOf(reqOptions.BaseRelease.End).AddDays(1), + } + baseIsGA := p.baseMatchesGAWindow(ctx, baseRelease, baseRange) + + var baseSpec testStatusSpec + if baseIsGA { + baseWindowDays := baseRange.End.AddDays(-1).DaysSince(baseRange.Start) + baseSpec = gaSpec(baseRelease, baseWindowDays) + } else { + if err := query.ResolveDateRanges(p.dbc, baseRelease, &baseRange); err != nil { + return nil, nil, []error{err} + } + baseLookupEnd := baseRange.End.AddDays(-1) + baseLookupStart := baseRange.Start.AddDays(-1) + baseSpec = prefixSumSpec(baseRelease, baseLookupEnd, baseLookupStart, nil) + } + + filters := buildDrilldownFilters(reqOptions) + dbGroupBy := reqOptions.VariantOption.DBGroupBy + minimumFailure := reqOptions.AdvancedOption.MinimumFailure + + sampleVF, err := resolveVariantFilter(ctx, p.dbc, sampleIncludeVariants, dbGroupBy) + if err != nil { + return nil, nil, []error{err} + } + baseVF, err := resolveVariantFilter(ctx, p.dbc, baseIncludeVariants, dbGroupBy) + if err != nil { + return nil, nil, []error{err} + } + if len(sampleVF.lookup) == 0 && len(baseVF.lookup) == 0 { + return map[string]crstatus.TestStatus{}, map[string]crstatus.TestStatus{}, nil + } + + mergedLookup := make(map[uint]map[string]string, len(sampleVF.lookup)+len(baseVF.lookup)) + maps.Copy(mergedLookup, sampleVF.lookup) + maps.Copy(mergedLookup, baseVF.lookup) + groupMapping := buildVariantGroupMapping(mergedLookup) + colMapping := buildColumnGroupMapping(groupMapping.groupToVariants, reqOptions.VariantOption.ColumnGroupBy) + + sampleProwJobJoin := prowJobVariantJoin(sampleVF.variantSubquery) + baseProwJobJoin := prowJobVariantJoin(baseVF.variantSubquery) + + sampleSpec := prefixSumSpec(sampleRelease, sampleLookupEnd, sampleLookupStart, reqOptions.Lifecycles) + sampleInnerSQL, sampleInnerArgs := buildInnerAggregation(sampleSpec, sampleProwJobJoin, sampleVF.filterArgs, filters) + sampleCTE, sampleCTEArgs := buildStatusCTE("sample_agg", sampleInnerSQL, sampleInnerArgs, "cm", filters) + + baseInnerSQL, baseInnerArgs := buildInnerAggregation(baseSpec, baseProwJobJoin, baseVF.filterArgs, filters) + baseCTE, baseCTEArgs := buildStatusCTE("base_agg", baseInnerSQL, baseInnerArgs, "cm", filters) + + // The combined query reuses the branch templates with a "? AS source, " + // prefix so the row scanner can split results into sample and base maps. + sourcePrefix := "? AS source, " + + fullSQL := fmt.Sprintf("WITH vg(vcid, group_id) AS (%s),\ncm(group_id, col_group_id) AS (%s),\n%s,\n%s\n%s\nUNION ALL\n%s\nUNION ALL\n%s\nUNION ALL\n%s", + groupMapping.valuesClause, colMapping.valuesClause, + sampleCTE, baseCTE, + fmt.Sprintf(failureBranchTemplate, sourcePrefix, "sample_agg"), + fmt.Sprintf(placeholderBranchTemplate, sourcePrefix, "sample_agg"), + fmt.Sprintf(failureBranchTemplate, sourcePrefix, "base_agg"), + fmt.Sprintf(placeholderBranchTemplate, sourcePrefix, "base_agg")) + + var allArgs []any + allArgs = append(allArgs, sampleCTEArgs...) + allArgs = append(allArgs, baseCTEArgs...) + allArgs = append(allArgs, "S", minimumFailure) + allArgs = append(allArgs, "S") + allArgs = append(allArgs, "B", minimumFailure) + allArgs = append(allArgs, "B") + + // Execute with parallel hints + var sampleResult, baseResult map[string]crstatus.TestStatus + txErr := p.dbc.DB.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + if txErr := tx.Exec(queryPlannerHints).Error; txErr != nil { + return fmt.Errorf("setting parallel query hints: %w", txErr) + } + + type combinedRow struct { + Source string `gorm:"column:source"` + TestID string `gorm:"column:test_id"` + TestName string `gorm:"column:test_name"` + TestSuite string `gorm:"column:test_suite"` + Component string `gorm:"column:component"` + Capabilities pq.StringArray `gorm:"column:capabilities;type:text[]"` + VariantGroupID int `gorm:"column:variant_group_id"` + TotalCount int `gorm:"column:total_count"` + SuccessCount int `gorm:"column:success_count"` + FlakeCount int `gorm:"column:flake_count"` + LastFailure sql.NullTime `gorm:"column:last_failure"` + } + + var allRows []combinedRow + if qErr := tx.Raw(fullSQL, allArgs...).Scan(&allRows).Error; qErr != nil { + return fmt.Errorf("querying combined test status: %w", qErr) + } + + sampleFailures := make(map[string]crstatus.TestStatus) + samplePlaceholders := make(map[string]crstatus.TestStatus) + baseFailures := make(map[string]crstatus.TestStatus) + basePlaceholders := make(map[string]crstatus.TestStatus) + + scanStart := time.Now() + for _, row := range allRows { + variantMap := groupMapping.groupToVariants[row.VariantGroupID] + key := crtest.KeyWithVariants{TestID: row.TestID, Variants: variantMap} + keyStr := key.Encode() + + ts := buildTestStatus( + row.TestID, row.TestName, row.TestSuite, row.Component, row.Capabilities, + variantMap, row.TotalCount, row.SuccessCount, row.FlakeCount, row.LastFailure, + ) + + isPlaceholder := strings.HasPrefix(row.TestID, "grid:") + if row.Source == "S" { + if isPlaceholder { + samplePlaceholders[keyStr] = ts + } else { + sampleFailures[keyStr] = ts + } + } else { + if isPlaceholder { + basePlaceholders[keyStr] = ts + } else { + baseFailures[keyStr] = ts + } + } + } + log.WithField("rowCount", len(allRows)). + WithField("scanDuration", time.Since(scanStart).String()). + Debug("combined query: row scan complete") + + mergePlaceholders(sampleFailures, samplePlaceholders, "sample") + mergePlaceholders(baseFailures, basePlaceholders, "base") + + sampleResult = sampleFailures + baseResult = baseFailures + return nil + }) + if txErr != nil { + return nil, nil, []error{txErr} + } + + return baseResult, sampleResult, nil +} + +func mergePlaceholders(failures, placeholders map[string]crstatus.TestStatus, label string) { + merged := 0 + for k, v := range placeholders { + if _, exists := failures[k]; !exists { + failures[k] = v + merged++ + } + } + log.WithField("side", label). + WithField("placeholders", len(placeholders)). + WithField("merged", merged). + WithField("failures", len(failures)-merged). + WithField("total", len(failures)). + Debug("combined query: placeholder merge complete") +} + +func buildTestStatus( + testID, testName, testSuite, component string, + capabilities pq.StringArray, + variants map[string]string, + totalCount, successCount, flakeCount int, + lastFailure sql.NullTime, +) crstatus.TestStatus { + ts := crstatus.TestStatus{ + TestID: testID, + TestName: testName, + TestSuite: testSuite, + Component: component, + Capabilities: capabilities, + Variants: variants, + Count: crtest.Count{ + TotalCount: totalCount, + SuccessCount: successCount, + FlakeCount: flakeCount, + }, + } + if lastFailure.Valid { + ts.LastFailure = lastFailure.Time + } + return ts +} + func scanRows( gormDB *gorm.DB, sqlQuery string, @@ -435,30 +592,11 @@ func scanRows( } variantMap := groupMapping.groupToVariants[variantGroupID] - - key := crtest.KeyWithVariants{ - TestID: testID, - Variants: variantMap, - } - keyStr := key.Encode() - - ts := crstatus.TestStatus{ - TestID: testID, - TestName: testName, - TestSuite: testSuite, - Component: component, - Capabilities: capabilities, - Variants: variantMap, - Count: crtest.Count{ - TotalCount: totalCount, - SuccessCount: successCount, - FlakeCount: flakeCount, - }, - } - if lastFailure.Valid { - ts.LastFailure = lastFailure.Time - } - result[keyStr] = ts + key := crtest.KeyWithVariants{TestID: testID, Variants: variantMap} + result[key.Encode()] = buildTestStatus( + testID, testName, testSuite, component, capabilities, + variantMap, totalCount, successCount, flakeCount, lastFailure, + ) } if err := rows.Err(); err != nil { diff --git a/pkg/api/componentreadiness/dataprovider/postgres/provider.go b/pkg/api/componentreadiness/dataprovider/postgres/provider.go index 0d8ba4460..2988a9b90 100644 --- a/pkg/api/componentreadiness/dataprovider/postgres/provider.go +++ b/pkg/api/componentreadiness/dataprovider/postgres/provider.go @@ -326,17 +326,11 @@ func mergeCompareVariants(reqOptions reqopts.RequestOptions, includeVariants map return merged } -func (p *PostgresProvider) QuerySampleTestStatus(ctx context.Context, reqOptions reqopts.RequestOptions, - includeVariants map[string][]string, - start, end time.Time) (map[string]crstatus.TestStatus, []error) { - includeVariants = mergeCompareVariants(reqOptions, includeVariants) - return p.queryTestStatusPrefixSum(ctx, reqOptions, reqOptions.SampleRelease.Name, - reqOptions.Lifecycles, - includeVariants, - query.DateRange{ - Start: civil.DateOf(start), - End: civil.DateOf(end).AddDays(1), - }) +func (p *PostgresProvider) QueryTestStatus( + ctx context.Context, + reqOptions reqopts.RequestOptions, +) (baseStatus, sampleStatus map[string]crstatus.TestStatus, errs []error) { + return p.queryCombinedTestStatus(ctx, reqOptions) } // --- TestDetailsQuerier --- diff --git a/pkg/api/componentreadiness/dataprovider/postgres/variants.go b/pkg/api/componentreadiness/dataprovider/postgres/variants.go index 038211a08..668ff63b2 100644 --- a/pkg/api/componentreadiness/dataprovider/postgres/variants.go +++ b/pkg/api/componentreadiness/dataprovider/postgres/variants.go @@ -44,18 +44,19 @@ func buildVariantFilterClause(includeVariants map[string][]string) (string, []an return strings.Join(clauses, " AND "), args } -// lookupVariantValues queries variant_combinations matching the filter and -// returns a map from variant_combination_id to the extracted variant values -// for each dbGroupBy key. This runs as a small, fast query (~6ms for ~400 -// rows) and the result is used to enrich aggregated rows in Go. +// lookupVariantValues queries variant_combinations matching the given filter +// clause and returns a map from variant_combination_id to the extracted variant +// values for each dbGroupBy key. This runs as a small, fast query (~6ms for +// ~400 rows) and the result is used to enrich aggregated rows in Go. The filter +// clause/args come from buildVariantFilterClause; the caller passes them in so +// the same pure call is not repeated within one code path. func lookupVariantValues( ctx context.Context, dbc *db.DB, - includeVariants map[string][]string, + filterClause string, + filterArgs []any, dbGroupBy sets.Set[string], ) (map[uint]map[string]string, error) { - filterClause, args := buildVariantFilterClause(includeVariants) - query := "SELECT id, variants FROM variant_combinations" if filterClause != "" { query += " WHERE " + filterClause @@ -67,7 +68,7 @@ func lookupVariantValues( } var rows []vcRow - if err := dbc.DB.WithContext(ctx).Raw(query, args...).Scan(&rows).Error; err != nil { + if err := dbc.DB.WithContext(ctx).Raw(query, filterArgs...).Scan(&rows).Error; err != nil { return nil, fmt.Errorf("looking up variant values: %w", err) } @@ -85,6 +86,64 @@ func lookupVariantValues( return result, nil } +// variantFilterResult bundles the outputs needed to join prow_jobs against a +// filtered set of variant_combinations: the resolved variant values (for +// building a group mapping), the bind args for that filter, and a subquery +// selecting matching variant_combination_ids. +type variantFilterResult struct { + lookup map[uint]map[string]string + filterArgs []any + variantSubquery string +} + +// resolveVariantFilter looks up variant_combinations matching includeVariants +// and builds the subquery/args used to join prow_jobs against them. This is +// the shared bundle behind both the standalone (prepareVariantQuery) and +// combined (queryCombinedTestStatus) query paths; keep it as the single place +// that turns includeVariants into a lookup + filter + subquery so the two +// paths can't drift apart. +func resolveVariantFilter( + ctx context.Context, + dbc *db.DB, + includeVariants map[string][]string, + dbGroupBy sets.Set[string], +) (*variantFilterResult, error) { + if includeVariants == nil { + includeVariants = map[string][]string{} + } + + filterClause, filterArgs := buildVariantFilterClause(includeVariants) + + lookup, err := lookupVariantValues(ctx, dbc, filterClause, filterArgs, dbGroupBy) + if err != nil { + return nil, err + } + + variantSubquery := "SELECT vc.id FROM variant_combinations vc" + if filterClause != "" { + variantSubquery += " WHERE " + filterClause + } + + return &variantFilterResult{ + lookup: lookup, + filterArgs: filterArgs, + variantSubquery: variantSubquery, + }, nil +} + +// prowJobVariantJoin builds the prow_jobs join that restricts to jobs whose +// variant_combination_id matches the given subquery, plus the vg join that maps +// each combination to its variant group. variantSubquery is the +// "SELECT vc.id FROM variant_combinations vc [WHERE ...]" produced by +// resolveVariantFilter. Both the standalone and combined query paths share this +// join so their materialized CTEs line up identically. +func prowJobVariantJoin(variantSubquery string) string { + return fmt.Sprintf(`JOIN prow_jobs pj ON pj.id = e.prow_job_id + AND pj.deleted_at IS NULL + AND pj.variant_combination_id IN (%s) + JOIN vg ON vg.vcid = pj.variant_combination_id`, variantSubquery) +} + // variantGroupMapping holds the result of grouping variant_combination_ids by // their dbGroupBy dimension values. Multiple VCIDs that share the same // (Platform, Architecture, Network, ...) combo get the same group ID. diff --git a/pkg/api/componentreadiness/middleware/interface.go b/pkg/api/componentreadiness/middleware/interface.go index 4bd2ab859..6368a6ea6 100644 --- a/pkg/api/componentreadiness/middleware/interface.go +++ b/pkg/api/componentreadiness/middleware/interface.go @@ -13,11 +13,8 @@ import ( // being added to component readiness. It's important to note that the interface covers // both major code paths through, component reports, and test details reports. type Middleware interface { - // Query phase allows middleware to inject additional TestStatus beyond the normal base/sample queries. - // Base and sample status can be submitted using the provided channels for a map of ALL test keys - // (ID plus variant info serialized) to TestStatus. - Query(ctx context.Context, wg *sync.WaitGroup, - baseStatusCh, sampleStatusCh chan map[string]crstatus.TestStatus, errCh chan error) + // Query phase allows middleware to load data needed for the test status report. + Query(ctx context.Context, wg *sync.WaitGroup, errCh chan error) // QueryTestDetails phase allow middleware to load data that will later be used. QueryTestDetails(ctx context.Context, wg *sync.WaitGroup, errCh chan error) diff --git a/pkg/api/componentreadiness/middleware/linkinjector/linkinjector.go b/pkg/api/componentreadiness/middleware/linkinjector/linkinjector.go index 7d7fca6f4..c25e0edbb 100644 --- a/pkg/api/componentreadiness/middleware/linkinjector/linkinjector.go +++ b/pkg/api/componentreadiness/middleware/linkinjector/linkinjector.go @@ -32,7 +32,7 @@ type LinkInjector struct { baseURL string } -func (l *LinkInjector) Query(_ context.Context, _ *sync.WaitGroup, _, _ chan map[string]crstatus.TestStatus, _ chan error) { +func (l *LinkInjector) Query(_ context.Context, _ *sync.WaitGroup, _ chan error) { // unused } diff --git a/pkg/api/componentreadiness/middleware/list.go b/pkg/api/componentreadiness/middleware/list.go index 61ac4036d..ca2ddf06c 100644 --- a/pkg/api/componentreadiness/middleware/list.go +++ b/pkg/api/componentreadiness/middleware/list.go @@ -11,10 +11,10 @@ import ( type List []Middleware -func (l List) Query(ctx context.Context, wg *sync.WaitGroup, baseStatusCh, sampleStatusCh chan map[string]crstatus.TestStatus, errCh chan error) { +func (l List) Query(ctx context.Context, wg *sync.WaitGroup, errCh chan error) { // Invoke the Query phase for each middleware configured: for _, mw := range l { - mw.Query(ctx, wg, baseStatusCh, sampleStatusCh, errCh) + mw.Query(ctx, wg, errCh) } } diff --git a/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go b/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go index bd8c56ed6..a4409cd79 100644 --- a/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go +++ b/pkg/api/componentreadiness/middleware/regressionallowances/regressionallowances.go @@ -40,8 +40,7 @@ type RegressionAllowances struct { regressionGetterFunc func(releaseString string, variant crtest.ColumnIdentification, testID string) *regressionallowances.IntentionalRegression } -func (r *RegressionAllowances) Query(_ context.Context, _ *sync.WaitGroup, - _, _ chan map[string]crstatus.TestStatus, _ chan error) { +func (r *RegressionAllowances) Query(_ context.Context, _ *sync.WaitGroup, _ chan error) { // unused } diff --git a/pkg/api/componentreadiness/middleware/regressiontracker/regressiontracker.go b/pkg/api/componentreadiness/middleware/regressiontracker/regressiontracker.go index 0e3b1fdb7..1c959c2bf 100644 --- a/pkg/api/componentreadiness/middleware/regressiontracker/regressiontracker.go +++ b/pkg/api/componentreadiness/middleware/regressiontracker/regressiontracker.go @@ -66,7 +66,7 @@ type RegressionTracker struct { hasLoadedRegressions bool } -func (r *RegressionTracker) Query(ctx context.Context, wg *sync.WaitGroup, baseStatusCh, sampleStatusCh chan map[string]crstatus.TestStatus, errCh chan error) { +func (r *RegressionTracker) Query(ctx context.Context, wg *sync.WaitGroup, errCh chan error) { err := r.ensureRegressionsLoaded() if err != nil { utils.EnqueueAsync(wg, errCh, err) diff --git a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go index bbce7994e..a2eb52f2d 100644 --- a/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go +++ b/pkg/api/componentreadiness/middleware/releasefallback/releasefallback.go @@ -70,8 +70,7 @@ func (r *ReleaseFallback) Analyze(testID string, variants map[string]string, rep return nil } -func (r *ReleaseFallback) Query(ctx context.Context, wg *sync.WaitGroup, - _, _ chan map[string]crstatus.TestStatus, errCh chan error) { +func (r *ReleaseFallback) Query(ctx context.Context, wg *sync.WaitGroup, errCh chan error) { wg.Add(1) go func() { defer wg.Done() diff --git a/test/integration/component_readiness_test.go b/test/integration/component_readiness_test.go index b92dc66a2..24eca16bb 100644 --- a/test/integration/component_readiness_test.go +++ b/test/integration/component_readiness_test.go @@ -2,6 +2,7 @@ package integration import ( "context" + "maps" "math/big" "testing" "time" @@ -15,6 +16,7 @@ import ( componentreadiness "github.com/openshift/sippy/pkg/api/componentreadiness" "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider/postgres" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" + crtype "github.com/openshift/sippy/pkg/apis/api/componentreport" "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" @@ -257,7 +259,7 @@ func seedCRData(t *testing.T, dbc *db.DB) crSeedData { } } -func TestQuerySampleTestStatus(t *testing.T) { +func TestQueryTestStatus_SampleResults(t *testing.T) { t.Run("basic aggregation", func(t *testing.T) { dbc := crTestDB(t) release := "4.16" @@ -270,8 +272,8 @@ func TestQuerySampleTestStatus(t *testing.T) { "Network": {"ovn", "sdn"}, } - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) require.NotEmpty(t, result) @@ -322,8 +324,8 @@ func TestQuerySampleTestStatus(t *testing.T) { "Platform": {"aws"}, } - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) for _, ts := range result { @@ -370,9 +372,8 @@ func TestQuerySampleTestStatus(t *testing.T) { opts := defaultReqOptions(release) opts.VariantOption.DBGroupBy = sets.New[string]("Platform") - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) // Both jobs collapse to the same group (Platform:aws), aggregating counts @@ -400,8 +401,8 @@ func TestQuerySampleTestStatus(t *testing.T) { "Network": {"ovn", "sdn"}, } - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) // test2/jobGCP has 3 failures (>= 2), should appear via failure query @@ -434,9 +435,8 @@ func TestQuerySampleTestStatus(t *testing.T) { provider := postgres.NewPostgresProvider(dbc, nil) opts := defaultReqOptions(release) - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) assert.Empty(t, result) }) @@ -456,8 +456,8 @@ func TestQuerySampleTestStatus(t *testing.T) { "Network": {"ovn", "sdn"}, } - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) // Only aws+ovn VC should match @@ -496,8 +496,8 @@ func TestQuerySampleTestStatus(t *testing.T) { "Network": {"ovn", "sdn"}, } - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) // Only test1 entries should appear (excluding grid placeholders) @@ -525,8 +525,8 @@ func TestQuerySampleTestStatus(t *testing.T) { "Network": {"ovn", "sdn"}, } - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) // test3 has RBAC capability but 0 failures, so with MinimumFailure=1 @@ -566,9 +566,8 @@ func TestQuerySampleTestStatus(t *testing.T) { provider := postgres.NewPostgresProvider(dbc, nil) opts := defaultReqOptions(sampleRelease) - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) key := crtest.KeyWithVariants{ @@ -609,9 +608,8 @@ func TestQuerySampleTestStatus(t *testing.T) { provider := postgres.NewPostgresProvider(dbc, nil) opts := defaultReqOptions(release) - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) for _, ts := range result { @@ -648,9 +646,8 @@ func TestQuerySampleTestStatus(t *testing.T) { provider := postgres.NewPostgresProvider(dbc, nil) opts := defaultReqOptions(release) - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) keyA := crtest.KeyWithVariants{ @@ -712,8 +709,8 @@ func TestQuerySampleTestStatus(t *testing.T) { "Topology": {"ha"}, } - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) nonPlaceholders := filterPlaceholders(result) @@ -749,9 +746,8 @@ func TestQuerySampleTestStatus(t *testing.T) { provider := postgres.NewPostgresProvider(dbc, nil) opts := defaultReqOptions(release) - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) key := crtest.KeyWithVariants{ @@ -796,9 +792,8 @@ func TestQuerySampleTestStatus(t *testing.T) { opts := defaultReqOptions(release) opts.VariantOption.DBGroupBy = sets.New[string]("Platform") - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) key := crtest.KeyWithVariants{ @@ -842,9 +837,8 @@ func TestQuerySampleTestStatus(t *testing.T) { opts := defaultReqOptions(release) opts.VariantOption.DBGroupBy = sets.New[string]("Platform") - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) key := crtest.KeyWithVariants{ @@ -878,9 +872,8 @@ func TestQuerySampleTestStatus(t *testing.T) { provider := postgres.NewPostgresProvider(dbc, nil) opts := defaultReqOptions(release) - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) key := crtest.KeyWithVariants{ @@ -911,9 +904,8 @@ func TestQuerySampleTestStatus(t *testing.T) { provider := postgres.NewPostgresProvider(dbc, nil) opts := defaultReqOptions(release) - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) key := crtest.KeyWithVariants{ @@ -943,9 +935,8 @@ func TestQuerySampleTestStatus(t *testing.T) { provider := postgres.NewPostgresProvider(dbc, nil) opts := defaultReqOptions(release) - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) key := crtest.KeyWithVariants{ @@ -981,9 +972,8 @@ func TestQuerySampleTestStatus(t *testing.T) { opts := defaultReqOptions(release) opts.AdvancedOption.MinimumFailure = 1 - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) key := crtest.KeyWithVariants{ @@ -1014,8 +1004,8 @@ func TestQuerySampleTestStatus(t *testing.T) { "Network": {"ovn", "sdn"}, } - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) // Should get exactly test1 on aws+ovn @@ -1130,9 +1120,7 @@ func TestQueryTestStatus_DifferentBaseAndSampleReleases(t *testing.T) { assert.Equal(t, 20, baseTS.TotalCount, "base should reflect baseRelease data only") assert.Equal(t, 15, baseTS.SuccessCount) - sampleResult, errs := provider.QuerySampleTestStatus(context.Background(), opts, - opts.VariantOption.IncludeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + _, sampleResult, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) sampleTS, ok := sampleResult[key.Encode()] @@ -2481,9 +2469,8 @@ func TestMultipleTestsInSameComponent(t *testing.T) { provider := postgres.NewPostgresProvider(dbc, nil) opts := defaultReqOptions(release) - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) keyA := crtest.KeyWithVariants{ @@ -2717,8 +2704,8 @@ func TestTestExistsInBaseButNotSample(t *testing.T) { baseResult, errs := provider.QueryBaseTestStatus(context.Background(), opts) require.Empty(t, errs) - sampleResult, errs := provider.QuerySampleTestStatus(context.Background(), opts, - includeVariants, opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, sampleResult, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) // baseOnlyTest: should be in base results, absent from sample @@ -2774,9 +2761,8 @@ func TestSingleDayPeriod(t *testing.T) { opts.SampleRelease.Start = time.Date(2024, 6, 10, 0, 0, 0, 0, time.UTC) opts.SampleRelease.End = time.Date(2024, 6, 11, 0, 0, 0, 0, time.UTC) - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) key := crtest.KeyWithVariants{ @@ -2829,9 +2815,8 @@ func TestMinimumFailureWithCapabilityFilter(t *testing.T) { Capability: "PVC", }} - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) nonPlaceholders := filterPlaceholders(result) @@ -2879,9 +2864,8 @@ func TestDrillDownBySecondaryCapability(t *testing.T) { Capability: "IPv4", }} - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, - map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) nonPlaceholders := filterPlaceholders(result) @@ -2897,6 +2881,96 @@ func TestDrillDownBySecondaryCapability(t *testing.T) { } } +func TestCapabilitiesArrayOverlapFilter(t *testing.T) { + dbc := crTestDB(t) + release := "4.16" + + vc := createVariantCombination(t, dbc, []string{"Platform:aws", "Network:ovn"}) + job := createProwJobWithVC(t, dbc, "periodic-e2e-aws-capoverlap", release, vc) + + testPVC := intutil.CreateTest(t, dbc, "openshift-tests:[sig-storage] PVC overlap test") + testIPv4 := intutil.CreateTest(t, dbc, "openshift-tests:[sig-network] IPv4 overlap test") + testRBAC := intutil.CreateTest(t, dbc, "openshift-tests:[sig-auth] RBAC overlap test") + suite := intutil.CreateSuite(t, dbc, "openshift-tests-capoverlap") + + createTestOwnership(t, dbc, testPVC.ID, &suite.ID, "openshift-tests:pvc-overlap", "Storage", []string{"PVC", "IPv4"}) + createTestOwnership(t, dbc, testIPv4.ID, &suite.ID, "openshift-tests:ipv4-overlap", "Networking", []string{"IPv4", "Services"}) + createTestOwnership(t, dbc, testRBAC.ID, &suite.ID, "openshift-tests:rbac-overlap", "Authentication", []string{"RBAC"}) + + startMinus1 := civil.Date{Year: 2024, Month: 5, Day: 31} + endMinus1 := civil.Date{Year: 2024, Month: 6, Day: 14} + + for _, testModel := range []models.Test{testPVC, testIPv4, testRBAC} { + createCumulativeSummary(t, dbc, startMinus1, release, testModel.ID, job.ID, suite.ID, 100, 90, 5) + createCumulativeSummary(t, dbc, endMinus1, release, testModel.ID, job.ID, suite.ID, 110, 98, 6) + } + + provider := postgres.NewPostgresProvider(dbc, nil) + + t.Run("single capability matches tests that contain it", func(t *testing.T) { + opts := defaultReqOptions(release) + opts.Capabilities = []string{"IPv4"} + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) + require.Empty(t, errs) + + nonPlaceholders := filterPlaceholders(result) + for _, ts := range nonPlaceholders { + assert.Contains(t, ts.Capabilities, "IPv4", + "only tests with IPv4 capability should appear, got %v for %s", ts.Capabilities, ts.TestID) + } + foundPVC := false + foundIPv4 := false + for _, ts := range nonPlaceholders { + switch ts.TestID { + case "openshift-tests:pvc-overlap": + foundPVC = true + case "openshift-tests:ipv4-overlap": + foundIPv4 = true + case "openshift-tests:rbac-overlap": + t.Error("RBAC-only test should not appear with IPv4 filter") + } + } + assert.True(t, foundPVC, "PVC test has IPv4 in its capabilities and should appear") + assert.True(t, foundIPv4, "IPv4 test should appear") + }) + + t.Run("multiple capabilities match tests overlapping any", func(t *testing.T) { + opts := defaultReqOptions(release) + opts.Capabilities = []string{"PVC", "RBAC"} + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) + require.Empty(t, errs) + + nonPlaceholders := filterPlaceholders(result) + foundPVC := false + foundRBAC := false + for _, ts := range nonPlaceholders { + switch ts.TestID { + case "openshift-tests:pvc-overlap": + foundPVC = true + case "openshift-tests:rbac-overlap": + foundRBAC = true + case "openshift-tests:ipv4-overlap": + t.Error("IPv4-only test (no PVC or RBAC) should not appear") + } + } + assert.True(t, foundPVC, "PVC test should appear (overlaps PVC)") + assert.True(t, foundRBAC, "RBAC test should appear (overlaps RBAC)") + }) + + t.Run("non-matching capability returns no non-placeholder results", func(t *testing.T) { + opts := defaultReqOptions(release) + opts.Capabilities = []string{"Nonexistent"} + opts.VariantOption.IncludeVariants = map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}} + _, result, errs := provider.QueryTestStatus(context.Background(), opts) + require.Empty(t, errs) + + nonPlaceholders := filterPlaceholders(result) + assert.Empty(t, nonPlaceholders, "no tests should match a nonexistent capability") + }) +} + func TestMixedLifecycleRowsProduceCorrectCounts(t *testing.T) { dbc := crTestDB(t) release := "4.16" @@ -2925,8 +2999,8 @@ func TestMixedLifecycleRowsProduceCorrectCounts(t *testing.T) { "Network": {"ovn"}, } - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) require.NotEmpty(t, result) @@ -2990,8 +3064,8 @@ func TestLifecycleFilterExcludesInformingFromSample(t *testing.T) { opts := defaultReqOptions(release) opts.Lifecycles = []string{"blocking"} - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) require.NotEmpty(t, result) @@ -3024,8 +3098,8 @@ func TestLifecycleFilterExcludesInformingFromSample(t *testing.T) { t.Run("sample without lifecycle filter includes all", func(t *testing.T) { opts := defaultReqOptions(release) - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) require.NotEmpty(t, result) @@ -3042,8 +3116,8 @@ func TestLifecycleFilterExcludesInformingFromSample(t *testing.T) { opts := defaultReqOptions(release) opts.Lifecycles = []string{"informing"} - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) require.NotEmpty(t, result) @@ -3060,8 +3134,8 @@ func TestLifecycleFilterExcludesInformingFromSample(t *testing.T) { opts := defaultReqOptions(release) opts.Lifecycles = []string{"blocking", "informing"} - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) require.NotEmpty(t, result) @@ -3112,8 +3186,8 @@ func TestInformingOnlyTestExcludedFromSamplePlaceholders(t *testing.T) { opts := defaultReqOptions(release) opts.Lifecycles = []string{"blocking"} - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) keyBoth := crtest.KeyWithVariants{ @@ -3178,8 +3252,8 @@ func TestCrossCompareWithLifecycleFilter(t *testing.T) { "Topology": {"ha"}, } - result, errs := provider.QuerySampleTestStatus(context.Background(), opts, includeVariants, - opts.SampleRelease.Start, opts.SampleRelease.End) + opts.VariantOption.IncludeVariants = includeVariants + _, result, errs := provider.QueryTestStatus(context.Background(), opts) require.Empty(t, errs) nonPlaceholders := filterPlaceholders(result) @@ -3444,6 +3518,747 @@ func TestTestDetailsReport_FlakeAsFailure(t *testing.T) { sampleFalse.SuccessRate, sampleTrue.SuccessRate) } +// --- GenerateReport Tests --- +// +// These tests exercise the full GenerateReport pipeline, which uses the combined +// query path (QueryTestStatus) when the postgres provider is used. Each +// test seeds both base and sample data, calls GenerateReport, and asserts on the +// resulting ComponentReport structure. + +func TestGenerateReport_NoRegression(t *testing.T) { + dbc := crTestDB(t) + release := "4.16" + + vc := createVariantCombination(t, dbc, []string{"Platform:aws", "Network:ovn"}) + job := createProwJobWithVC(t, dbc, "periodic-e2e-aws-noreg", release, vc) + test := intutil.CreateTest(t, dbc, "openshift-tests:[sig-storage] no regression test") + suite := intutil.CreateSuite(t, dbc, "openshift-tests-noreg") + createTestOwnership(t, dbc, test.ID, &suite.ID, "openshift-tests:noreg", "Storage", []string{"PVC"}) + + baseLookupStart := civil.Date{Year: 2024, Month: 5, Day: 14} + baseLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 1} + sampleLookupStart := civil.Date{Year: 2024, Month: 5, Day: 31} + sampleLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 14} + + // Base: 100 runs, 90 success, 0 flakes → 90% pass rate + createCumulativeSummary(t, dbc, baseLookupStart, release, test.ID, job.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, release, test.ID, job.ID, suite.ID, 100, 90, 0) + // Sample: 100 runs, 90 success, 0 flakes → same pass rate as base + createCumulativeSummary(t, dbc, sampleLookupStart, release, test.ID, job.ID, suite.ID, 100, 90, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, release, test.ID, job.ID, suite.ID, 200, 180, 0) + + provider := postgres.NewPostgresProvider(dbc, nil) + opts := defaultReqOptions(release) + opts.AdvancedOption.Confidence = 95 + opts.IncludeAllTests = true + opts.VariantOption.IncludeVariants = map[string][]string{ + "Platform": {"aws"}, + "Network": {"ovn"}, + } + + generator := componentreadiness.NewComponentReportGenerator(provider, opts, dbc, nil, "") + report, errs := generator.GenerateReport(context.Background()) + require.Empty(t, errs) + + row := findReportRow(t, report, "Storage") + col := findReportColumn(t, row, map[string]string{"Platform": "aws"}) + assert.GreaterOrEqual(t, int(col.Status), int(crtest.NotSignificant), "same pass rate should not be a regression") + assert.Empty(t, col.RegressedTests) + tests := filterReportPlaceholders(col.AllTests) + require.Len(t, tests, 1) + assert.Equal(t, 100, tests[0].SampleStats.Total()) + assert.Equal(t, 90, tests[0].SampleStats.SuccessCount) + require.NotNil(t, tests[0].BaseStats) + assert.Equal(t, 100, tests[0].BaseStats.Total()) + assert.Equal(t, 90, tests[0].BaseStats.SuccessCount) +} + +func TestGenerateReport_RegressionDetected(t *testing.T) { + dbc := crTestDB(t) + release := "4.16" + + vc := createVariantCombination(t, dbc, []string{"Platform:aws", "Network:ovn"}) + job := createProwJobWithVC(t, dbc, "periodic-e2e-aws-reg", release, vc) + test := intutil.CreateTest(t, dbc, "openshift-tests:[sig-storage] regression test") + suite := intutil.CreateSuite(t, dbc, "openshift-tests-reg") + createTestOwnership(t, dbc, test.ID, &suite.ID, "openshift-tests:reg-test", "Storage", []string{"PVC"}) + + baseLookupStart := civil.Date{Year: 2024, Month: 5, Day: 14} + baseLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 1} + sampleLookupStart := civil.Date{Year: 2024, Month: 5, Day: 31} + sampleLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 14} + + // Base: 200 runs, 190 success → 95% pass rate + createCumulativeSummary(t, dbc, baseLookupStart, release, test.ID, job.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, release, test.ID, job.ID, suite.ID, 200, 190, 0) + // Sample: 200 runs, 140 success → 70% pass rate (25% drop, >15% threshold for extreme) + createCumulativeSummary(t, dbc, sampleLookupStart, release, test.ID, job.ID, suite.ID, 200, 190, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, release, test.ID, job.ID, suite.ID, 400, 330, 0) + + provider := postgres.NewPostgresProvider(dbc, nil) + opts := defaultReqOptions(release) + opts.AdvancedOption.Confidence = 95 + opts.VariantOption.IncludeVariants = map[string][]string{ + "Platform": {"aws"}, + "Network": {"ovn"}, + } + + generator := componentreadiness.NewComponentReportGenerator(provider, opts, dbc, nil, "") + report, errs := generator.GenerateReport(context.Background()) + require.Empty(t, errs) + + row := findReportRow(t, report, "Storage") + col := findReportColumn(t, row, map[string]string{"Platform": "aws"}) + assert.Equal(t, crtest.ExtremeRegression, col.Status, "95%→70% should be extreme regression") + require.Len(t, col.RegressedTests, 1) + assert.Equal(t, crtest.ExtremeRegression, col.RegressedTests[0].ReportStatus) + assert.Equal(t, 200, col.RegressedTests[0].SampleStats.Total()) + require.NotNil(t, col.RegressedTests[0].BaseStats) + assert.Equal(t, 200, col.RegressedTests[0].BaseStats.Total()) +} + +func TestGenerateReport_DifferentReleases(t *testing.T) { + dbc := crTestDB(t) + baseRelease := "4.16" + sampleRelease := "4.17" + + vc := createVariantCombination(t, dbc, []string{"Platform:aws", "Network:ovn"}) + baseJob := createProwJobWithVC(t, dbc, "periodic-e2e-aws-base-xr-report", baseRelease, vc) + sampleJob := createProwJobWithVC(t, dbc, "periodic-e2e-aws-sample-xr-report", sampleRelease, vc) + + test := intutil.CreateTest(t, dbc, "openshift-tests:[sig-storage] cross-release report test") + suite := intutil.CreateSuite(t, dbc, "openshift-tests-xr-report") + createTestOwnership(t, dbc, test.ID, &suite.ID, "openshift-tests:xr-report", "Storage", []string{"PVC"}) + + baseLookupStart := civil.Date{Year: 2024, Month: 5, Day: 14} + baseLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 1} + sampleLookupStart := civil.Date{Year: 2024, Month: 5, Day: 31} + sampleLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 14} + + // Base: 100 runs, 90 success (baseRelease) + createCumulativeSummary(t, dbc, baseLookupStart, baseRelease, test.ID, baseJob.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, baseRelease, test.ID, baseJob.ID, suite.ID, 100, 90, 0) + + // Sample: 100 runs, 88 success (sampleRelease) + createCumulativeSummary(t, dbc, sampleLookupStart, sampleRelease, test.ID, sampleJob.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, sampleRelease, test.ID, sampleJob.ID, suite.ID, 100, 88, 0) + + // Distractor data in baseRelease at sample dates should not leak into sample results + createCumulativeSummary(t, dbc, sampleLookupStart, baseRelease, test.ID, baseJob.ID, suite.ID, 100, 90, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, baseRelease, test.ID, baseJob.ID, suite.ID, 1000, 900, 0) + + provider := postgres.NewPostgresProvider(dbc, nil) + opts := reqopts.RequestOptions{ + BaseRelease: reqopts.Release{ + Name: baseRelease, + Start: time.Date(2024, 5, 15, 0, 0, 0, 0, time.UTC), + End: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + }, + SampleRelease: reqopts.Release{ + Name: sampleRelease, + Start: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + End: time.Date(2024, 6, 15, 0, 0, 0, 0, time.UTC), + }, + VariantOption: reqopts.Variants{ + DBGroupBy: sets.New[string]("Platform", "Network"), + ColumnGroupBy: sets.New[string]("Platform"), + IncludeVariants: map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, + }, + AdvancedOption: reqopts.Advanced{ + MinimumFailure: 1, + Confidence: 95, + }, + IncludeAllTests: true, + } + + generator := componentreadiness.NewComponentReportGenerator(provider, opts, dbc, nil, "") + report, errs := generator.GenerateReport(context.Background()) + require.Empty(t, errs) + + row := findReportRow(t, report, "Storage") + col := findReportColumn(t, row, map[string]string{"Platform": "aws"}) + tests := filterReportPlaceholders(col.AllTests) + require.Len(t, tests, 1) + assert.Equal(t, 100, tests[0].SampleStats.Total(), "sample should use sampleRelease data only") + assert.Equal(t, 88, tests[0].SampleStats.SuccessCount) + require.NotNil(t, tests[0].BaseStats) + assert.Equal(t, 100, tests[0].BaseStats.Total(), "base should use baseRelease data only") + assert.Equal(t, 90, tests[0].BaseStats.SuccessCount) +} + +func TestGenerateReport_MissingSampleAndBasis(t *testing.T) { + dbc := crTestDB(t) + baseRelease := "4.16" + sampleRelease := "4.17" + + vc := createVariantCombination(t, dbc, []string{"Platform:aws", "Network:ovn"}) + baseJob := createProwJobWithVC(t, dbc, "periodic-base-missing-report", baseRelease, vc) + sampleJob := createProwJobWithVC(t, dbc, "periodic-sample-missing-report", sampleRelease, vc) + + baseOnlyTest := intutil.CreateTest(t, dbc, "openshift-tests:[sig-storage] base-only report test") + sampleOnlyTest := intutil.CreateTest(t, dbc, "openshift-tests:[sig-network] sample-only report test") + sharedTest := intutil.CreateTest(t, dbc, "openshift-tests:[sig-auth] shared report test") + suite := intutil.CreateSuite(t, dbc, "openshift-tests-missing-report") + + createTestOwnership(t, dbc, baseOnlyTest.ID, &suite.ID, "openshift-tests:base-only-rep", "OldComponent", []string{"Legacy"}) + createTestOwnership(t, dbc, sampleOnlyTest.ID, &suite.ID, "openshift-tests:sample-only-rep", "NewComponent", []string{"Fresh"}) + createTestOwnership(t, dbc, sharedTest.ID, &suite.ID, "openshift-tests:shared-rep", "SharedComponent", []string{"Common"}) + + baseLookupStart := civil.Date{Year: 2024, Month: 5, Day: 14} + baseLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 1} + sampleLookupStart := civil.Date{Year: 2024, Month: 5, Day: 31} + sampleLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 14} + + // baseOnlyTest: data only in base release + createCumulativeSummary(t, dbc, baseLookupStart, baseRelease, baseOnlyTest.ID, baseJob.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, baseRelease, baseOnlyTest.ID, baseJob.ID, suite.ID, 100, 90, 0) + + // sampleOnlyTest: data only in sample release + createCumulativeSummary(t, dbc, sampleLookupStart, sampleRelease, sampleOnlyTest.ID, sampleJob.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, sampleRelease, sampleOnlyTest.ID, sampleJob.ID, suite.ID, 100, 90, 0) + + // sharedTest: data in both releases + createCumulativeSummary(t, dbc, baseLookupStart, baseRelease, sharedTest.ID, baseJob.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, baseRelease, sharedTest.ID, baseJob.ID, suite.ID, 100, 90, 0) + createCumulativeSummary(t, dbc, sampleLookupStart, sampleRelease, sharedTest.ID, sampleJob.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, sampleRelease, sharedTest.ID, sampleJob.ID, suite.ID, 100, 90, 0) + + provider := postgres.NewPostgresProvider(dbc, nil) + opts := reqopts.RequestOptions{ + BaseRelease: reqopts.Release{ + Name: baseRelease, + Start: time.Date(2024, 5, 15, 0, 0, 0, 0, time.UTC), + End: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + }, + SampleRelease: reqopts.Release{ + Name: sampleRelease, + Start: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + End: time.Date(2024, 6, 15, 0, 0, 0, 0, time.UTC), + }, + VariantOption: reqopts.Variants{ + DBGroupBy: sets.New[string]("Platform", "Network"), + ColumnGroupBy: sets.New[string]("Platform"), + IncludeVariants: map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, + }, + AdvancedOption: reqopts.Advanced{ + MinimumFailure: 1, + Confidence: 95, + }, + } + + generator := componentreadiness.NewComponentReportGenerator(provider, opts, dbc, nil, "") + report, errs := generator.GenerateReport(context.Background()) + require.Empty(t, errs) + + // Test that exists in base but not sample should show MissingSample + oldRow := findReportRow(t, report, "OldComponent") + oldCol := findReportColumn(t, oldRow, map[string]string{"Platform": "aws"}) + assert.Equal(t, crtest.MissingSample, oldCol.Status) + assert.Empty(t, oldCol.RegressedTests, "MissingSample should not appear in regressed tests") + + // Test that exists in sample but not base should show MissingBasis + newRow := findReportRow(t, report, "NewComponent") + newCol := findReportColumn(t, newRow, map[string]string{"Platform": "aws"}) + assert.Equal(t, crtest.MissingBasis, newCol.Status) + + // Test that exists in both should be assessed normally + sharedRow := findReportRow(t, report, "SharedComponent") + sharedCol := findReportColumn(t, sharedRow, map[string]string{"Platform": "aws"}) + assert.NotEqual(t, crtest.MissingBasis, sharedCol.Status, "shared test has base data") + assert.NotEqual(t, crtest.MissingSample, sharedCol.Status, "shared test has sample data") + assert.GreaterOrEqual(t, int(sharedCol.Status), int(crtest.NotSignificant)) +} + +func TestGenerateReport_VariantGroupingCollapse(t *testing.T) { + dbc := crTestDB(t) + release := "4.16" + + vc1 := createVariantCombination(t, dbc, []string{"Platform:aws", "Topology:ha"}) + vc2 := createVariantCombination(t, dbc, []string{"Platform:aws", "Topology:single"}) + job1 := createProwJobWithVC(t, dbc, "periodic-e2e-aws-ha-grp", release, vc1) + job2 := createProwJobWithVC(t, dbc, "periodic-e2e-aws-single-grp", release, vc2) + + test := intutil.CreateTest(t, dbc, "openshift-tests:[sig-storage] grouping report test") + suite := intutil.CreateSuite(t, dbc, "openshift-tests-grp-report") + createTestOwnership(t, dbc, test.ID, &suite.ID, "openshift-tests:grp-report", "Storage", []string{"PVC"}) + + baseLookupStart := civil.Date{Year: 2024, Month: 5, Day: 14} + baseLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 1} + sampleLookupStart := civil.Date{Year: 2024, Month: 5, Day: 31} + sampleLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 14} + + // job1 (ha): base 60 runs/55 success, sample 50 runs/45 success + createCumulativeSummary(t, dbc, baseLookupStart, release, test.ID, job1.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, release, test.ID, job1.ID, suite.ID, 60, 55, 0) + createCumulativeSummary(t, dbc, sampleLookupStart, release, test.ID, job1.ID, suite.ID, 60, 55, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, release, test.ID, job1.ID, suite.ID, 110, 100, 0) + + // job2 (single): base 40 runs/35 success, sample 50 runs/46 success + createCumulativeSummary(t, dbc, baseLookupStart, release, test.ID, job2.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, release, test.ID, job2.ID, suite.ID, 40, 35, 0) + createCumulativeSummary(t, dbc, sampleLookupStart, release, test.ID, job2.ID, suite.ID, 40, 35, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, release, test.ID, job2.ID, suite.ID, 90, 81, 0) + + provider := postgres.NewPostgresProvider(dbc, nil) + opts := defaultReqOptions(release) + opts.AdvancedOption.Confidence = 95 + opts.IncludeAllTests = true + opts.VariantOption.DBGroupBy = sets.New[string]("Platform") + opts.VariantOption.ColumnGroupBy = sets.New[string]("Platform") + opts.VariantOption.IncludeVariants = map[string][]string{ + "Platform": {"aws"}, + } + + generator := componentreadiness.NewComponentReportGenerator(provider, opts, dbc, nil, "") + report, errs := generator.GenerateReport(context.Background()) + require.Empty(t, errs) + + row := findReportRow(t, report, "Storage") + col := findReportColumn(t, row, map[string]string{"Platform": "aws"}) + tests := filterReportPlaceholders(col.AllTests) + require.Len(t, tests, 1, "both VCs should collapse into one test entry") + + // Sample: 50+50=100 runs, 45+46=91 success + assert.Equal(t, 100, tests[0].SampleStats.Total(), "sample runs should aggregate across VCs") + assert.Equal(t, 91, tests[0].SampleStats.SuccessCount) + // Base: 60+40=100 runs, 55+35=90 success + require.NotNil(t, tests[0].BaseStats) + assert.Equal(t, 100, tests[0].BaseStats.Total(), "base runs should aggregate across VCs") + assert.Equal(t, 90, tests[0].BaseStats.SuccessCount) +} + +func TestGenerateReport_CrossVariantCompare(t *testing.T) { + dbc := crTestDB(t) + release := "4.16" + + vcHA := createVariantCombination(t, dbc, []string{"Platform:aws", "Topology:ha"}) + vcSingle := createVariantCombination(t, dbc, []string{"Platform:aws", "Topology:single"}) + jobHA := createProwJobWithVC(t, dbc, "periodic-e2e-aws-ha-xv-report", release, vcHA) + jobSingle := createProwJobWithVC(t, dbc, "periodic-e2e-aws-single-xv-report", release, vcSingle) + + test := intutil.CreateTest(t, dbc, "openshift-tests:[sig-storage] cross-variant report test") + suite := intutil.CreateSuite(t, dbc, "openshift-tests-xv-report") + createTestOwnership(t, dbc, test.ID, &suite.ID, "openshift-tests:xv-report", "Storage", []string{"PVC"}) + + baseLookupStart := civil.Date{Year: 2024, Month: 5, Day: 14} + baseLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 1} + sampleLookupStart := civil.Date{Year: 2024, Month: 5, Day: 31} + sampleLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 14} + + // HA job (used for base side): 100 runs, 90 success in base period + createCumulativeSummary(t, dbc, baseLookupStart, release, test.ID, jobHA.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, release, test.ID, jobHA.ID, suite.ID, 100, 90, 0) + createCumulativeSummary(t, dbc, sampleLookupStart, release, test.ID, jobHA.ID, suite.ID, 100, 90, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, release, test.ID, jobHA.ID, suite.ID, 200, 180, 0) + + // Single job (used for sample side): 80 runs, 72 success in sample period + createCumulativeSummary(t, dbc, baseLookupStart, release, test.ID, jobSingle.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, release, test.ID, jobSingle.ID, suite.ID, 80, 72, 0) + createCumulativeSummary(t, dbc, sampleLookupStart, release, test.ID, jobSingle.ID, suite.ID, 80, 72, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, release, test.ID, jobSingle.ID, suite.ID, 160, 144, 0) + + provider := postgres.NewPostgresProvider(dbc, nil) + opts := defaultReqOptions(release) + opts.AdvancedOption.Confidence = 95 + opts.IncludeAllTests = true + // Production cross-compare config: cross-compared variant NOT in DBGroupBy + opts.VariantOption.DBGroupBy = sets.New[string]("Platform") + opts.VariantOption.ColumnGroupBy = sets.New[string]("Platform") + opts.VariantOption.VariantCrossCompare = []string{"Topology"} + opts.VariantOption.CompareVariants = map[string][]string{"Topology": {"single"}} + opts.VariantOption.IncludeVariants = map[string][]string{ + "Platform": {"aws"}, + "Topology": {"ha"}, + } + + generator := componentreadiness.NewComponentReportGenerator(provider, opts, dbc, nil, "") + report, errs := generator.GenerateReport(context.Background()) + require.Empty(t, errs) + + row := findReportRow(t, report, "Storage") + col := findReportColumn(t, row, map[string]string{"Platform": "aws"}) + tests := filterReportPlaceholders(col.AllTests) + require.Len(t, tests, 1) + + // Sample uses single variant: 80 runs, 72 success from sample period + assert.Equal(t, 80, tests[0].SampleStats.Total()) + assert.Equal(t, 72, tests[0].SampleStats.SuccessCount) + // Base uses ha variant: 100 runs, 90 success from base period + require.NotNil(t, tests[0].BaseStats) + assert.Equal(t, 100, tests[0].BaseStats.Total()) + assert.Equal(t, 90, tests[0].BaseStats.SuccessCount) +} + +func TestGenerateReport_DisjointVariantsBetweenBaseAndSample(t *testing.T) { + dbc := crTestDB(t) + baseRelease := "4.16" + sampleRelease := "4.17" + + // Base has aws+ovn only; sample has aws+ovn AND gcp+sdn. + // The combined query merges variant lookups from both sides. This test + // verifies that the merge produces correct results when each side has + // variant combinations the other lacks. + vcAWS := createVariantCombination(t, dbc, []string{"Platform:aws", "Network:ovn"}) + vcGCP := createVariantCombination(t, dbc, []string{"Platform:gcp", "Network:sdn"}) + + baseJob := createProwJobWithVC(t, dbc, "periodic-disjoint-base-aws", baseRelease, vcAWS) + sampleJobAWS := createProwJobWithVC(t, dbc, "periodic-disjoint-sample-aws", sampleRelease, vcAWS) + sampleJobGCP := createProwJobWithVC(t, dbc, "periodic-disjoint-sample-gcp", sampleRelease, vcGCP) + + test := intutil.CreateTest(t, dbc, "openshift-tests:[sig-storage] disjoint variants test") + suite := intutil.CreateSuite(t, dbc, "openshift-tests-disjoint") + createTestOwnership(t, dbc, test.ID, &suite.ID, "openshift-tests:disjoint", "Storage", []string{"PVC"}) + + baseLookupStart := civil.Date{Year: 2024, Month: 5, Day: 14} + baseLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 1} + sampleLookupStart := civil.Date{Year: 2024, Month: 5, Day: 31} + sampleLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 14} + + // Base: aws+ovn only, 100 runs, 90 success + createCumulativeSummary(t, dbc, baseLookupStart, baseRelease, test.ID, baseJob.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, baseRelease, test.ID, baseJob.ID, suite.ID, 100, 90, 0) + + // Sample: aws+ovn 80 runs, 72 success + createCumulativeSummary(t, dbc, sampleLookupStart, sampleRelease, test.ID, sampleJobAWS.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, sampleRelease, test.ID, sampleJobAWS.ID, suite.ID, 80, 72, 0) + + // Sample: gcp+sdn 60 runs, 54 success (no base counterpart) + createCumulativeSummary(t, dbc, sampleLookupStart, sampleRelease, test.ID, sampleJobGCP.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, sampleRelease, test.ID, sampleJobGCP.ID, suite.ID, 60, 54, 0) + + provider := postgres.NewPostgresProvider(dbc, nil) + opts := reqopts.RequestOptions{ + BaseRelease: reqopts.Release{ + Name: baseRelease, + Start: time.Date(2024, 5, 15, 0, 0, 0, 0, time.UTC), + End: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + }, + SampleRelease: reqopts.Release{ + Name: sampleRelease, + Start: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + End: time.Date(2024, 6, 15, 0, 0, 0, 0, time.UTC), + }, + VariantOption: reqopts.Variants{ + DBGroupBy: sets.New[string]("Platform", "Network"), + ColumnGroupBy: sets.New[string]("Platform"), + IncludeVariants: map[string][]string{"Platform": {"aws", "gcp"}, "Network": {"ovn", "sdn"}}, + }, + AdvancedOption: reqopts.Advanced{ + MinimumFailure: 1, + Confidence: 95, + }, + } + + generator := componentreadiness.NewComponentReportGenerator(provider, opts, dbc, nil, "") + report, errs := generator.GenerateReport(context.Background()) + require.Empty(t, errs) + + row := findReportRow(t, report, "Storage") + + // aws column: base has 100 runs, sample has 80 runs, both present + awsCol := findReportColumn(t, row, map[string]string{"Platform": "aws"}) + assert.NotEqual(t, crtest.MissingBasis, awsCol.Status, "aws has base data") + assert.NotEqual(t, crtest.MissingSample, awsCol.Status, "aws has sample data") + assert.GreaterOrEqual(t, int(awsCol.Status), int(crtest.NotSignificant), + "aws should have both base and sample data, not MissingSample/MissingBasis") + + // gcp column: sample has 60 runs, base has none -> MissingBasis + gcpCol := findReportColumn(t, row, map[string]string{"Platform": "gcp"}) + assert.Equal(t, crtest.MissingBasis, gcpCol.Status, + "gcp should be MissingBasis since base release has no gcp data") +} + +func TestGenerateReport_EmptyBaseLookupStillReturnsSampleResults(t *testing.T) { + dbc := crTestDB(t) + baseRelease := "4.16" + sampleRelease := "4.17" + + // Only create a variant combination for Topology:ha. There is no + // Topology:single VC in the database, so the base-side variant lookup + // will be empty (base uses IncludeVariants which requests Topology:single). + // The sample side uses CompareVariants which requests Topology:ha. + vcHA := createVariantCombination(t, dbc, []string{"Platform:aws", "Topology:ha"}) + + baseJob := createProwJobWithVC(t, dbc, "periodic-empty-base-lookup-base", baseRelease, vcHA) + sampleJob := createProwJobWithVC(t, dbc, "periodic-empty-base-lookup-sample", sampleRelease, vcHA) + + test := intutil.CreateTest(t, dbc, "openshift-tests:[sig-storage] empty base lookup test") + suite := intutil.CreateSuite(t, dbc, "openshift-tests-empty-base-lookup") + createTestOwnership(t, dbc, test.ID, &suite.ID, "openshift-tests:empty-base-lookup", "Storage", []string{"PVC"}) + + baseLookupStart := civil.Date{Year: 2024, Month: 5, Day: 14} + baseLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 1} + sampleLookupStart := civil.Date{Year: 2024, Month: 5, Day: 31} + sampleLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 14} + + // Base data exists under ha, but the base-side filter will request "single" + // which matches nothing. This data is unreachable via the base filter. + createCumulativeSummary(t, dbc, baseLookupStart, baseRelease, test.ID, baseJob.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, baseRelease, test.ID, baseJob.ID, suite.ID, 100, 90, 0) + + // Sample data under ha, reachable via the sample-side CompareVariants filter. + createCumulativeSummary(t, dbc, sampleLookupStart, sampleRelease, test.ID, sampleJob.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, sampleRelease, test.ID, sampleJob.ID, suite.ID, 100, 85, 0) + + provider := postgres.NewPostgresProvider(dbc, nil) + opts := reqopts.RequestOptions{ + BaseRelease: reqopts.Release{ + Name: baseRelease, + Start: time.Date(2024, 5, 15, 0, 0, 0, 0, time.UTC), + End: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + }, + SampleRelease: reqopts.Release{ + Name: sampleRelease, + Start: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + End: time.Date(2024, 6, 15, 0, 0, 0, 0, time.UTC), + }, + VariantOption: reqopts.Variants{ + DBGroupBy: sets.New[string]("Platform", "Topology"), + ColumnGroupBy: sets.New[string]("Platform"), + // Base side uses Topology:single (no matching VCs exist) + IncludeVariants: map[string][]string{ + "Platform": {"aws"}, + "Topology": {"single"}, + }, + // Cross-compare overrides sample side to Topology:ha (matches vcHA) + VariantCrossCompare: []string{"Topology"}, + CompareVariants: map[string][]string{"Topology": {"ha"}}, + }, + AdvancedOption: reqopts.Advanced{ + MinimumFailure: 1, + Confidence: 95, + }, + } + + generator := componentreadiness.NewComponentReportGenerator(provider, opts, dbc, nil, "") + report, errs := generator.GenerateReport(context.Background()) + require.Empty(t, errs) + + // The sample side should produce results even though the base lookup is empty. + // With the old || guard this returned zero rows; with && it correctly returns + // sample data and the report shows MissingBasis. + row := findReportRow(t, report, "Storage") + col := findReportColumn(t, row, map[string]string{"Platform": "aws"}) + assert.Equal(t, crtest.MissingBasis, col.Status, + "base has no matching variant combinations, so should be MissingBasis") +} + +func TestGenerateReport_GABasePath(t *testing.T) { + dbc := crTestDB(t) + baseRelease := "4.15" + sampleRelease := "4.16" + + gaDate := time.Date(2024, 3, 1, 0, 0, 0, 0, time.UTC) + createReleaseDefinition(t, dbc, baseRelease, &gaDate) + + gaCivil := civil.DateOf(gaDate) + gaEnd := utils.GAWindowEnd(gaCivil) + windowDays := 30 + gaStart := gaCivil.AddDays(-windowDays) + + vc := createVariantCombination(t, dbc, []string{"Platform:aws", "Network:ovn"}) + baseJob := createProwJobWithVC(t, dbc, "periodic-ga-base-report", baseRelease, vc) + sampleJob := createProwJobWithVC(t, dbc, "periodic-ga-sample-report", sampleRelease, vc) + test := intutil.CreateTest(t, dbc, "openshift-tests:[sig-storage] GA report test") + suite := intutil.CreateSuite(t, dbc, "openshift-tests-ga-report") + createTestOwnership(t, dbc, test.ID, &suite.ID, "openshift-tests:ga-report", "Storage", []string{"PVC"}) + + // GA raw data for base: 100 runs, 90 passes, 0 flakes + createGARawData(t, dbc, baseRelease, windowDays, test.ID, baseJob.ID, suite.ID, 100, 90, 0) + + // Sample: cumulative data, 100 runs, 90 success + sampleLookupStart := civil.Date{Year: 2024, Month: 5, Day: 31} + sampleLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 14} + createCumulativeSummary(t, dbc, sampleLookupStart, sampleRelease, test.ID, sampleJob.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, sampleRelease, test.ID, sampleJob.ID, suite.ID, 100, 90, 0) + + provider := postgres.NewPostgresProvider(dbc, nil) + opts := reqopts.RequestOptions{ + BaseRelease: reqopts.Release{ + Name: baseRelease, + Start: gaStart.In(time.UTC), + End: gaEnd.AddDays(-1).In(time.UTC), + }, + SampleRelease: reqopts.Release{ + Name: sampleRelease, + Start: time.Date(2024, 6, 1, 0, 0, 0, 0, time.UTC), + End: time.Date(2024, 6, 15, 0, 0, 0, 0, time.UTC), + }, + VariantOption: reqopts.Variants{ + DBGroupBy: sets.New[string]("Platform", "Network"), + ColumnGroupBy: sets.New[string]("Platform"), + IncludeVariants: map[string][]string{"Platform": {"aws"}, "Network": {"ovn"}}, + }, + AdvancedOption: reqopts.Advanced{ + MinimumFailure: 1, + Confidence: 95, + }, + IncludeAllTests: true, + } + + generator := componentreadiness.NewComponentReportGenerator(provider, opts, dbc, nil, "") + report, errs := generator.GenerateReport(context.Background()) + require.Empty(t, errs) + + row := findReportRow(t, report, "Storage") + col := findReportColumn(t, row, map[string]string{"Platform": "aws"}) + tests := filterReportPlaceholders(col.AllTests) + require.Len(t, tests, 1) + + // Base: GA raw data = 100 runs, 90 success + require.NotNil(t, tests[0].BaseStats) + assert.Equal(t, 100, tests[0].BaseStats.Total()) + assert.Equal(t, 90, tests[0].BaseStats.SuccessCount) + // Sample: cumulative data = 100 runs, 90 success + assert.Equal(t, 100, tests[0].SampleStats.Total()) + assert.Equal(t, 90, tests[0].SampleStats.SuccessCount) +} + +func TestGenerateReport_LifecycleFilter(t *testing.T) { + dbc := crTestDB(t) + release := "4.16" + + vc := createVariantCombination(t, dbc, []string{"Platform:aws", "Network:ovn"}) + job := createProwJobWithVC(t, dbc, "periodic-e2e-aws-lc-report", release, vc) + test := intutil.CreateTest(t, dbc, "openshift-tests:[sig-storage] lifecycle report test") + suite := intutil.CreateSuite(t, dbc, "openshift-tests-lc-report") + createTestOwnership(t, dbc, test.ID, &suite.ID, "openshift-tests:lc-report", "Storage", []string{"PVC"}) + + baseLookupStart := civil.Date{Year: 2024, Month: 5, Day: 14} + baseLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 1} + sampleLookupStart := civil.Date{Year: 2024, Month: 5, Day: 31} + sampleLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 14} + + // Blocking: base 60 runs/55 success, sample 50 runs/45 success + createCumulativeSummary(t, dbc, baseLookupStart, release, test.ID, job.ID, suite.ID, 0, 0, 0, withLifecycle("blocking")) + createCumulativeSummary(t, dbc, baseLookupEnd, release, test.ID, job.ID, suite.ID, 60, 55, 0, withLifecycle("blocking")) + createCumulativeSummary(t, dbc, sampleLookupStart, release, test.ID, job.ID, suite.ID, 60, 55, 0, withLifecycle("blocking")) + createCumulativeSummary(t, dbc, sampleLookupEnd, release, test.ID, job.ID, suite.ID, 110, 100, 0, withLifecycle("blocking")) + + // Informing: base 40 runs/35 success, sample 30 runs/25 success + createCumulativeSummary(t, dbc, baseLookupStart, release, test.ID, job.ID, suite.ID, 0, 0, 0, withLifecycle("informing")) + createCumulativeSummary(t, dbc, baseLookupEnd, release, test.ID, job.ID, suite.ID, 40, 35, 0, withLifecycle("informing")) + createCumulativeSummary(t, dbc, sampleLookupStart, release, test.ID, job.ID, suite.ID, 40, 35, 0, withLifecycle("informing")) + createCumulativeSummary(t, dbc, sampleLookupEnd, release, test.ID, job.ID, suite.ID, 70, 60, 0, withLifecycle("informing")) + + provider := postgres.NewPostgresProvider(dbc, nil) + opts := defaultReqOptions(release) + opts.AdvancedOption.Confidence = 95 + opts.IncludeAllTests = true + opts.Lifecycles = []string{"blocking"} + opts.VariantOption.IncludeVariants = map[string][]string{ + "Platform": {"aws"}, + "Network": {"ovn"}, + } + + generator := componentreadiness.NewComponentReportGenerator(provider, opts, dbc, nil, "") + report, errs := generator.GenerateReport(context.Background()) + require.Empty(t, errs) + + row := findReportRow(t, report, "Storage") + col := findReportColumn(t, row, map[string]string{"Platform": "aws"}) + tests := filterReportPlaceholders(col.AllTests) + require.Len(t, tests, 1) + + // Sample: only blocking data = 50 runs, 45 success + assert.Equal(t, 50, tests[0].SampleStats.Total()) + assert.Equal(t, 45, tests[0].SampleStats.SuccessCount) + // Base: includes all lifecycles = blocking (60) + informing (40) = 100 runs, 90 success + require.NotNil(t, tests[0].BaseStats) + assert.Equal(t, 100, tests[0].BaseStats.Total()) + assert.Equal(t, 90, tests[0].BaseStats.SuccessCount) +} + +func TestGenerateReport_MinimumFailureThreshold(t *testing.T) { + dbc := crTestDB(t) + release := "4.16" + + vc := createVariantCombination(t, dbc, []string{"Platform:aws", "Network:ovn"}) + job := createProwJobWithVC(t, dbc, "periodic-e2e-aws-mf-report", release, vc) + + testHigh := intutil.CreateTest(t, dbc, "openshift-tests:[sig-storage] high failure report test") + testLow := intutil.CreateTest(t, dbc, "openshift-tests:[sig-network] low failure report test") + suite := intutil.CreateSuite(t, dbc, "openshift-tests-mf-report") + + createTestOwnership(t, dbc, testHigh.ID, &suite.ID, "openshift-tests:high-fail", "Storage", []string{"PVC"}) + createTestOwnership(t, dbc, testLow.ID, &suite.ID, "openshift-tests:low-fail", "Networking", []string{"Services"}) + + baseLookupStart := civil.Date{Year: 2024, Month: 5, Day: 14} + baseLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 1} + sampleLookupStart := civil.Date{Year: 2024, Month: 5, Day: 31} + sampleLookupEnd := civil.Date{Year: 2024, Month: 6, Day: 14} + + // testHigh: base 200 runs/190 success (95%), sample 200 runs/140 success (70%) + createCumulativeSummary(t, dbc, baseLookupStart, release, testHigh.ID, job.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, release, testHigh.ID, job.ID, suite.ID, 200, 190, 0) + createCumulativeSummary(t, dbc, sampleLookupStart, release, testHigh.ID, job.ID, suite.ID, 200, 190, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, release, testHigh.ID, job.ID, suite.ID, 400, 330, 0) + + // testLow: base 100 runs/95 success (95%), sample 100 runs/98 success → 2 failures < MinimumFailure + createCumulativeSummary(t, dbc, baseLookupStart, release, testLow.ID, job.ID, suite.ID, 0, 0, 0) + createCumulativeSummary(t, dbc, baseLookupEnd, release, testLow.ID, job.ID, suite.ID, 100, 95, 0) + createCumulativeSummary(t, dbc, sampleLookupStart, release, testLow.ID, job.ID, suite.ID, 100, 95, 0) + createCumulativeSummary(t, dbc, sampleLookupEnd, release, testLow.ID, job.ID, suite.ID, 200, 193, 0) + + provider := postgres.NewPostgresProvider(dbc, nil) + opts := defaultReqOptions(release) + opts.AdvancedOption.Confidence = 95 + opts.AdvancedOption.MinimumFailure = 3 + opts.VariantOption.IncludeVariants = map[string][]string{ + "Platform": {"aws"}, + "Network": {"ovn"}, + } + + generator := componentreadiness.NewComponentReportGenerator(provider, opts, dbc, nil, "") + report, errs := generator.GenerateReport(context.Background()) + require.Empty(t, errs) + + // testHigh: 60 failures >= MinimumFailure=3, should be flagged as regression + storageRow := findReportRow(t, report, "Storage") + storageCol := findReportColumn(t, storageRow, map[string]string{"Platform": "aws"}) + assert.Equal(t, crtest.ExtremeRegression, storageCol.Status, "test with 60 failures should be extreme regression") + require.NotEmpty(t, storageCol.RegressedTests) + + // testLow: 2 failures < MinimumFailure=3, should not be flagged as a regression + networkRow := findReportRow(t, report, "Networking") + networkCol := findReportColumn(t, networkRow, map[string]string{"Platform": "aws"}) + assert.Empty(t, networkCol.RegressedTests, "test below MinimumFailure should not appear as regression") +} + +// --- Report helpers --- + +func findReportRow(t *testing.T, report crtype.ComponentReport, component string) crtype.ReportRow { + t.Helper() + for _, row := range report.Rows { + if row.Component == component { + return row + } + } + t.Fatalf("no row found for component %q in report with %d rows", component, len(report.Rows)) + return crtype.ReportRow{} +} + +func findReportColumn(t *testing.T, row crtype.ReportRow, variants map[string]string) crtype.ReportColumn { + t.Helper() + for _, col := range row.Columns { + if maps.Equal(col.Variants, variants) { + return col + } + } + t.Fatalf("no column found for variants %v in row %q with %d columns", variants, row.Component, len(row.Columns)) + return crtype.ReportColumn{} +} + +func filterReportPlaceholders(allTests []crtype.ReportTestSummary) []crtype.ReportTestSummary { + var actual []crtype.ReportTestSummary + for _, ts := range allTests { + if !isPlaceholderKey(ts.TestID) { + actual = append(actual, ts) + } + } + return actual +} + // --- Helpers --- func isPlaceholderKey(testID string) bool {