From 5d8e3c2ec93181d141ad469585ad1308f3696bbc Mon Sep 17 00:00:00 2001 From: Matthew Staebler Date: Tue, 7 Jul 2026 12:47:31 -0400 Subject: [PATCH] Add release_definitions table and loader Create a release_definitions table to store release metadata (GA dates, development start dates, previous release, capabilities, product, status) synced from BigQuery. This is the first phase of moving release metadata from BigQuery to PostgreSQL. Key changes: - Add ReleaseDefinition model with capability constants and HasCapability method - Add release-definitions loader (--loader release-definitions) that fetches from BQ and syncs to PG via upsert - Add GetReleasesFromDB and DefinitionToRelease for reading from PG - Add GetReleaseRowsFromBigQuery for direct BQ row access - Seed data populates release_definitions for local development - Load command reads release configs from PG with BQ fallback for first run when the table is empty No consumers are switched in this phase. The existing getReleases() and QueryReleases() paths continue to use BigQuery / hardcoded metadata. A follow-up PR will switch consumers to read from the new table once it has been populated by the load cycle. Ref: TRT-2734 Co-Authored-By: Claude Opus 4.6 --- cmd/sippy/load.go | 31 +++++- cmd/sippy/seed_data.go | 52 +++++++++ pkg/api/releases.go | 60 ++++++++++ .../loaderwithmetrics/loaderwithmetrics.go | 2 +- .../releasedefloader/releasedefloader.go | 103 ++++++++++++++++++ pkg/db/db.go | 1 + pkg/db/models/releases.go | 35 ++++++ 7 files changed, 279 insertions(+), 5 deletions(-) create mode 100644 pkg/dataloader/releasedefloader/releasedefloader.go diff --git a/cmd/sippy/load.go b/cmd/sippy/load.go index 8a75b65223..77be99579c 100644 --- a/cmd/sippy/load.go +++ b/cmd/sippy/load.go @@ -37,6 +37,7 @@ import ( "github.com/openshift/sippy/pkg/dataloader/prowloader" "github.com/openshift/sippy/pkg/dataloader/prowloader/gcs" "github.com/openshift/sippy/pkg/dataloader/prowloader/github" + releasedefloader "github.com/openshift/sippy/pkg/dataloader/releasedefloader" "github.com/openshift/sippy/pkg/dataloader/releaseloader" "github.com/openshift/sippy/pkg/dataloader/testownershiploader" "github.com/openshift/sippy/pkg/db" @@ -100,7 +101,7 @@ func (f *LoadFlags) BindFlags(fs *pflag.FlagSet) { f.JiraFlags.BindFlags(fs) fs.BoolVar(&f.InitDatabase, "init-database", false, "Migrate the DB before loading") - fs.StringArrayVar(&f.Loaders, "loader", []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading") + fs.StringArrayVar(&f.Loaders, "loader", []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading") fs.StringArrayVar(&f.Releases, "release", f.Releases, "Which releases to load (one per arg instance)") fs.StringArrayVar(&f.Architectures, "arch", f.Architectures, "Which architectures to load (one per arg instance)") fs.StringVar(&f.JobVariantsInputFile, "job-variants-input-file", "expected-job-variants.json", "JSON input file for the job-variants loader") @@ -152,8 +153,6 @@ func NewLoadCommand() *cobra.Command { cacheClient = nil // error hygiene, since we pass this down to quite a few functions } - releaseConfigs := []sippyv1.Release{} - // initializing a bigquery client different from the normal one opCtx, ctx := bqcachedclient.OpCtxForCronEnv(ctx, "load") bqc, bigqueryErr := bqcachedclient.New( @@ -164,9 +163,22 @@ func NewLoadCommand() *cobra.Command { if f.CacheFlags.EnablePersistentCaching { bqc = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bqc) } + } + + // Read release definitions from PG for downstream loader construction. + // Falls back to BQ if PG is empty (e.g., first run before the + // release-definitions loader has populated the table). + releaseConfigs := []sippyv1.Release{} + if dbErr == nil { + releaseConfigs, err = api.GetReleasesFromDB(context.Background(), dbc) + if err != nil { + return errors.Wrapf(err, "error querying release definitions from postgres") + } + } + if len(releaseConfigs) == 0 && bigqueryErr == nil { releaseConfigs, err = api.GetReleasesFromBigQuery(context.Background(), bqc) if err != nil { - return errors.Wrapf(err, "error querying releases from bq") + return errors.Wrapf(err, "error querying releases from bigquery") } } @@ -201,6 +213,17 @@ func NewLoadCommand() *cobra.Command { var regressionCacheAdded bool for _, l := range f.Loaders { + if l == "release-definitions" { + if bigqueryErr != nil { + return errors.Wrap(bigqueryErr, "CRITICAL error getting BigQuery client which prevents release-definitions loading") + } + if dbErr != nil { + return errors.Wrap(dbErr, "CRITICAL error getting postgres client which prevents release-definitions loading") + } + rdl := releasedefloader.NewReleaseDefinitionLoader(ctx, dbc, bqc) + loaders = append(loaders, rdl) + } + // TODO: remove "component-readiness-cache" and "regression-tracker" once the cronjob // manifests are updated to use "regression-cache". if l == "component-readiness-cache" || l == "regression-tracker" || l == "regression-cache" { diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index adc6817edc..b8736bc5cf 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -411,6 +411,11 @@ func seedSyntheticData(dbc *db.DB) error { return nil } + if err := seedReleaseDefinitions(dbc); err != nil { + return errors.WithMessage(err, "failed to seed release definitions") + } + log.Info("Seeded release definitions") + if err := createTestSuite(dbc, "synthetic"); err != nil { return errors.WithMessage(err, "failed to create test suite") } @@ -447,6 +452,53 @@ func seedSyntheticData(dbc *db.DB) error { return nil } +func seedReleaseDefinitions(dbc *db.DB) error { + now := time.Now().UTC() + allCaps := pq.StringArray{models.CapComponentReadiness, models.CapFeatureGates, models.CapMetrics, models.CapPayloadTags, models.CapSippyClassic} + + type relMeta struct { + previous string + gaDays int // negative = days before now; 0 = no GA (in development) + } + meta := map[string]relMeta{ + "4.19": {previous: "4.18", gaDays: -289}, + "4.20": {previous: "4.19", gaDays: -163}, + "4.21": {previous: "4.20", gaDays: -58}, + "4.22": {previous: "4.21"}, + } + + for _, release := range syntheticReleases { + m := meta[release] + parts := strings.Split(release, ".") + major, minor := 0, 0 + if len(parts) >= 2 { + _, _ = fmt.Sscanf(parts[0], "%d", &major) + _, _ = fmt.Sscanf(parts[1], "%d", &minor) + } + + develStart := now.AddDate(0, 0, m.gaDays-180) + def := models.ReleaseDefinition{ + Release: release, + Major: major, + Minor: minor, + PreviousRelease: m.previous, + DevelopmentStartDate: &develStart, + Product: "OCP", + Status: "Full Support", + Capabilities: allCaps, + } + if m.gaDays != 0 { + ga := now.AddDate(0, 0, m.gaDays) + def.GADate = &ga + } + + if err := dbc.DB.Where("release = ?", release).FirstOrCreate(&def).Error; err != nil { + return fmt.Errorf("failed to create release definition %s: %w", release, err) + } + } + return nil +} + func seedProwJobs(dbc *db.DB) error { for _, release := range syntheticReleases { for _, job := range syntheticJobs { diff --git a/pkg/api/releases.go b/pkg/api/releases.go index c543450568..39857600be 100644 --- a/pkg/api/releases.go +++ b/pkg/api/releases.go @@ -512,6 +512,66 @@ func transformRelease(r sippyv1.ReleaseRow) sippyv1.Release { return release } +// GetReleaseRowsFromBigQuery fetches raw release rows from BigQuery's Releases table. +func GetReleaseRowsFromBigQuery(ctx context.Context, client *bqcachedclient.Client) ([]sippyv1.ReleaseRow, error) { + var rows []sippyv1.ReleaseRow + + queryString := fmt.Sprintf("SELECT * FROM `%s` ORDER BY DevelStartDate DESC", client.ReleasesTable) + + q := client.Query(ctx, bqlabel.ReleaseAllReleases, queryString) + it, err := q.Read(ctx) + if err != nil { + log.WithError(err).Error("error querying releases data from bigquery") + return rows, err + } + + for { + r := sippyv1.ReleaseRow{} + err := it.Next(&r) + if err == iterator.Done { + break + } + if err != nil { + log.WithError(err).Error("error parsing release row from bigquery") + return rows, err + } + rows = append(rows, r) + } + return rows, nil +} + +// GetReleasesFromDB queries release metadata from the release_definitions table +// and converts to []sippyv1.Release for use by existing callers. +func GetReleasesFromDB(ctx context.Context, dbc *db.DB) ([]sippyv1.Release, error) { + var defs []models.ReleaseDefinition + err := dbc.DB.WithContext(ctx).Order("development_start_date DESC").Find(&defs).Error + if err != nil { + return nil, fmt.Errorf("querying release definitions: %w", err) + } + releases := make([]sippyv1.Release, 0, len(defs)) + for _, def := range defs { + releases = append(releases, DefinitionToRelease(def)) + } + return releases, nil +} + +// DefinitionToRelease converts a models.ReleaseDefinition to a sippyv1.Release. +func DefinitionToRelease(def models.ReleaseDefinition) sippyv1.Release { + caps := make(map[sippyv1.ReleaseCapability]bool, len(def.Capabilities)) + for _, cap := range def.Capabilities { + caps[sippyv1.ReleaseCapability(cap)] = true + } + return sippyv1.Release{ + Release: def.Release, + Status: def.Status, + GADate: def.GADate, + DevelopmentStartDate: def.DevelopmentStartDate, + PreviousRelease: def.PreviousRelease, + Capabilities: caps, + Product: def.Product, + } +} + // BuildReleasesResponse creates the API response structure for releases func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) apitype.Releases { gaDateMap := make(map[string]time.Time) diff --git a/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go b/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go index fc2d5424c5..485ab53664 100644 --- a/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go +++ b/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go @@ -46,7 +46,7 @@ func New(wrappedLoaders []dataloader.DataLoader) *LoaderWithMetrics { return loader } -var loaderOrder = []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"} +var loaderOrder = []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"} // sortLoaders guarantees that the loaders run in a predictable and proper order func (l *LoaderWithMetrics) sortLoaders() { diff --git a/pkg/dataloader/releasedefloader/releasedefloader.go b/pkg/dataloader/releasedefloader/releasedefloader.go new file mode 100644 index 0000000000..3defa97261 --- /dev/null +++ b/pkg/dataloader/releasedefloader/releasedefloader.go @@ -0,0 +1,103 @@ +package releasedefloader + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/lib/pq" + log "github.com/sirupsen/logrus" + "gorm.io/gorm/clause" + + "github.com/openshift/sippy/pkg/api" + v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + bqcachedclient "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" +) + +// ReleaseDefinitionLoader fetches release metadata from BigQuery and syncs it to PostgreSQL. +type ReleaseDefinitionLoader struct { + ctx context.Context + dbc *db.DB + bqClient *bqcachedclient.Client + errs []error +} + +func NewReleaseDefinitionLoader(ctx context.Context, dbc *db.DB, bqClient *bqcachedclient.Client) *ReleaseDefinitionLoader { + return &ReleaseDefinitionLoader{ + ctx: ctx, + dbc: dbc, + bqClient: bqClient, + } +} + +func (l *ReleaseDefinitionLoader) Name() string { + return "release-definitions" +} + +func (l *ReleaseDefinitionLoader) Load() { + releaseRows, err := api.GetReleaseRowsFromBigQuery(l.ctx, l.bqClient) + if err != nil { + l.errs = append(l.errs, fmt.Errorf("fetching releases from bigquery: %w", err)) + return + } + defs := make([]models.ReleaseDefinition, 0, len(releaseRows)) + for _, row := range releaseRows { + defs = append(defs, ReleaseRowToDefinition(row)) + } + if err := syncReleaseDefinitions(l.dbc, defs); err != nil { + l.errs = append(l.errs, fmt.Errorf("syncing release definitions: %w", err)) + } +} + +func (l *ReleaseDefinitionLoader) Errors() []error { + return l.errs +} + +func syncReleaseDefinitions(dbc *db.DB, defs []models.ReleaseDefinition) error { + for _, def := range defs { + err := dbc.DB.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "release"}}, + DoUpdates: clause.AssignmentColumns([]string{"major", "minor", "patch", "previous_release", "ga_date", "development_start_date", "product", "status", "capabilities", "updated_at"}), + }).Create(&def).Error + if err != nil { + return fmt.Errorf("upserting release definition %s: %w", def.Release, err) + } + } + log.WithField("count", len(defs)).Info("synced release definitions to postgres") + return nil +} + +// ReleaseRowToDefinition converts a BigQuery ReleaseRow directly to a ReleaseDefinition DB model. +func ReleaseRowToDefinition(r v1.ReleaseRow) models.ReleaseDefinition { + caps := make(pq.StringArray, 0, len(r.Capabilities)) + for _, cap := range r.Capabilities { + caps = append(caps, string(cap)) + } + sort.Strings(caps) + + def := models.ReleaseDefinition{ + Release: r.Release, + Major: r.Major, + Minor: r.Minor, + PreviousRelease: r.PreviousRelease.StringVal, + Product: r.Product.StringVal, + Status: r.ReleaseStatus.StringVal, + Capabilities: caps, + } + if r.Patch.Valid { + p := int(r.Patch.Int64) + def.Patch = &p + } + if r.GADate.Valid { + ga := r.GADate.Date.In(time.UTC) + def.GADate = &ga + } + if r.DevelStartDate.IsValid() { + ds := r.DevelStartDate.In(time.UTC) + def.DevelopmentStartDate = &ds + } + return def +} diff --git a/pkg/db/db.go b/pkg/db/db.go index 3a76773d93..096d61fc10 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -137,6 +137,7 @@ func (d *DB) UpdateSchema(reportEnd *time.Time) error { // List of all models to migrate modelsToMigrate := []any{ + &models.ReleaseDefinition{}, &models.ReleaseTag{}, &models.ReleasePullRequest{}, &models.ReleaseRepository{}, diff --git a/pkg/db/models/releases.go b/pkg/db/models/releases.go index 01c578e628..a02f533f72 100644 --- a/pkg/db/models/releases.go +++ b/pkg/db/models/releases.go @@ -6,6 +6,41 @@ import ( "github.com/lib/pq" ) +// ReleaseDefinition stores release metadata synced from BigQuery's Releases table. +// This is distinct from ReleaseTag, which tracks individual payload tags. +type ReleaseDefinition struct { + Model + + Release string `json:"release" gorm:"uniqueIndex;column:release"` + Major int `json:"major" gorm:"column:major"` + Minor int `json:"minor" gorm:"column:minor"` + Patch *int `json:"patch,omitempty" gorm:"column:patch"` + PreviousRelease string `json:"previous_release" gorm:"column:previous_release"` + GADate *time.Time `json:"ga_date,omitempty" gorm:"column:ga_date;type:date"` + DevelopmentStartDate *time.Time `json:"development_start_date" gorm:"column:development_start_date;type:date"` + Product string `json:"product" gorm:"column:product"` + Status string `json:"status" gorm:"column:status"` + Capabilities pq.StringArray `json:"capabilities" gorm:"type:text[];column:capabilities"` +} + +const ( + CapComponentReadiness = "componentReadiness" + CapSippyClassic = "sippyClassic" + CapMetrics = "metrics" + CapPullRequests = "pullRequests" + CapFeatureGates = "featureGates" + CapPayloadTags = "payloadTags" +) + +func (rd *ReleaseDefinition) HasCapability(capability string) bool { + for _, c := range rd.Capabilities { + if c == capability { + return true + } + } + return false +} + type ReleaseTag struct { Model