From 9ba64f3e40433fcbfea191c6e1c38c5454f8288e Mon Sep 17 00:00:00 2001 From: Matthew Staebler Date: Thu, 9 Jul 2026 16:29:04 -0400 Subject: [PATCH] TRT-2364: Fix timestamp and date type inconsistencies Replace epoch millisecond integers with proper timestamp and date types across the database schema, API layer, and frontend. Backend: - Enforce UTC timezone on all PostgreSQL connections via pgx RuntimeParams - Change matview timestamp from bigint epoch to TIMESTAMP WITH TIME ZONE - Use civil.Date for date-only fields (GA dates, development start dates, CountByDate, jobDetailAPIResult start/end, ReleaseDefinition) - Change CalendarEvent.Start/End from string to time.Time - Change ChatConversationResponse.CreatedAt from string to time.Time - Change JobRun.Timestamp from int to time.Time (RFC 3339 in JSON) - Remove epoch extraction from SQL filters; compare timestamptz directly - Handle ColumnTypeTimestamp in Compare() via GetNumericalValue - Fix PrintJobsReportFromDB to strip job-run filters (timestamp, cluster) before querying the prow_jobs table (pre-existing bug) - Add filter.StripJobRunFilters for reusable job-run filter removal - Fix GetTestAnalysisOverallFromDB to use reportEnd instead of time.Now() for consistent date windowing with pinned time (pre-existing bug) Frontend: - Use ISO 8601 strings for timestamp filter values throughout - Remove dead ga_dates timezone hack in App.js - Use valueGetter returning Date objects for DataGrid timestamp columns - Add type: 'date' to all date/timestamp DataGrid columns - Preserve 'not' flag when updating date filter values - Rewrite JobsDetail day bucketing with Temporal.PlainDate - Update DateTimePicker and filter display to use ISO strings Documentation: - Add timestamp/date type guidelines to backend and frontend instructions - Update API docs to show RFC 3339 and YYYY-MM-DD formats Note: production databases should also have their default timezone set to UTC via ALTER DATABASE SET timezone = 'UTC'. Co-Authored-By: Claude Opus 4.6 (1M context) --- .apm/instructions/backend.instructions.md | 6 + .apm/instructions/frontend.instructions.md | 7 ++ .claude/rules/backend.md | 6 + .claude/rules/frontend.md | 7 ++ .cursor/rules/backend.mdc | 6 + .cursor/rules/frontend.mdc | 7 ++ AGENTS.md | 8 +- CLAUDE.md | 8 +- apm.lock.yaml | 8 +- cmd/sippy/main.go | 10 ++ cmd/sippy/seed_data.go | 15 ++- pkg/api/README.md | 38 +++--- pkg/api/backend_disruption.go | 30 ++--- pkg/api/backend_disruption_test.go | 17 ++- .../dataprovider/bigquery/releasedates.go | 6 +- .../dataprovider/postgres/provider.go | 2 +- .../queryparamparser_test.go | 8 +- pkg/api/componentreadiness/test_details.go | 2 +- pkg/api/componentreadiness/triage_test.go | 3 +- .../componentreadiness/utils/utils_test.go | 5 +- pkg/api/jira.go | 9 +- pkg/api/job_runs.go | 4 +- pkg/api/jobs.go | 25 ++-- pkg/api/releases.go | 14 +-- pkg/api/releases_test.go | 6 +- pkg/api/tests.go | 2 +- pkg/apis/api/recent_test_failures.go | 4 + pkg/apis/api/types.go | 102 +++++++++++----- pkg/apis/sippy/v1/types.go | 6 +- pkg/apis/sippyprocessing/v1/types.go | 14 +-- pkg/apis/workloadmetrics/v1/types.go | 4 + pkg/cache/bigquerycache/bigquery.go | 14 +-- pkg/dataloader/gateststatus/loader.go | 6 +- .../prowloader/pgwriter/pgwriter.go | 19 ++- pkg/dataloader/prowloader/prow.go | 4 +- .../releasedefloader/releasedefloader.go | 7 +- .../releasedefloader/releasedefloader_test.go | 3 - pkg/db/db.go | 1 + pkg/db/functions.go | 6 +- pkg/db/models/releases.go | 4 +- pkg/db/query/test_queries.go | 11 +- pkg/filter/filterable.go | 115 +++++++++++++++--- pkg/flags/postgres_benchmarking_test.go | 5 +- pkg/sippyserver/chat_conversations.go | 4 +- pkg/sippyserver/parameters.go | 19 ++- pkg/sippyserver/server.go | 20 +-- pkg/util/utils.go | 11 +- pkg/util/utils_test.go | 2 +- sippy-ng/AGENTS.md | 9 +- sippy-ng/CLAUDE.md | 9 +- sippy-ng/src/App.jsx | 7 -- .../build_clusters/BuildClusterDetails.jsx | 4 +- .../RegressedTestsPanel.jsx | 10 +- .../TriagePotentialMatches.jsx | 10 +- .../TriagedRegressionTestList.jsx | 16 +-- .../TriagedRegressions.jsx | 55 +++++---- .../src/datagrid/GridToolbarFilterItem.jsx | 8 +- sippy-ng/src/datagrid/utils.jsx | 2 +- sippy-ng/src/helpers.jsx | 4 +- sippy-ng/src/jobs/JobRunsTable.jsx | 8 +- sippy-ng/src/jobs/JobStackedChart.jsx | 10 +- sippy-ng/src/jobs/JobTable.jsx | 12 +- sippy-ng/src/jobs/JobsDetail.jsx | 74 +++++------ .../src/pull_requests/PullRequestsTable.jsx | 7 +- sippy-ng/src/releases/ReleasePayloadTable.jsx | 30 ++--- test/integration/backfill_test.go | 14 +-- test/integration/component_readiness_test.go | 54 ++++---- test/integration/job_runs_report_test.go | 7 +- test/integration/jobs_test.go | 4 +- test/integration/pgwriter_test.go | 32 ++--- test/integration/recent_test_failures_test.go | 10 +- 71 files changed, 609 insertions(+), 427 deletions(-) diff --git a/.apm/instructions/backend.instructions.md b/.apm/instructions/backend.instructions.md index 78e62d141c..70bfe99518 100644 --- a/.apm/instructions/backend.instructions.md +++ b/.apm/instructions/backend.instructions.md @@ -11,3 +11,9 @@ applyTo: "**/*.go" counts, a log.WithField() call is preferred over formatting values into a string. * After making changes, always run `gofmt -w` on modified files to ensure proper formatting. * When modifying any data provider (BigQuery or PostgreSQL), ensure **parity between both implementations**. Changes to query logic, filtering, or returned data in one provider must be reflected in the other. +* **Timestamps and dates**: Use proper types, never epoch integers. + - PostgreSQL columns: use `TIMESTAMP WITH TIME ZONE` for timestamps and `DATE` for date-only values. All timestamps are UTC. + - Go structs: use `time.Time` for timestamps. Use `civil.Date` (`cloud.google.com/go/civil`) for date-only values (e.g., GA dates, development start dates). Do not use `string` for date or timestamp fields; let JSON marshaling produce the correct format. + - API responses: timestamps serialize as RFC 3339 strings, dates as `YYYY-MM-DD` strings. Never return epoch millisecond integers. Never manually format with `time.Format()` into a string field. + - Materialized views: project timestamp columns directly from the source table. Do not convert to epoch with `EXTRACT(epoch FROM ...)`. + - GORM model tags: include `gorm:"type:date"` on date-only columns so GORM and migrations use the correct PostgreSQL type. diff --git a/.apm/instructions/frontend.instructions.md b/.apm/instructions/frontend.instructions.md index 72c368175e..332b8da9a2 100644 --- a/.apm/instructions/frontend.instructions.md +++ b/.apm/instructions/frontend.instructions.md @@ -14,3 +14,10 @@ npx prettier --write . * Keep UI elements consistent with Material-UI standards. The frontend uses `npm`. If you must install or update any dependencies, always use the `--ignore-scripts` flag. + +* **Timestamps and dates from the API**: + - Timestamps arrive as RFC 3339 strings (e.g., `"2024-06-27T15:30:00Z"`), not epoch millisecond integers. Use `new Date(value)` or `Temporal.Instant.from(value)` to parse them. + - Dates arrive as `YYYY-MM-DD` strings (e.g., `"2024-06-27"`). Use `Temporal.PlainDate.from(value)` for date arithmetic. + - For MUI DataGrid timestamp columns, use `type: 'date'` with a `valueGetter` that returns a `Date` object. Do not return epoch milliseconds from `valueGetter`. + - For filter values sent to the API, use ISO 8601 strings (e.g., `new Date(...).toISOString()`), not epoch millisecond integers. + - For day-level bucketing or date arithmetic, prefer `Temporal.PlainDate` over `Date` with manual millisecond math. diff --git a/.claude/rules/backend.md b/.claude/rules/backend.md index e00cbb2031..5842594eb1 100644 --- a/.claude/rules/backend.md +++ b/.claude/rules/backend.md @@ -11,3 +11,9 @@ paths: counts, a log.WithField() call is preferred over formatting values into a string. * After making changes, always run `gofmt -w` on modified files to ensure proper formatting. * When modifying any data provider (BigQuery or PostgreSQL), ensure **parity between both implementations**. Changes to query logic, filtering, or returned data in one provider must be reflected in the other. +* **Timestamps and dates**: Use proper types, never epoch integers. + - PostgreSQL columns: use `TIMESTAMP WITH TIME ZONE` for timestamps and `DATE` for date-only values. All timestamps are UTC. + - Go structs: use `time.Time` for timestamps. Use `civil.Date` (`cloud.google.com/go/civil`) for date-only values (e.g., GA dates, development start dates). Do not use `string` for date or timestamp fields; let JSON marshaling produce the correct format. + - API responses: timestamps serialize as RFC 3339 strings, dates as `YYYY-MM-DD` strings. Never return epoch millisecond integers. Never manually format with `time.Format()` into a string field. + - Materialized views: project timestamp columns directly from the source table. Do not convert to epoch with `EXTRACT(epoch FROM ...)`. + - GORM model tags: include `gorm:"type:date"` on date-only columns so GORM and migrations use the correct PostgreSQL type. diff --git a/.claude/rules/frontend.md b/.claude/rules/frontend.md index 0551c30f22..0beefa4ba6 100644 --- a/.claude/rules/frontend.md +++ b/.claude/rules/frontend.md @@ -14,3 +14,10 @@ npx prettier --write . * Keep UI elements consistent with Material-UI standards. The frontend uses `npm`. If you must install or update any dependencies, always use the `--ignore-scripts` flag. + +* **Timestamps and dates from the API**: + - Timestamps arrive as RFC 3339 strings (e.g., `"2024-06-27T15:30:00Z"`), not epoch millisecond integers. Use `new Date(value)` or `Temporal.Instant.from(value)` to parse them. + - Dates arrive as `YYYY-MM-DD` strings (e.g., `"2024-06-27"`). Use `Temporal.PlainDate.from(value)` for date arithmetic. + - For MUI DataGrid timestamp columns, use `type: 'date'` with a `valueGetter` that returns a `Date` object. Do not return epoch milliseconds from `valueGetter`. + - For filter values sent to the API, use ISO 8601 strings (e.g., `new Date(...).toISOString()`), not epoch millisecond integers. + - For day-level bucketing or date arithmetic, prefer `Temporal.PlainDate` over `Date` with manual millisecond math. diff --git a/.cursor/rules/backend.mdc b/.cursor/rules/backend.mdc index 13c07fc93e..d94b2e3bb9 100644 --- a/.cursor/rules/backend.mdc +++ b/.cursor/rules/backend.mdc @@ -11,3 +11,9 @@ globs: "**/*.go" counts, a log.WithField() call is preferred over formatting values into a string. * After making changes, always run `gofmt -w` on modified files to ensure proper formatting. * When modifying any data provider (BigQuery or PostgreSQL), ensure **parity between both implementations**. Changes to query logic, filtering, or returned data in one provider must be reflected in the other. +* **Timestamps and dates**: Use proper types, never epoch integers. + - PostgreSQL columns: use `TIMESTAMP WITH TIME ZONE` for timestamps and `DATE` for date-only values. All timestamps are UTC. + - Go structs: use `time.Time` for timestamps. Use `civil.Date` (`cloud.google.com/go/civil`) for date-only values (e.g., GA dates, development start dates). Do not use `string` for date or timestamp fields; let JSON marshaling produce the correct format. + - API responses: timestamps serialize as RFC 3339 strings, dates as `YYYY-MM-DD` strings. Never return epoch millisecond integers. Never manually format with `time.Format()` into a string field. + - Materialized views: project timestamp columns directly from the source table. Do not convert to epoch with `EXTRACT(epoch FROM ...)`. + - GORM model tags: include `gorm:"type:date"` on date-only columns so GORM and migrations use the correct PostgreSQL type. diff --git a/.cursor/rules/frontend.mdc b/.cursor/rules/frontend.mdc index fe7ff9c36e..f56ba9c834 100644 --- a/.cursor/rules/frontend.mdc +++ b/.cursor/rules/frontend.mdc @@ -14,3 +14,10 @@ npx prettier --write . * Keep UI elements consistent with Material-UI standards. The frontend uses `npm`. If you must install or update any dependencies, always use the `--ignore-scripts` flag. + +* **Timestamps and dates from the API**: + - Timestamps arrive as RFC 3339 strings (e.g., `"2024-06-27T15:30:00Z"`), not epoch millisecond integers. Use `new Date(value)` or `Temporal.Instant.from(value)` to parse them. + - Dates arrive as `YYYY-MM-DD` strings (e.g., `"2024-06-27"`). Use `Temporal.PlainDate.from(value)` for date arithmetic. + - For MUI DataGrid timestamp columns, use `type: 'date'` with a `valueGetter` that returns a `Date` object. Do not return epoch milliseconds from `valueGetter`. + - For filter values sent to the API, use ISO 8601 strings (e.g., `new Date(...).toISOString()`), not epoch millisecond integers. + - For day-level bucketing or date arithmetic, prefer `Temporal.PlainDate` over `Date` with manual millisecond math. diff --git a/AGENTS.md b/AGENTS.md index 2e6655239b..d34890c1b6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # AGENTS.md - + @@ -88,6 +88,12 @@ Favor clarity and maintainability over cleverness. Comments should be minimal, h counts, a log.WithField() call is preferred over formatting values into a string. * After making changes, always run `gofmt -w` on modified files to ensure proper formatting. * When modifying any data provider (BigQuery or PostgreSQL), ensure **parity between both implementations**. Changes to query logic, filtering, or returned data in one provider must be reflected in the other. +* **Timestamps and dates**: Use proper types, never epoch integers. + - PostgreSQL columns: use `TIMESTAMP WITH TIME ZONE` for timestamps and `DATE` for date-only values. All timestamps are UTC. + - Go structs: use `time.Time` for timestamps. Use `civil.Date` (`cloud.google.com/go/civil`) for date-only values (e.g., GA dates, development start dates). Do not use `string` for date or timestamp fields; let JSON marshaling produce the correct format. + - API responses: timestamps serialize as RFC 3339 strings, dates as `YYYY-MM-DD` strings. Never return epoch millisecond integers. Never manually format with `time.Format()` into a string field. + - Materialized views: project timestamp columns directly from the source table. Do not convert to epoch with `EXTRACT(epoch FROM ...)`. + - GORM model tags: include `gorm:"type:date"` on date-only columns so GORM and migrations use the correct PostgreSQL type. ## Files matching `**/*_test.go` diff --git a/CLAUDE.md b/CLAUDE.md index e437b1f63e..03e3105d09 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,6 +1,6 @@ # CLAUDE.md - + # Project Standards @@ -89,6 +89,12 @@ Favor clarity and maintainability over cleverness. Comments should be minimal, h counts, a log.WithField() call is preferred over formatting values into a string. * After making changes, always run `gofmt -w` on modified files to ensure proper formatting. * When modifying any data provider (BigQuery or PostgreSQL), ensure **parity between both implementations**. Changes to query logic, filtering, or returned data in one provider must be reflected in the other. +* **Timestamps and dates**: Use proper types, never epoch integers. + - PostgreSQL columns: use `TIMESTAMP WITH TIME ZONE` for timestamps and `DATE` for date-only values. All timestamps are UTC. + - Go structs: use `time.Time` for timestamps. Use `civil.Date` (`cloud.google.com/go/civil`) for date-only values (e.g., GA dates, development start dates). Do not use `string` for date or timestamp fields; let JSON marshaling produce the correct format. + - API responses: timestamps serialize as RFC 3339 strings, dates as `YYYY-MM-DD` strings. Never return epoch millisecond integers. Never manually format with `time.Format()` into a string field. + - Materialized views: project timestamp columns directly from the source table. Do not convert to epoch with `EXTRACT(epoch FROM ...)`. + - GORM model tags: include `gorm:"type:date"` on date-only columns so GORM and migrations use the correct PostgreSQL type. ## Files matching `**/*_test.go` diff --git a/apm.lock.yaml b/apm.lock.yaml index 0d45670800..a2d0912ef5 100644 --- a/apm.lock.yaml +++ b/apm.lock.yaml @@ -94,10 +94,10 @@ local_deployed_file_hashes: .claude/commands/sippy-generate-release-views.md: sha256:eb4c9eeeea2ab2a90e8a8839147d8a1a309ea6ce3dafd397c6d2485c93068a9a .claude/commands/sippy-update-ga-release-views.md: sha256:4a5589bacc05127e427a2de4d34a8f13e05e297bdf6ebf7473c9e71f47a6b4f4 .claude/commands/sippy-update-job-variant.md: sha256:f88742dddeec5024931959a8330fdce362ffdd9b8825e808830ac346605cbd16 - .claude/rules/backend.md: sha256:ac3d618bf53dc847f52e0e13f6a3675301722259d9fbf12b4a40a191b880d05a + .claude/rules/backend.md: sha256:5a5dc512c2429362c7db75f52e9407a5a1576864b59361c4df5d4d8601d17de8 .claude/rules/config.md: sha256:96c5e42c039230f1e4e7f9ba56d04e6f76f23deeecdd858634c440d6029a6a83 .claude/rules/dev-commands.md: sha256:171b806b2f75a71f20126b8a9ceabe88592c11cede3fe510498e855d53d43b9e - .claude/rules/frontend.md: sha256:ff22046c5b951769218bbdf36499e67c70896811b8ef161ca6d3729a3423997d + .claude/rules/frontend.md: sha256:cdfc2cdc3981c43d91dcf9583d0fff2d3f0b4f355ded4237aa264211fe42600c .claude/rules/general.md: sha256:997f68e86cb43485ec5f108be3417f9bbb43ae1faffd660d598f18260f5df3ce .claude/rules/mcp.md: sha256:ddfe965e7cf8cddbba1374c6ae582a20ac0af17c958bf10e1a4edff6ff2ad0b8 .claude/rules/testing.md: sha256:a28be642547f1093d2f39d3a29e114dc11b480a1e130b3dfd4a19cd8ffe348de @@ -112,10 +112,10 @@ local_deployed_file_hashes: .cursor/commands/sippy-generate-release-views.md: sha256:eb4c9eeeea2ab2a90e8a8839147d8a1a309ea6ce3dafd397c6d2485c93068a9a .cursor/commands/sippy-update-ga-release-views.md: sha256:4a5589bacc05127e427a2de4d34a8f13e05e297bdf6ebf7473c9e71f47a6b4f4 .cursor/commands/sippy-update-job-variant.md: sha256:f88742dddeec5024931959a8330fdce362ffdd9b8825e808830ac346605cbd16 - .cursor/rules/backend.mdc: sha256:8930c2071ddd02fe3c7597b77bffa726c45ffedd66ec587d86adb23fb6814e38 + .cursor/rules/backend.mdc: sha256:5355475617074d8212962b19c94649c9609f3565d2e844caa6f4730c619c0dea .cursor/rules/config.mdc: sha256:d6e2195399bbb26a3fef7e54bd01862ffce39d89dce8aabdb0b89c89028192eb .cursor/rules/dev-commands.mdc: sha256:7e9635959af4dd2bf54b348dfdf41e3cf2b77c01aa3496a2e42a273c372ff9c0 - .cursor/rules/frontend.mdc: sha256:497f39372724f1ae127181fe3dac9ea9a95a51c532b68ccfee6080832cf9c556 + .cursor/rules/frontend.mdc: sha256:6cbbb94363b782dc6763344a72f5a996fe25a4bd5db08862e578b611ed083612 .cursor/rules/general.mdc: sha256:5bc6e1e12d53d85656248c9dc1239c74bcc0df29d5987f3b08e3d79e3df413b7 .cursor/rules/mcp.mdc: sha256:c02472afd46e4c89f71d4487dcd5da98b0c1bcbcf7f9cbc4d7ed4e7d3a206ec1 .cursor/rules/testing.mdc: sha256:5192be9724566f57a7423dbb65780b54d99b79d3f11e99d2225257e705f99a96 diff --git a/cmd/sippy/main.go b/cmd/sippy/main.go index e0266fb1c9..ad34bde175 100644 --- a/cmd/sippy/main.go +++ b/cmd/sippy/main.go @@ -1,10 +1,20 @@ package main import ( + "time" + log "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) +func init() { + // Sippy operates exclusively in UTC. This must be set before any time + // function runs so that libraries like pgx, which internally call + // time.Unix() (returning time.Local), produce time.Time values with + // the UTC location instead of the host's local timezone. + time.Local = time.UTC +} + var logLevel = "info" // rootCmd represents the base command when called without any subcommands diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index 4a642535e1..7f5a5e5adb 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -84,8 +84,8 @@ Drop and recreate the database to re-seed (e.g. docker compose down -v). // Create partitions for synthetic releases log.Info("Creating partitions for synthetic test data...") - startDate := time.Now().AddDate(0, 0, -190) // Cover all seed data date ranges - endDate := time.Now().AddDate(0, 0, 2) // Small buffer into future + startDate := time.Now().UTC().AddDate(0, 0, -190) // Cover all seed data date ranges + endDate := time.Now().UTC().AddDate(0, 0, 2) // Small buffer into future count, err := dbc.EnsurePartitions(syntheticReleases, startDate, endDate, false) if err != nil { return errors.WithMessage(err, "could not create partitions") @@ -580,7 +580,7 @@ func seedSyntheticData(dbc *db.DB) error { } func seedReleaseDefinitions(dbc *db.DB) error { - now := time.Now().UTC() + today := civil.DateOf(time.Now().UTC()) allCaps := pq.StringArray{models.CapComponentReadiness, models.CapFeatureGates, models.CapMetrics, models.CapPayloadTags, models.CapSippyClassic} type relMeta struct { @@ -613,7 +613,7 @@ func seedReleaseDefinitions(dbc *db.DB) error { _, _ = fmt.Sscanf(parts[1], "%d", &minor) } - develStart := now.AddDate(0, 0, m.gaDays-180) + develStart := today.AddDays(m.gaDays - 180) def = models.ReleaseDefinition{ Release: release, Major: major, @@ -625,7 +625,7 @@ func seedReleaseDefinitions(dbc *db.DB) error { Capabilities: allCaps, } if m.gaDays != 0 { - ga := now.AddDate(0, 0, m.gaDays) + ga := today.AddDays(m.gaDays) def.GADate = &ga } } @@ -1456,7 +1456,10 @@ func seedGARawTestData(dbc *db.DB) error { log.WithField("release", rel.Release).Warn("No GA seed data generated, skipping ga_data_loaded_date") continue } - gaDate := civil.DateOf(rel.GADate.UTC()) + if rel.GADate == nil { + return fmt.Errorf("release %s has nil GA date", rel.Release) + } + gaDate := *rel.GADate if err := dbc.DB.Model(&models.ReleaseDefinition{}). Where("release = ?", rel.Release). Update("ga_data_loaded_date", gaDate).Error; err != nil { diff --git a/pkg/api/README.md b/pkg/api/README.md index 35cf16475a..7f35cae7d5 100644 --- a/pkg/api/README.md +++ b/pkg/api/README.md @@ -294,95 +294,95 @@ A summary of runs for job(s). Results contains of the following values for each "name": "periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6", "results": [ { - "timestamp": 1628207039000, + "timestamp": "2021-08-06T03:43:59Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1423429598720299008" }, { - "timestamp": 1628045973000, + "timestamp": "2021-08-04T03:59:33Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1422754032564310016" }, { - "timestamp": 1628198644000, + "timestamp": "2021-08-05T21:24:04Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1423394362347229184" }, { - "timestamp": 1628485392000, + "timestamp": "2021-08-09T05:03:12Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1424597097709047808" }, { - "timestamp": 1628343908000, + "timestamp": "2021-08-07T14:25:08Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1424003666343366656" }, { - "timestamp": 1628325313000, + "timestamp": "2021-08-07T09:15:13Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1423925674229370880" }, { - "timestamp": 1628289649000, + "timestamp": "2021-08-06T23:20:49Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1423776089259380736" }, { - "timestamp": 1628277370000, + "timestamp": "2021-08-06T19:56:10Z", "result": "S", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1423724523844276224" }, { - "timestamp": 1628358891000, + "timestamp": "2021-08-07T18:34:51Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1424066513538650112" }, { - "timestamp": 1628190532000, + "timestamp": "2021-08-05T19:08:52Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1423360364472438784" }, { - "timestamp": 1628274962000, + "timestamp": "2021-08-06T19:16:02Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1423714481237659648" }, { - "timestamp": 1627391095000, + "timestamp": "2021-07-27T13:24:55Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1420007279679246336" }, { - "timestamp": 1627473363000, + "timestamp": "2021-07-28T12:16:03Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1420352338517823488" }, { - "timestamp": 1627617630000, + "timestamp": "2021-07-30T04:20:30Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1420957438630170624" }, { - "timestamp": 1627515377000, + "timestamp": "2021-07-29T00:56:17Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1420528516700573696" }, { - "timestamp": 1627396851000, + "timestamp": "2021-07-27T15:00:51Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1420031423921786880" }, { - "timestamp": 1627363991000, + "timestamp": "2021-07-27T05:53:11Z", "result": "F", "url": "https://prow.ci.openshift.org/view/gcs/origin-ci-test/logs/periodic-ci-openshift-release-master-nightly-4.9-e2e-metal-ipi-ovn-ipv6/1419893597473345536" } ] } ], - "start": 1627317573000, - "end": 1628508950000 + "start": "2021-07-26", + "end": "2021-08-09" } ``` diff --git a/pkg/api/backend_disruption.go b/pkg/api/backend_disruption.go index 2f5ab4b46d..5cb2c05540 100644 --- a/pkg/api/backend_disruption.go +++ b/pkg/api/backend_disruption.go @@ -32,8 +32,8 @@ func GetBackendDisruptionByRun(ctx context.Context, bigQueryClient *bq.Client, j DisruptionSeconds, JobName, JobRunName, - CAST(JobRunStartTime AS STRING) AS JobRunStartTime, - CAST(JobRunEndTime AS STRING) AS JobRunEndTime, + JobRunStartTime, + JobRunEndTime, Cluster, ReleaseTag, MasterNodesUpdated, @@ -76,16 +76,16 @@ ORDER BY JobRunName, DisruptionSeconds DESC` } type bqRow struct { - BackendName string `bigquery:"BackendName"` - DisruptionSeconds int `bigquery:"DisruptionSeconds"` - JobName bigquery.NullString `bigquery:"JobName"` - JobRunName string `bigquery:"JobRunName"` - JobRunStartTime bigquery.NullString `bigquery:"JobRunStartTime"` - JobRunEndTime bigquery.NullString `bigquery:"JobRunEndTime"` - Cluster bigquery.NullString `bigquery:"Cluster"` - ReleaseTag bigquery.NullString `bigquery:"ReleaseTag"` - MasterNodesUpdated bigquery.NullString `bigquery:"MasterNodesUpdated"` - JobRunStatus bigquery.NullString `bigquery:"JobRunStatus"` + BackendName string `bigquery:"BackendName"` + DisruptionSeconds int `bigquery:"DisruptionSeconds"` + JobName bigquery.NullString `bigquery:"JobName"` + JobRunName string `bigquery:"JobRunName"` + JobRunStartTime bigquery.NullTimestamp `bigquery:"JobRunStartTime"` + JobRunEndTime bigquery.NullTimestamp `bigquery:"JobRunEndTime"` + Cluster bigquery.NullString `bigquery:"Cluster"` + ReleaseTag bigquery.NullString `bigquery:"ReleaseTag"` + MasterNodesUpdated bigquery.NullString `bigquery:"MasterNodesUpdated"` + JobRunStatus bigquery.NullString `bigquery:"JobRunStatus"` } var rows []apitype.BackendDisruptionRunRow @@ -108,10 +108,12 @@ ORDER BY JobRunName, DisruptionSeconds DESC` apiRow.JobName = row.JobName.StringVal } if row.JobRunStartTime.Valid { - apiRow.JobRunStartTime = row.JobRunStartTime.StringVal + t := row.JobRunStartTime.Timestamp + apiRow.JobRunStartTime = &t } if row.JobRunEndTime.Valid { - apiRow.JobRunEndTime = row.JobRunEndTime.StringVal + t := row.JobRunEndTime.Timestamp + apiRow.JobRunEndTime = &t } if row.Cluster.Valid { apiRow.Cluster = row.Cluster.StringVal diff --git a/pkg/api/backend_disruption_test.go b/pkg/api/backend_disruption_test.go index 7d365e1f5e..e8cea7ed33 100644 --- a/pkg/api/backend_disruption_test.go +++ b/pkg/api/backend_disruption_test.go @@ -3,11 +3,14 @@ package api import ( "encoding/json" "testing" + "time" apitype "github.com/openshift/sippy/pkg/apis/api" ) func TestBackendDisruptionRunsResultSerialization(t *testing.T) { + startTime := time.Date(2026, 8, 1, 14, 23, 0, 0, time.UTC) + endTime := time.Date(2026, 8, 1, 16, 45, 0, 0, time.UTC) result := apitype.BackendDisruptionRunsResult{ Rows: []apitype.BackendDisruptionRunRow{ { @@ -15,8 +18,8 @@ func TestBackendDisruptionRunsResultSerialization(t *testing.T) { DisruptionSeconds: 12, JobName: "periodic-ci-openshift-release-master-ci-5.0-e2e-gcp-ovn-upgrade", JobRunName: "2084247445587365888", - JobRunStartTime: "2026-08-01 14:23:00 UTC", - JobRunEndTime: "2026-08-01 16:45:00 UTC", + JobRunStartTime: &startTime, + JobRunEndTime: &endTime, Cluster: "build01", ReleaseTag: "5.0.0-0.ci-2026-08-01-142300", MasterNodesUpdated: "Y", @@ -27,8 +30,8 @@ func TestBackendDisruptionRunsResultSerialization(t *testing.T) { DisruptionSeconds: 0, JobName: "periodic-ci-openshift-release-master-ci-5.0-e2e-gcp-ovn-upgrade", JobRunName: "2084247445587365888", - JobRunStartTime: "2026-08-01 14:23:00 UTC", - JobRunEndTime: "2026-08-01 16:45:00 UTC", + JobRunStartTime: &startTime, + JobRunEndTime: &endTime, Cluster: "build01", ReleaseTag: "5.0.0-0.ci-2026-08-01-142300", }, @@ -65,6 +68,12 @@ func TestBackendDisruptionRunsResultSerialization(t *testing.T) { if row.JobRunStatus != "failure" { t.Errorf("JobRunStatus = %q, want %q", row.JobRunStatus, "failure") } + if row.JobRunStartTime == nil || !row.JobRunStartTime.Equal(startTime) { + t.Errorf("JobRunStartTime = %v, want %v", row.JobRunStartTime, startTime) + } + if row.JobRunEndTime == nil || !row.JobRunEndTime.Equal(endTime) { + t.Errorf("JobRunEndTime = %v, want %v", row.JobRunEndTime, endTime) + } emptyRow := decoded.Rows[1] if emptyRow.MasterNodesUpdated != "" { diff --git a/pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go b/pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go index 9f6fea2e4b..fa1dcce671 100644 --- a/pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go +++ b/pkg/api/componentreadiness/dataprovider/bigquery/releasedates.go @@ -2,6 +2,7 @@ package bigquery import ( "context" + "time" "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/apis/api/componentreport/crtest" @@ -34,9 +35,10 @@ func (c *releaseDateQuerier) QueryReleaseDates(ctx context.Context) ([]crtest.Re for _, release := range releases { timeRange := crtest.ReleaseTimeRange{Release: release.Release} if release.GADate != nil { - prior := util.AdjustReleaseTime(*release.GADate, true, "30", c.reqOptions.CacheOption.CRTimeRoundingFactor, c.reqOptions.CacheOption.CRTimeRoundingOffset) + gaTime := release.GADate.In(time.UTC) + prior := util.AdjustReleaseTime(gaTime, true, "30", c.reqOptions.CacheOption.CRTimeRoundingFactor, c.reqOptions.CacheOption.CRTimeRoundingOffset) timeRange.Start = &prior - timeRange.End = release.GADate + timeRange.End = &gaTime } timeRanges = append(timeRanges, timeRange) } diff --git a/pkg/api/componentreadiness/dataprovider/postgres/provider.go b/pkg/api/componentreadiness/dataprovider/postgres/provider.go index 0d8ba44605..87fbc89b1a 100644 --- a/pkg/api/componentreadiness/dataprovider/postgres/provider.go +++ b/pkg/api/componentreadiness/dataprovider/postgres/provider.go @@ -279,7 +279,7 @@ func (p *PostgresProvider) baseMatchesGAWindow(ctx context.Context, release stri return false } - gaDate := civil.DateOf(*rd.GADate) + gaDate := *rd.GADate if gaDate.After(civil.DateOf(time.Now().UTC())) { return false } diff --git a/pkg/api/componentreadiness/queryparamparser_test.go b/pkg/api/componentreadiness/queryparamparser_test.go index 7843ada39b..c764ed91b9 100644 --- a/pkg/api/componentreadiness/queryparamparser_test.go +++ b/pkg/api/componentreadiness/queryparamparser_test.go @@ -32,8 +32,8 @@ var ( func TestParseComponentReportRequest(t *testing.T) { releases := []v1.Release{ - {Release: "4.16", Status: "", GADate: util.DatePtr(2024, 6, 27, 0, 0, 0, 0, time.UTC)}, - {Release: "4.15", Status: "", GADate: util.DatePtr(2024, 2, 28, 0, 0, 0, 0, time.UTC)}, + {Release: "4.16", Status: "", GADate: util.CivilDatePtr(2024, time.June, 27)}, + {Release: "4.15", Status: "", GADate: util.CivilDatePtr(2024, time.February, 28)}, } allJobVariants := crtest.JobVariants{Variants: map[string][]string{ @@ -495,8 +495,8 @@ func TestHATEOASLinkCacheConsistency(t *testing.T) { roundingOffset := 4 * time.Hour releases := []v1.Release{ - {Release: "4.16", Status: "", GADate: util.DatePtr(2024, 6, 27, 0, 0, 0, 0, time.UTC)}, - {Release: "4.17", Status: "", GADate: util.DatePtr(2024, 12, 10, 0, 0, 0, 0, time.UTC)}, + {Release: "4.16", Status: "", GADate: util.CivilDatePtr(2024, time.June, 27)}, + {Release: "4.17", Status: "", GADate: util.CivilDatePtr(2024, time.December, 10)}, } allJobVariants := crtest.JobVariants{Variants: map[string][]string{ diff --git a/pkg/api/componentreadiness/test_details.go b/pkg/api/componentreadiness/test_details.go index fdf5baaf5b..58de753e8b 100644 --- a/pkg/api/componentreadiness/test_details.go +++ b/pkg/api/componentreadiness/test_details.go @@ -173,7 +173,7 @@ func (c *ComponentReportGenerator) GenerateDetailsReportForTest( return testdetails.Report{}, errs } - now := time.Now() + now := time.Now().UTC() componentJobRunTestReportStatus.GeneratedAt = &now // Generate the report for the main release that was originally requested: diff --git a/pkg/api/componentreadiness/triage_test.go b/pkg/api/componentreadiness/triage_test.go index 30575c5ced..9c082f73a1 100644 --- a/pkg/api/componentreadiness/triage_test.go +++ b/pkg/api/componentreadiness/triage_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "cloud.google.com/go/civil" "github.com/lib/pq" "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" @@ -611,7 +612,7 @@ func TestCompareTriageObjects(t *testing.T) { } func TestInjectRegressionHATEOASLinks(t *testing.T) { - ga421 := time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC) + ga421 := civil.Date{Year: 2025, Month: 6, Day: 1} releases := []v1.Release{ {Release: "4.20", GADate: &ga421}, {Release: "4.21", GADate: &ga421}, diff --git a/pkg/api/componentreadiness/utils/utils_test.go b/pkg/api/componentreadiness/utils/utils_test.go index 4aff7c59b7..636cf9b638 100644 --- a/pkg/api/componentreadiness/utils/utils_test.go +++ b/pkg/api/componentreadiness/utils/utils_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "cloud.google.com/go/civil" "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" "github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts" v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" @@ -15,8 +16,8 @@ import ( func TestGenerateTestDetailsURL(t *testing.T) { // Define releases with GA dates for all tests - ga419 := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) - ga420 := time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC) + ga419 := civil.Date{Year: 2025, Month: 1, Day: 1} + ga420 := civil.Date{Year: 2025, Month: 6, Day: 1} releases := []v1.Release{ { Release: "4.19", diff --git a/pkg/api/jira.go b/pkg/api/jira.go index 8e651f0776..70bc4b5ad4 100644 --- a/pkg/api/jira.go +++ b/pkg/api/jira.go @@ -11,20 +11,19 @@ func GetJIRAIncidentsFromDB(dbClient *db.DB, start, end *time.Time) ([]apitype.C // Rounding to start of next day because of https://github.com/fullcalendar/fullcalendar/issues/7413 now := time.Now().UTC() startOfDay := now.Truncate(24 * time.Hour) - startOfNextDay := startOfDay.Add(24 * time.Hour).Format("2006-01-02T15:04:05-07:00") + startOfNextDay := startOfDay.Add(24 * time.Hour) // Get JIRA Incidents for display in calendar incidents := make([]apitype.CalendarEvent, 0) res := dbClient.DB.Table("jira_incidents").Select(` start_time AS start, - COALESCE(DATE_TRUNC('day', resolution_time) + interval '1 day', '`+startOfNextDay+`') AS end, + COALESCE(DATE_TRUNC('day', resolution_time) + interval '1 day', ?) AS end, key as jira, key || ': ' || summary AS title, 'incident' AS phase, - 'TRUE' as all_day`, + 'TRUE' as all_day`, startOfNextDay, ). - Where(`(start_time, COALESCE(resolution_time, '`+startOfNextDay+`')) OVERLAPS (?, ?)`, start, end). - Where(`(start_time, resolution_time) OVERLAPS (?, ?)`, start, end). + Where(`(start_time, COALESCE(resolution_time, ?)) OVERLAPS (?, ?)`, startOfNextDay, start, end). Scan(&incidents) return incidents, res.Error diff --git a/pkg/api/job_runs.go b/pkg/api/job_runs.go index b20bb04cbe..721a08aaa2 100644 --- a/pkg/api/job_runs.go +++ b/pkg/api/job_runs.go @@ -165,7 +165,7 @@ const jobRunsBaseSelect = `prow_job_runs.id, prow_job_runs.succeeded, prow_job_runs.infrastructure_failure, prow_job_runs.known_failure, - (EXTRACT(epoch FROM (prow_job_runs."timestamp" AT TIME ZONE 'utc')) * 1000)::bigint AS "timestamp", + prow_job_runs."timestamp", prow_job_runs.id AS prow_id, prow_job_runs.cluster, prow_job_runs.labels, @@ -204,7 +204,7 @@ var columnAliases = map[string]string{ "brief_name": "regexp_replace(prow_jobs.name, 'periodic-ci-openshift-(multiarch|release)-(master|main)-(ci|nightly)-[0-9]+.[0-9]+-', '')", "prow_id": "prow_job_runs.id", "test_grid_url": "prow_job_runs.url", - "timestamp": `(EXTRACT(epoch FROM (prow_job_runs."timestamp" AT TIME ZONE 'utc')) * 1000)::bigint`, + "timestamp": `prow_job_runs."timestamp"`, "pull_request_link": "pp.link", "pull_request_sha": "pp.sha", "pull_request_org": "pp.org", diff --git a/pkg/api/jobs.go b/pkg/api/jobs.go index 30f67e5822..564c8282fd 100644 --- a/pkg/api/jobs.go +++ b/pkg/api/jobs.go @@ -8,6 +8,7 @@ import ( "strconv" "time" + "cloud.google.com/go/civil" log "github.com/sirupsen/logrus" apitype "github.com/openshift/sippy/pkg/apis/api" @@ -238,8 +239,8 @@ type jobDetail struct { type jobDetailAPIResult struct { Jobs []jobDetail `json:"jobs"` - Start int `json:"start"` - End int `json:"end"` + Start civil.Date `json:"start"` + End civil.Date `json:"end"` } func (jobs jobDetailAPIResult) limit(req *http.Request) jobDetailAPIResult { @@ -252,29 +253,31 @@ func (jobs jobDetailAPIResult) limit(req *http.Request) jobDetailAPIResult { return ret } -// JobDetailsReport runs the job details query without HTTP response handling. -func JobDetailsReport(dbc *db.DB, release, jobSearchStr string, reportEnd time.Time) ([]*models.ProwJobRun, error) { - since := reportEnd.Add(-14 * 24 * time.Hour) +// JobDetailsReport runs the job details query for the half-open date range [start, end). +func JobDetailsReport(dbc *db.DB, release, jobSearchStr string, start, end civil.Date) ([]*models.ProwJobRun, error) { prowJobRuns := make([]*models.ProwJobRun, 0) res := dbc.DB.Joins("ProwJob"). Where("name LIKE ?", "%"+jobSearchStr+"%"). - Where("timestamp > ?", since). + Where("timestamp >= ? AND timestamp < ?", start, end). Where("release = ?", release). - Preload("Tests", "status = ? AND prow_job_run_release = ? AND prow_job_run_timestamp > ?", 12, release, since). + Preload("Tests", "status = ? AND prow_job_run_release = ? AND prow_job_run_timestamp >= ? AND prow_job_run_timestamp < ?", 12, release, start, end). Preload("Tests.Test"). Find(&prowJobRuns) if res.Error != nil { return nil, res.Error } - log.WithFields(log.Fields{"prowJobRuns": len(prowJobRuns), "since": since}).Info("loaded ProwJobRuns from db") + log.WithFields(log.Fields{"prowJobRuns": len(prowJobRuns), "start": start, "end": end}).Info("loaded ProwJobRuns from db") return prowJobRuns, nil } +const jobDetailsLookbackDays = 14 + // PrintJobDetailsReportFromDB renders the detailed list of runs for matching jobs. func PrintJobDetailsReportFromDB(w http.ResponseWriter, req *http.Request, dbc *db.DB, release, jobSearchStr string, reportEnd time.Time) error { - var start, end int + end := civil.DateOf(reportEnd.UTC()) + start := end.AddDays(-jobDetailsLookbackDays) - prowJobRuns, err := JobDetailsReport(dbc, release, jobSearchStr, reportEnd) + prowJobRuns, err := JobDetailsReport(dbc, release, jobSearchStr, start, end.AddDays(1)) if err != nil { log.Errorf("error querying %s ProwJobRuns from db: %v", jobSearchStr, err) return err @@ -303,7 +306,7 @@ func PrintJobDetailsReportFromDB(w http.ResponseWriter, req *http.Request, dbc * InfrastructureFailure: pjr.InfrastructureFailure, KnownFailure: pjr.KnownFailure, Succeeded: pjr.Succeeded, - Timestamp: int(pjr.Timestamp.Unix() * 1000), + Timestamp: pjr.Timestamp, OverallResult: pjr.OverallResult, } jobDetails[jobName].Results = append(jobDetails[jobName].Results, newRun) diff --git a/pkg/api/releases.go b/pkg/api/releases.go index 4e509ca726..e395f039e2 100644 --- a/pkg/api/releases.go +++ b/pkg/api/releases.go @@ -9,6 +9,7 @@ import ( "sort" "time" + "cloud.google.com/go/civil" "github.com/lib/pq" pkgerrors "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -532,12 +533,10 @@ func transformRelease(r sippyv1.ReleaseRow) sippyv1.Release { Product: r.Product.StringVal, } if r.GADate.Valid { - gaDate := r.GADate.Date.In(time.UTC) - release.GADate = &gaDate + release.GADate = &r.GADate.Date } if r.DevelStartDate.IsValid() { - develStartDate := r.DevelStartDate.In(time.UTC) - release.DevelopmentStartDate = &develStartDate + release.DevelopmentStartDate = &r.DevelStartDate } if r.Capabilities != nil { for _, capability := range r.Capabilities { @@ -588,9 +587,10 @@ func GetReleaseDatesFromDB(ctx context.Context, dbc *db.DB, reqOptions reqopts.R for _, release := range releases { tr := crtest.ReleaseTimeRange{Release: release.Release} if release.GADate != nil { - prior := util.AdjustReleaseTime(*release.GADate, true, "30", reqOptions.CacheOption.CRTimeRoundingFactor, reqOptions.CacheOption.CRTimeRoundingOffset) + gaTime := release.GADate.In(time.UTC) + prior := util.AdjustReleaseTime(gaTime, true, "30", reqOptions.CacheOption.CRTimeRoundingFactor, reqOptions.CacheOption.CRTimeRoundingOffset) tr.Start = &prior - tr.End = release.GADate + tr.End = &gaTime } timeRanges = append(timeRanges, tr) } @@ -633,7 +633,7 @@ func DefinitionToRelease(def models.ReleaseDefinition) sippyv1.Release { // BuildReleasesResponse creates the API response structure for releases func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) apitype.Releases { - gaDateMap := make(map[string]time.Time) + gaDateMap := make(map[string]civil.Date) dateMap := make(map[string]apitype.ReleaseDates) response := apitype.Releases{ DeprecatedGADates: gaDateMap, diff --git a/pkg/api/releases_test.go b/pkg/api/releases_test.go index 9c6a778de2..0a4d0c0b7f 100644 --- a/pkg/api/releases_test.go +++ b/pkg/api/releases_test.go @@ -2,8 +2,8 @@ package api import ( "testing" - "time" + "cloud.google.com/go/civil" "github.com/lib/pq" "github.com/stretchr/testify/assert" @@ -14,8 +14,8 @@ import ( ) func TestDefinitionToRelease(t *testing.T) { - ga := time.Date(2026, 6, 9, 0, 0, 0, 0, time.UTC) - devStart := time.Date(2025, 12, 1, 0, 0, 0, 0, time.UTC) + ga := civil.Date{Year: 2026, Month: 6, Day: 9} + devStart := civil.Date{Year: 2025, Month: 12, Day: 1} tests := []struct { name string diff --git a/pkg/api/tests.go b/pkg/api/tests.go index 1fd421e162..c29c0c20aa 100644 --- a/pkg/api/tests.go +++ b/pkg/api/tests.go @@ -207,7 +207,7 @@ LIMIT 500` return outputs, nil } -func GetTestDurationsFromDB(dbc *db.DB, release, test string, filters *filter.Filter) (map[string]float64, error) { +func GetTestDurationsFromDB(dbc *db.DB, release, test string, filters *filter.Filter) (map[civil.Date]float64, error) { var includedVariants, excludedVariants []string if filters != nil { for _, f := range filters.Items { diff --git a/pkg/apis/api/recent_test_failures.go b/pkg/apis/api/recent_test_failures.go index 0e42822961..a5360bddff 100644 --- a/pkg/apis/api/recent_test_failures.go +++ b/pkg/apis/api/recent_test_failures.go @@ -75,6 +75,10 @@ func (r RecentTestFailure) GetNumericalValue(param string) (float64, error) { } } +func (r RecentTestFailure) GetTimestampValue(param string) (time.Time, error) { + return time.Time{}, fmt.Errorf("unknown timestamp field %s", param) +} + func (r RecentTestFailure) GetArrayValue(param string) ([]string, error) { return nil, fmt.Errorf("unknown array value field %s", param) } diff --git a/pkg/apis/api/types.go b/pkg/apis/api/types.go index 26106d1c52..b094e6c791 100644 --- a/pkg/apis/api/types.go +++ b/pkg/apis/api/types.go @@ -6,6 +6,7 @@ import ( "math/big" "time" + "cloud.google.com/go/civil" "github.com/lib/pq" "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" @@ -106,6 +107,10 @@ func (r Repository) GetNumericalValue(param string) (float64, error) { } } +func (r Repository) GetTimestampValue(param string) (time.Time, error) { + return time.Time{}, fmt.Errorf("unknown timestamp field %s", param) +} + func (r Repository) GetArrayValue(param string) ([]string, error) { return nil, fmt.Errorf("unknown array value field %s", param) } @@ -183,13 +188,23 @@ func (pr PullRequest) GetNumericalValue(param string) (float64, error) { return float64(pr.ID), nil case "number": return float64(pr.Number), nil - case "merged_at": - return float64(pr.MergedAt.Unix()), nil default: return 0, fmt.Errorf("unknown numerical field %s", param) } } +func (pr PullRequest) GetTimestampValue(param string) (time.Time, error) { + switch param { + case "merged_at": + if pr.MergedAt != nil { + return *pr.MergedAt, nil + } + return time.Time{}, nil + default: + return time.Time{}, fmt.Errorf("unknown timestamp field %s", param) + } +} + func (pr PullRequest) GetArrayValue(param string) ([]string, error) { return nil, fmt.Errorf("unknown array value field %s", param) } @@ -269,6 +284,8 @@ func (job Job) GetFieldType(param string) ColumnType { //nolint:goconst case "test_grid_url": return ColumnTypeString + case "last_pass": + return ColumnTypeTimestamp default: return ColumnTypeNumerical } @@ -319,13 +336,23 @@ func (job Job) GetNumericalValue(param string) (float64, error) { return float64(job.CurrentAverageDurationMinutes), nil case "previous_average_duration_minutes": return float64(job.PreviousAverageDurationMinutes), nil - case "last_pass": - return float64(job.LastPass.Unix()), nil default: return 0, fmt.Errorf("unknown numerical field %s", param) } } +func (job Job) GetTimestampValue(param string) (time.Time, error) { + switch param { + case "last_pass": + if job.LastPass == nil { + return time.Time{}, nil + } + return *job.LastPass, nil + default: + return time.Time{}, fmt.Errorf("unknown timestamp field %s", param) + } +} + func (job Job) GetArrayValue(param string) ([]string, error) { switch param { case "variants": @@ -354,7 +381,7 @@ type JobRun struct { InfrastructureFailure bool `json:"infrastructure_failure"` KnownFailure bool `json:"known_failure"` Succeeded bool `json:"succeeded"` - Timestamp int `json:"timestamp"` + Timestamp time.Time `json:"timestamp"` OverallResult v1.JobOverallResult `json:"overall_result"` PullRequestOrg string `json:"pull_request_org"` PullRequestRepo string `json:"pull_request_repo"` @@ -393,7 +420,7 @@ func (run JobRun) GetFieldType(param string) ColumnType { case "test_grid_url": return ColumnTypeString case "timestamp": - return ColumnTypeNumerical + return ColumnTypeTimestamp case "pull_request_org": return ColumnTypeString case "pull_request_repo": @@ -440,13 +467,20 @@ func (run JobRun) GetNumericalValue(param string) (float64, error) { return float64(run.ID), nil case "test_failures": return float64(run.TestFailures), nil - case "timestamp": - return float64(run.Timestamp), nil default: return 0, fmt.Errorf("unknown numerical field %s", param) } } +func (run JobRun) GetTimestampValue(param string) (time.Time, error) { + switch param { + case "timestamp": + return run.Timestamp, nil + default: + return time.Time{}, fmt.Errorf("unknown timestamp field %s", param) + } +} + func (run JobRun) GetArrayValue(param string) ([]string, error) { switch param { case "failed_test_names": @@ -609,6 +643,10 @@ func (test Test) GetNumericalValue(param string) (float64, error) { } } +func (test Test) GetTimestampValue(param string) (time.Time, error) { + return time.Time{}, fmt.Errorf("unknown timestamp field %s", param) +} + func (test Test) GetArrayValue(param string) ([]string, error) { switch param { case "tags": @@ -768,6 +806,10 @@ func (test TestBQ) GetNumericalValue(param string) (float64, error) { } } +func (test TestBQ) GetTimestampValue(param string) (time.Time, error) { + return time.Time{}, fmt.Errorf("unknown timestamp field %s", param) +} + func (test TestBQ) GetArrayValue(param string) ([]string, error) { switch param { case "tags": @@ -868,13 +910,13 @@ type JobPayload struct { // CalendarEvent is an API type representing a FullCalendar.io event type, for use // with calendering. type CalendarEvent struct { - Title string `json:"title"` - Start string `json:"start"` - End string `json:"end"` - AllDay bool `json:"allDay"` - Display string `json:"display,omitempty"` - Phase string `json:"phase"` - JIRA string `json:"jira"` + Title string `json:"title"` + Start time.Time `json:"start"` + End *time.Time `json:"end,omitempty"` + AllDay bool `json:"allDay"` + Display string `json:"display,omitempty"` + Phase string `json:"phase"` + JIRA string `json:"jira"` } type BuildClusterHealthAnalysis struct { @@ -910,8 +952,8 @@ type TestOutputBigQuery struct { } type ReleaseDates struct { - GA *time.Time `json:"ga,omitempty"` - DevelopmentStart *time.Time `json:"development_start,omitempty"` + GA *civil.Date `json:"ga,omitempty"` + DevelopmentStart *civil.Date `json:"development_start,omitempty"` } type Release struct { // this is the Release that goes out to the UI Name string `json:"name"` @@ -922,7 +964,7 @@ type Release struct { // this is the Release that goes out to the UI } type Releases struct { Releases []string `json:"releases"` - DeprecatedGADates map[string]time.Time `json:"ga_dates"` + DeprecatedGADates map[string]civil.Date `json:"ga_dates"` Dates map[string]ReleaseDates `json:"dates"` LastUpdated time.Time `json:"last_updated"` ReleaseAttrs map[string]Release `json:"release_attrs"` @@ -1031,16 +1073,16 @@ type DisruptionReportRow struct { } type BackendDisruptionRunRow struct { - BackendName string `json:"backend_name"` - DisruptionSeconds int `json:"disruption_seconds"` - JobName string `json:"job_name"` - JobRunName string `json:"job_run_name"` - JobRunStartTime string `json:"job_run_start_time"` - JobRunEndTime string `json:"job_run_end_time"` - Cluster string `json:"cluster"` - ReleaseTag string `json:"release_tag"` - MasterNodesUpdated string `json:"master_nodes_updated"` - JobRunStatus string `json:"job_run_status"` + BackendName string `json:"backend_name"` + DisruptionSeconds int `json:"disruption_seconds"` + JobName string `json:"job_name"` + JobRunName string `json:"job_run_name"` + JobRunStartTime *time.Time `json:"job_run_start_time"` + JobRunEndTime *time.Time `json:"job_run_end_time"` + Cluster string `json:"cluster"` + ReleaseTag string `json:"release_tag"` + MasterNodesUpdated string `json:"master_nodes_updated"` + JobRunStatus string `json:"job_run_status"` } type BackendDisruptionRunsResult struct { @@ -1145,6 +1187,10 @@ func (fg FeatureGate) GetNumericalValue(param string) (float64, error) { } } +func (fg FeatureGate) GetTimestampValue(param string) (time.Time, error) { + return time.Time{}, fmt.Errorf("unknown timestamp field %s", param) +} + func (fg FeatureGate) GetArrayValue(param string) ([]string, error) { switch param { case "enabled": diff --git a/pkg/apis/sippy/v1/types.go b/pkg/apis/sippy/v1/types.go index 0daa96f0fe..b6d871e797 100644 --- a/pkg/apis/sippy/v1/types.go +++ b/pkg/apis/sippy/v1/types.go @@ -1,8 +1,6 @@ package v1 import ( - "time" - "cloud.google.com/go/bigquery" "cloud.google.com/go/civil" bugsv1 "github.com/openshift/sippy/pkg/apis/bugs/v1" @@ -73,8 +71,8 @@ type FailureGroup struct { type Release struct { // this is the Release that gets cached Release string Status string - GADate *time.Time - DevelopmentStartDate *time.Time + GADate *civil.Date + DevelopmentStartDate *civil.Date PreviousRelease string Capabilities map[ReleaseCapability]bool Product string diff --git a/pkg/apis/sippyprocessing/v1/types.go b/pkg/apis/sippyprocessing/v1/types.go index b4ab8e65fd..804ad9dd32 100644 --- a/pkg/apis/sippyprocessing/v1/types.go +++ b/pkg/apis/sippyprocessing/v1/types.go @@ -3,6 +3,8 @@ package v1 import ( + "time" + bugsv1 "github.com/openshift/sippy/pkg/apis/bugs/v1" ) @@ -114,10 +116,9 @@ type JobRunResult struct { // InfrastructureFailure is true if the job run failed, for reasons which appear to be related to test/CI infra. InfrastructureFailure bool `json:"infrastructureFailure"` // KnownFailure is true if the job run failed, but we found a bug that is likely related already filed. - KnownFailure bool `json:"knownFailure"` - Succeeded bool `json:"succeeded"` - // Timestamp is milliseconds since epoch when this job was run. - Timestamp int `json:"timestamp"` + KnownFailure bool `json:"knownFailure"` + Succeeded bool `json:"succeeded"` + Timestamp time.Time `json:"timestamp"` OverallResult JobOverallResult `json:"result"` } @@ -147,7 +148,7 @@ type RawJobResult struct { // It is used to build up a complete set of successes and failure, but until all the testgrid results have been checked, it will be incomplete type RawTestResult struct { Name string - Timestamps []int + Timestamps []time.Time Successes int Failures int Flakes int @@ -190,8 +191,7 @@ type RawJobRunResult struct { // Overall result OverallResult JobOverallResult - // Timestamp - Timestamp int + Timestamp time.Time } type OperatorState struct { diff --git a/pkg/apis/workloadmetrics/v1/types.go b/pkg/apis/workloadmetrics/v1/types.go index 639b21dc87..f27cbee455 100644 --- a/pkg/apis/workloadmetrics/v1/types.go +++ b/pkg/apis/workloadmetrics/v1/types.go @@ -178,6 +178,10 @@ func (wm *WorkloadMetricsRow) GetNumericalValue(param string) (float64, error) { } } +func (wm *WorkloadMetricsRow) GetTimestampValue(param string) (time.Time, error) { + return time.Time{}, fmt.Errorf("unknown timestamp field %s", param) +} + func (wm *WorkloadMetricsRow) GetArrayValue(param string) ([]string, error) { return nil, fmt.Errorf("unknown array value field %s", param) } diff --git a/pkg/cache/bigquerycache/bigquery.go b/pkg/cache/bigquerycache/bigquery.go index f480d33a65..cd081f36da 100644 --- a/pkg/cache/bigquerycache/bigquery.go +++ b/pkg/cache/bigquerycache/bigquery.go @@ -170,8 +170,8 @@ func (c Cache) findCacheEntry(ctx context.Context, key string) (CacheRecord, err // limit the columns so we don't query too much data query := c.client.Query(ctx, bqlabel.CacheLookup, fmt.Sprintf( "SELECT modified_time, expiration, uuid FROM `%s.%s` "+ - `WHERE %s > TIMESTAMP(@expByNowTime) - AND expiration > TIMESTAMP(@expTime) + `WHERE %s > @expByNowTime + AND expiration > @expTime AND key = @keyParam ORDER BY %s DESC LIMIT 1`, c.client.Dataset, cachedTable, @@ -179,11 +179,11 @@ func (c Cache) findCacheEntry(ctx context.Context, key string) (CacheRecord, err query.Parameters = []bigquery.QueryParameter{ { // limit partitions to those that could contain un-expired entries Name: "expByNowTime", - Value: time.Now().Add(-1 * c.maxExpiration).Format(time.RFC3339), + Value: time.Now().Add(-1 * c.maxExpiration), }, { // entry itself is not already expired Name: "expTime", - Value: time.Now().Format(time.RFC3339), + Value: time.Now(), }, { Name: "keyParam", @@ -207,7 +207,7 @@ func (c Cache) getFullCacheRecords(ctx context.Context, key string, metadataReco // we have to add a +/- 5 second grace as exact match doesn't work query := c.client.Query(ctx, bqlabel.CacheLookup, fmt.Sprintf( "SELECT * FROM `%s.%s` "+ - `WHERE %s BETWEEN TIMESTAMP(@tsLower) AND TIMESTAMP(@tsUpper) + `WHERE %s BETWEEN @tsLower AND @tsUpper AND key = @keyParam AND uuid = @uuidParam ORDER BY chunk_index ASC`, @@ -216,11 +216,11 @@ func (c Cache) getFullCacheRecords(ctx context.Context, key string, metadataReco query.Parameters = []bigquery.QueryParameter{ { Name: "tsLower", - Value: metadataRecord.Modified.Add(-5 * time.Second).Format(time.RFC3339), + Value: metadataRecord.Modified.Add(-5 * time.Second), }, { Name: "tsUpper", - Value: metadataRecord.Modified.Add(5 * time.Second).Format(time.RFC3339), + Value: metadataRecord.Modified.Add(5 * time.Second), }, { Name: "keyParam", diff --git a/pkg/dataloader/gateststatus/loader.go b/pkg/dataloader/gateststatus/loader.go index 42ebde95d9..4a94f156bd 100644 --- a/pkg/dataloader/gateststatus/loader.go +++ b/pkg/dataloader/gateststatus/loader.go @@ -62,7 +62,11 @@ func (l *GATestStatusLoader) Load() { loaded := 0 for _, rel := range allReleases { - gaDate := civil.DateOf(rel.GADate.UTC()) + if rel.GADate == nil { + l.errs = append(l.errs, fmt.Errorf("release %s has no GA date", rel.Release)) + continue + } + gaDate := *rel.GADate if !l.force && rel.LoadedGADate != nil && *rel.LoadedGADate == gaDate { log.WithField("release", rel.Release).Debug("ga-test-status: already loaded, skipping") continue diff --git a/pkg/dataloader/prowloader/pgwriter/pgwriter.go b/pkg/dataloader/prowloader/pgwriter/pgwriter.go index d6cfcdb7dc..3c355c7d99 100644 --- a/pkg/dataloader/prowloader/pgwriter/pgwriter.go +++ b/pkg/dataloader/prowloader/pgwriter/pgwriter.go @@ -370,7 +370,7 @@ func insertTestResults(ctx context.Context, tx pgx.Tx) error { type releaseDate struct { release string - date time.Time + date civil.Date } func upsertSummaryTables(ctx context.Context, tx pgx.Tx, currentDate civil.Date) error { @@ -394,11 +394,10 @@ func upsertSummaryTables(ctx context.Context, tx pgx.Tx, currentDate civil.Date) stepStart := time.Now() for _, rd := range releaseDates { - day := civil.DateOf(rd.date) - if err := ensureDailyTotalRows(ctx, tx, day, rd.release); err != nil { + if err := ensureDailyTotalRows(ctx, tx, rd.date, rd.release); err != nil { return err } - if err := updateDailyTotals(ctx, tx, day, rd.release); err != nil { + if err := updateDailyTotals(ctx, tx, rd.date, rd.release); err != nil { return err } } @@ -407,9 +406,8 @@ func upsertSummaryTables(ctx context.Context, tx pgx.Tx, currentDate civil.Date) stepStart = time.Now() releaseMinDate := make(map[string]civil.Date) for _, rd := range releaseDates { - day := civil.DateOf(rd.date) - if earliest, ok := releaseMinDate[rd.release]; !ok || day.Before(earliest) { - releaseMinDate[rd.release] = day + if earliest, ok := releaseMinDate[rd.release]; !ok || rd.date.Before(earliest) { + releaseMinDate[rd.release] = rd.date } } for release, minDate := range releaseMinDate { @@ -465,11 +463,12 @@ func queryReleaseDates(ctx context.Context, tx pgx.Tx) ([]releaseDate, error) { defer rows.Close() var releaseDates []releaseDate for rows.Next() { - var rd releaseDate - if err := rows.Scan(&rd.release, &rd.date); err != nil { + var release string + var date time.Time + if err := rows.Scan(&release, &date); err != nil { return nil, fmt.Errorf("scanning release-date: %w", err) } - releaseDates = append(releaseDates, rd) + releaseDates = append(releaseDates, releaseDate{release: release, date: civil.DateOf(date)}) } if err := rows.Err(); err != nil { return nil, fmt.Errorf("iterating release-dates: %w", err) diff --git a/pkg/dataloader/prowloader/prow.go b/pkg/dataloader/prowloader/prow.go index 45bcd866d7..13809c1125 100644 --- a/pkg/dataloader/prowloader/prow.go +++ b/pkg/dataloader/prowloader/prow.go @@ -138,7 +138,7 @@ func resolveFrom(since *time.Time, to time.Time) time.Time { } func (pl *ProwLoader) resolveLoadSince() time.Time { - return resolveFrom(pl.loadSince, time.Now()) + return resolveFrom(pl.loadSince, time.Now().UTC()) } var clusterDataDateTimeName = regexp.MustCompile(`cluster-data_(?P.*)-(?P