diff --git a/README.md b/README.md index a1c8e13..9340048 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ A Web GUI written in Go to manage S3 buckets from any provider. - Upload new objects to a bucket - Download object from a bucket - Delete an object in a bucket +- Show object metadata (including user metadata) and object versions ## Usage @@ -33,6 +34,8 @@ The application can be configured with the following environment variables: - `ALLOW_DELETE`: Enable buttons to delete objects (defaults to `true`) - `FORCE_DOWNLOAD`: Add response headers for object downloading instead of opening in a new tab (defaults to `true`) - `LIST_RECURSIVE`: List all objects in buckets recursively (defaults to `false`) +- `SHOW_VERSIONS`: Show all object versions in bucket view and enable version-specific downloads (defaults to `false`; bucket must have versioning enabled) +- `SHOW_METADATA`: Show the object metadata action and enable the metadata endpoint (defaults to `true`) - `TZ`: IANA timezone used when displaying object Last Modified times (defaults to UTC; for example `Europe/Berlin`) - `BUCKET_NAME`: Restrict the buckets view to a single named bucket (defaults to unset, showing all buckets) - `USE_IAM`: Use IAM role instead of key pair (defaults to `false`) diff --git a/internal/app/s3manager/bucket_view.go b/internal/app/s3manager/bucket_view.go index e27a68e..1e67723 100644 --- a/internal/app/s3manager/bucket_view.go +++ b/internal/app/s3manager/bucket_view.go @@ -1,6 +1,7 @@ package s3manager import ( + "context" "fmt" "html/template" "io/fs" @@ -20,39 +21,161 @@ const defaultPerPage = 25 // objectWithIcon represents an S3 object with additional display properties type objectWithIcon struct { - Key string - Size int64 - SizeDisplay string - LastModified time.Time - Owner string - Icon string - IsFolder bool - DisplayName string + Key string + Size int64 + SizeDisplay string + LastModified time.Time + Owner string + Icon string + IsFolder bool + DisplayName string + VersionID string + IsLatest bool + IsDeleteMarker bool + VersionCount int + GroupIndex int + IsPrimaryVersion bool +} + +// annotateVersionGroups sets VersionCount, GroupIndex and IsPrimaryVersion on +// each object so the template can collapse older versions under their latest +// version by default. IsPrimaryVersion picks exactly one visible row per key: +// the one the provider marked IsLatest, or (since some S3-compatible providers +// leave IsLatest unset — notably folder entries synthesized from +// CommonPrefixes, which are never version-aware) the first entry seen for that +// key. Relying on the raw IsLatest flag alone would hide every row in a group +// where no entry has it set, making the bucket appear empty. +func annotateVersionGroups(objs []objectWithIcon) { + counts := make(map[string]int, len(objs)) + groupIndex := make(map[string]int, len(objs)) + primaryIndex := make(map[string]int, len(objs)) + nextIndex := 0 + + for i, obj := range objs { + counts[obj.Key]++ + if _, ok := groupIndex[obj.Key]; !ok { + groupIndex[obj.Key] = nextIndex + nextIndex++ + primaryIndex[obj.Key] = i + } else if obj.IsLatest { + primaryIndex[obj.Key] = i + } + } + + for i := range objs { + key := objs[i].Key + objs[i].VersionCount = counts[key] + objs[i].GroupIndex = groupIndex[key] + objs[i].IsPrimaryVersion = primaryIndex[key] == i + } +} + +// listObjectsOptions builds the minio.ListObjectsOptions used to list a bucket's objects. +func listObjectsOptions(listRecursive, showVersions bool, prefix string) minio.ListObjectsOptions { + return minio.ListObjectsOptions{ + Recursive: listRecursive, + Prefix: prefix, + WithVersions: showVersions, + } +} + +// listObjectsForBucketView lists a bucket's objects, converting each minio.ObjectInfo +// into an objectWithIcon. If showVersions is set but the versioned listing fails, or +// comes back empty (some S3-compatible providers don't support listing object +// versions and either reject the request outright or silently return nothing +// instead of erroring), it transparently falls back to a normal listing so the +// bucket can still be browsed. The returned bool reports whether version +// information is actually present in the result. +func listObjectsForBucketView(ctx context.Context, s3 S3, bucketName, path string, listRecursive, showVersions bool) ([]objectWithIcon, bool, error) { + if !showVersions { + objs, err := collectObjects(ctx, s3, bucketName, path, listObjectsOptions(listRecursive, false, path)) + return objs, false, err + } + + objs, err := collectObjects(ctx, s3, bucketName, path, listObjectsOptions(listRecursive, true, path)) + if err == nil && len(objs) > 0 { + return objs, true, nil + } + + fallbackObjs, fallbackErr := collectObjects(ctx, s3, bucketName, path, listObjectsOptions(listRecursive, false, path)) + if fallbackErr != nil { + return nil, false, fallbackErr + } + return fallbackObjs, false, nil +} + +// collectObjects drains an S3 ListObjects channel into a slice, returning the +// first error encountered (if any) instead of a partial, half-listed result. +func collectObjects(ctx context.Context, s3 S3, bucketName, path string, opts minio.ListObjectsOptions) ([]objectWithIcon, error) { + var objs []objectWithIcon + objectCh := s3.ListObjects(ctx, bucketName, opts) + for object := range objectCh { + if object.Err != nil { + return nil, object.Err + } + objs = append(objs, toObjectWithIcon(object, path)) + } + return objs, nil +} + +// friendlyListObjectsErrorMessage turns a raw S3 listing error into an +// actionable, user-facing message for the bucket view's error banner. +func friendlyListObjectsErrorMessage(err error, bucketName, instanceName string) string { + msg := err.Error() + + switch { + case strings.Contains(msg, "AccessDenied") || strings.Contains(msg, "InvalidAccessKeyId") || strings.Contains(msg, "SignatureDoesNotMatch"): + return fmt.Sprintf("Unable to access bucket '%s' on S3 instance '%s'. Please check the credentials and try switching to another instance.", bucketName, instanceName) + case strings.Contains(msg, ErrBucketDoesNotExist): + return fmt.Sprintf("Bucket '%s' does not exist on S3 instance '%s'. Please try switching to another instance or go back to the buckets list.", bucketName, instanceName) + default: + return fmt.Sprintf("Unable to list objects in bucket '%s' on S3 instance '%s': %s", bucketName, instanceName, msg) + } +} + +// toObjectWithIcon converts a minio.ObjectInfo into the template-facing objectWithIcon. +func toObjectWithIcon(object minio.ObjectInfo, path string) objectWithIcon { + return objectWithIcon{ + Key: object.Key, + Size: object.Size, + SizeDisplay: FormatFileSize(object.Size), + LastModified: object.LastModified, + Owner: object.Owner.DisplayName, + Icon: icon(object.Key), + IsFolder: strings.HasSuffix(object.Key, "/"), + DisplayName: strings.TrimSuffix(strings.TrimPrefix(object.Key, path), "/"), + VersionID: object.VersionID, + IsLatest: object.IsLatest, + IsDeleteMarker: object.IsDeleteMarker, + } } // HandleBucketView shows the details page of a bucket. -func HandleBucketView(s3 S3, templates fs.FS, allowDelete bool, listRecursive bool, rootURL string) http.HandlerFunc { +func HandleBucketView(s3 S3, templates fs.FS, allowDelete bool, listRecursive bool, rootURL string, showVersions bool, showMetadata bool) http.HandlerFunc { type pageData struct { - RootURL string - BucketName string - Objects []objectWithIcon - AllowDelete bool - Paths []string - CurrentPath string - Endpoint string - CurrentS3 *S3Instance - S3Instances []*S3Instance - HasError bool - ErrorMessage string - SortBy string - SortOrder string - Page int - PerPage int - TotalItems int - TotalPages int - HasPrevPage bool - HasNextPage bool - Search string + RootURL string + BucketName string + Objects []objectWithIcon + AllowDelete bool + Paths []string + CurrentPath string + Endpoint string + CurrentS3 *S3Instance + S3Instances []*S3Instance + HasError bool + ErrorMessage string + SortBy string + SortOrder string + Page int + PerPage int + TotalItems int + TotalPages int + HasPrevPage bool + HasNextPage bool + Search string + ShowVersions bool + VersionsUnavailable bool + ShowMetadata bool } return func(w http.ResponseWriter, r *http.Request) { @@ -95,31 +218,20 @@ func HandleBucketView(s3 S3, templates fs.FS, allowDelete bool, listRecursive bo // Get search parameter search := strings.TrimSpace(r.URL.Query().Get("search")) - var objs []objectWithIcon - opts := minio.ListObjectsOptions{ - Recursive: listRecursive, - Prefix: path, + objs, versionsShown, err := listObjectsForBucketView(r.Context(), s3, bucketName, path, listRecursive, showVersions) + if err != nil { + handleHTTPError(w, fmt.Errorf("error listing objects: %w", err)) + return } - objectCh := s3.ListObjects(r.Context(), bucketName, opts) - for object := range objectCh { - if object.Err != nil { - handleHTTPError(w, fmt.Errorf("error listing objects: %w", object.Err)) - return - } - obj := objectWithIcon{ - Key: object.Key, - Size: object.Size, - SizeDisplay: FormatFileSize(object.Size), - LastModified: object.LastModified, - Owner: object.Owner.DisplayName, - Icon: icon(object.Key), - IsFolder: strings.HasSuffix(object.Key, "/"), - DisplayName: strings.TrimSuffix(strings.TrimPrefix(object.Key, path), "/"), - } - objs = append(objs, obj) + if versionsShown { + annotateVersionGroups(objs) } + // Only warn about unavailable versions when there is content to show; + // an empty bucket legitimately produces an empty versioned listing. + versionsUnavailable := showVersions && !versionsShown && len(objs) > 0 + // Filter objects based on search query if search != "" { searchLower := strings.ToLower(search) @@ -134,55 +246,34 @@ func HandleBucketView(s3 S3, templates fs.FS, allowDelete bool, listRecursive bo objs = filteredObjs } - // Sort objects based on sortBy and sortOrder - sortObjects(objs, sortBy, sortOrder) - - // Calculate pagination - totalItems := len(objs) - totalPages := (totalItems + perPage - 1) / perPage - if totalPages == 0 { - totalPages = 1 - } - if page > totalPages { - page = totalPages - } - - // Paginate objects - start := (page - 1) * perPage - end := start + perPage - if start < 0 { - start = 0 - } - if end > totalItems { - end = totalItems - } - if start < totalItems { - objs = objs[start:end] - } else { - objs = []objectWithIcon{} - } + // Sort and paginate; versions of the same object stay together + var totalItems, totalPages int + objs, totalItems, totalPages, page = sortAndPaginateObjects(objs, sortBy, sortOrder, page, perPage, false, versionsShown) data := pageData{ - RootURL: rootURL, - BucketName: bucketName, - Objects: objs, - AllowDelete: allowDelete, - Paths: removeEmptyStrings(strings.Split(path, "/")), - CurrentPath: path, - Endpoint: s3.EndpointURL().String(), - CurrentS3: nil, - S3Instances: nil, - HasError: false, - ErrorMessage: "", - SortBy: sortBy, - SortOrder: sortOrder, - Page: page, - PerPage: perPage, - TotalItems: totalItems, - TotalPages: totalPages, - HasPrevPage: page > 1, - HasNextPage: page < totalPages, - Search: search, + RootURL: rootURL, + BucketName: bucketName, + Objects: objs, + AllowDelete: allowDelete, + Paths: removeEmptyStrings(strings.Split(path, "/")), + CurrentPath: path, + Endpoint: s3.EndpointURL().String(), + CurrentS3: nil, + S3Instances: nil, + HasError: false, + ErrorMessage: "", + SortBy: sortBy, + SortOrder: sortOrder, + Page: page, + PerPage: perPage, + TotalItems: totalItems, + TotalPages: totalPages, + HasPrevPage: page > 1, + HasNextPage: page < totalPages, + Search: search, + ShowVersions: versionsShown, + VersionsUnavailable: versionsUnavailable, + ShowMetadata: showMetadata, } funcMap := template.FuncMap{ @@ -247,21 +338,63 @@ func removeEmptyStrings(input []string) []string { return result } -// sortObjects sorts the objects based on the specified field and order -func sortObjects(objs []objectWithIcon, sortBy, sortOrder string) { - sort.Slice(objs, func(i, j int) bool { +// groupObjects splits objs into version groups that move as one unit through +// sorting and pagination. Objects keep their listing order within a group. +// Without version grouping every object is its own group. +func groupObjects(objs []objectWithIcon, grouped bool) [][]objectWithIcon { + if !grouped { + groups := make([][]objectWithIcon, len(objs)) + for i := range objs { + groups[i] = objs[i : i+1 : i+1] + } + return groups + } + + positions := make(map[int]int, len(objs)) + var groups [][]objectWithIcon + for _, obj := range objs { + pos, ok := positions[obj.GroupIndex] + if !ok { + pos = len(groups) + positions[obj.GroupIndex] = pos + groups = append(groups, nil) + } + groups[pos] = append(groups[pos], obj) + } + return groups +} + +// primaryObject returns the row that represents a group when sorting: the one +// marked IsPrimaryVersion by annotateVersionGroups, or the first row otherwise. +func primaryObject(group []objectWithIcon) objectWithIcon { + for _, obj := range group { + if obj.IsPrimaryVersion { + return obj + } + } + return group[0] +} + +// sortObjectGroups sorts version groups based on the specified field and order, +// comparing groups by their primary row so all versions of a key move as one +// unit. The stable sort preserves the S3 listing order between equal groups. +func sortObjectGroups(groups [][]objectWithIcon, sortBy, sortOrder string) { + sort.SliceStable(groups, func(i, j int) bool { + a := primaryObject(groups[i]) + b := primaryObject(groups[j]) + var less bool switch sortBy { case "size": - less = objs[i].Size < objs[j].Size + less = a.Size < b.Size case "owner": - less = strings.ToLower(objs[i].Owner) < strings.ToLower(objs[j].Owner) + less = strings.ToLower(a.Owner) < strings.ToLower(b.Owner) case "lastModified": - less = objs[i].LastModified.Before(objs[j].LastModified) + less = a.LastModified.Before(b.LastModified) case "key": fallthrough default: - less = strings.ToLower(objs[i].DisplayName) < strings.ToLower(objs[j].DisplayName) + less = strings.ToLower(a.DisplayName) < strings.ToLower(b.DisplayName) } if sortOrder == "desc" { @@ -270,3 +403,45 @@ func sortObjects(objs []objectWithIcon, sortBy, sortOrder string) { return less }) } + +func flattenGroups(groups [][]objectWithIcon) []objectWithIcon { + objs := make([]objectWithIcon, 0, len(groups)) + for _, group := range groups { + objs = append(objs, group...) + } + return objs +} + +// sortAndPaginateObjects sorts objects and slices out the requested page. When +// grouped is set (versioned listing), all versions of a key travel together: +// groups are ordered by their primary row and are never split across page +// boundaries, and totalItems counts objects, not individual versions. showAll +// disables pagination. It returns the page's objects, the total item count, +// the total page count, and the page number clamped to the valid range. +func sortAndPaginateObjects(objs []objectWithIcon, sortBy, sortOrder string, page, perPage int, showAll, grouped bool) ([]objectWithIcon, int, int, int) { + groups := groupObjects(objs, grouped) + sortObjectGroups(groups, sortBy, sortOrder) + + totalItems := len(groups) + if showAll { + return flattenGroups(groups), totalItems, 1, 1 + } + + totalPages := (totalItems + perPage - 1) / perPage + if totalPages == 0 { + totalPages = 1 + } + if page > totalPages { + page = totalPages + } + + start := (page - 1) * perPage + end := start + perPage + if end > totalItems { + end = totalItems + } + if start >= totalItems { + return []objectWithIcon{}, totalItems, totalPages, page + } + return flattenGroups(groups[start:end]), totalItems, totalPages, page +} diff --git a/internal/app/s3manager/bucket_view_internal_test.go b/internal/app/s3manager/bucket_view_internal_test.go new file mode 100644 index 0000000..840547f --- /dev/null +++ b/internal/app/s3manager/bucket_view_internal_test.go @@ -0,0 +1,103 @@ +package s3manager + +import ( + "testing" + "time" + + "github.com/matryer/is" +) + +func TestSortAndPaginateObjects(t *testing.T) { + t.Parallel() + + // Two versions of a.txt (latest first, as S3 lists them) and one b.txt. + // Sorted flat by size this would interleave to a(1), b(50), a(100). + versionedObjs := func() []objectWithIcon { + objs := []objectWithIcon{ + {Key: "a.txt", DisplayName: "a.txt", VersionID: "v2", IsLatest: true, Size: 100}, + {Key: "a.txt", DisplayName: "a.txt", VersionID: "v1", Size: 1}, + {Key: "b.txt", DisplayName: "b.txt", VersionID: "v1", IsLatest: true, Size: 50}, + } + annotateVersionGroups(objs) + return objs + } + + t.Run("keeps versions adjacent when sorting by size", func(t *testing.T) { + t.Parallel() + is := is.New(t) + + objs, totalItems, totalPages, page := sortAndPaginateObjects(versionedObjs(), "size", "asc", 1, 25, false, true) + + is.Equal(2, totalItems) // groups, not versions + is.Equal(1, totalPages) + is.Equal(1, page) + is.Equal(3, len(objs)) + // b.txt (primary size 50) sorts before the a.txt group (primary size 100), + // and a.txt keeps its listing order (newest version first). + is.Equal("b.txt", objs[0].Key) + is.Equal("v2", objs[1].VersionID) + is.Equal("v1", objs[2].VersionID) + }) + + t.Run("never splits a version group across pages", func(t *testing.T) { + t.Parallel() + is := is.New(t) + + objs, totalItems, totalPages, page := sortAndPaginateObjects(versionedObjs(), "key", "asc", 1, 1, false, true) + + is.Equal(2, totalItems) + is.Equal(2, totalPages) + is.Equal(1, page) + // Page 1 holds the whole a.txt group. + is.Equal(2, len(objs)) + is.Equal("a.txt", objs[0].Key) + is.Equal("a.txt", objs[1].Key) + + objs, _, _, page = sortAndPaginateObjects(versionedObjs(), "key", "asc", 2, 1, false, true) + is.Equal(2, page) + is.Equal(1, len(objs)) + is.Equal("b.txt", objs[0].Key) + }) + + t.Run("clamps the page number to the last page", func(t *testing.T) { + t.Parallel() + is := is.New(t) + + objs, _, totalPages, page := sortAndPaginateObjects(versionedObjs(), "key", "asc", 99, 1, false, true) + + is.Equal(2, totalPages) + is.Equal(2, page) + is.Equal("b.txt", objs[0].Key) + }) + + t.Run("returns everything when showAll is set", func(t *testing.T) { + t.Parallel() + is := is.New(t) + + objs, totalItems, totalPages, page := sortAndPaginateObjects(versionedObjs(), "key", "asc", 3, 1, true, true) + + is.Equal(2, totalItems) + is.Equal(1, totalPages) + is.Equal(1, page) + is.Equal(3, len(objs)) + }) + + t.Run("sorts flat when grouping is disabled", func(t *testing.T) { + t.Parallel() + is := is.New(t) + + now := time.Now() + objs := []objectWithIcon{ + {Key: "b.txt", DisplayName: "b.txt", LastModified: now}, + {Key: "a.txt", DisplayName: "a.txt", LastModified: now.Add(time.Hour)}, + } + + sorted, totalItems, totalPages, page := sortAndPaginateObjects(objs, "lastModified", "desc", 1, 25, false, false) + + is.Equal(2, totalItems) + is.Equal(1, totalPages) + is.Equal(1, page) + is.Equal("a.txt", sorted[0].Key) + is.Equal("b.txt", sorted[1].Key) + }) +} diff --git a/internal/app/s3manager/bucket_view_test.go b/internal/app/s3manager/bucket_view_test.go index 5c9c84b..df961ea 100644 --- a/internal/app/s3manager/bucket_view_test.go +++ b/internal/app/s3manager/bucket_view_test.go @@ -28,8 +28,11 @@ func TestHandleBucketView(t *testing.T) { bucketName string rootUrl string path string + showVersions bool + showMetadata bool expectedStatusCode int expectedBodyContains string + unexpectedInBody []string }{ { it: "renders a bucket containing a file", @@ -177,6 +180,151 @@ func TestHandleBucketView(t *testing.T) { expectedStatusCode: http.StatusOK, expectedBodyContains: "def", }, + { + it: "does not show version columns when ShowVersions is disabled", + listObjectsFunc: func(context.Context, string, minio.ListObjectsOptions) <-chan minio.ObjectInfo { + objCh := make(chan minio.ObjectInfo) + go func() { + objCh <- minio.ObjectInfo{Key: "FILE-NAME", VersionID: "v1-abcdefghijk", IsLatest: true} + close(objCh) + }() + return objCh + }, + bucketName: "BUCKET-NAME", + showVersions: false, + expectedStatusCode: http.StatusOK, + expectedBodyContains: "FILE-NAME", + unexpectedInBody: []string{"Version ID", "v1-abcdef"}, + }, + { + it: "renders multiple versions when ShowVersions is enabled", + listObjectsFunc: func(_ context.Context, _ string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo { + objCh := make(chan minio.ObjectInfo) + go func() { + if opts.WithVersions { + objCh <- minio.ObjectInfo{Key: "FILE-NAME", VersionID: "v2-abcdefghijk", IsLatest: true} + objCh <- minio.ObjectInfo{Key: "FILE-NAME", VersionID: "v1-abcdefghijk", IsLatest: false} + } + close(objCh) + }() + return objCh + }, + bucketName: "BUCKET-NAME", + showVersions: true, + expectedStatusCode: http.StatusOK, + expectedBodyContains: "Latest", + }, + { + it: "falls back to a normal listing when the versioned listing fails", + listObjectsFunc: func(_ context.Context, _ string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo { + objCh := make(chan minio.ObjectInfo) + go func() { + defer close(objCh) + if opts.WithVersions { + objCh <- minio.ObjectInfo{Err: errS3} + return + } + objCh <- minio.ObjectInfo{Key: "FILE-NAME"} + }() + return objCh + }, + bucketName: "BUCKET-NAME", + showVersions: true, + expectedStatusCode: http.StatusOK, + expectedBodyContains: "FILE-NAME", + unexpectedInBody: []string{"Version ID"}, + }, + { + it: "falls back to a normal listing when the versioned listing succeeds but returns nothing", + listObjectsFunc: func(_ context.Context, _ string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo { + objCh := make(chan minio.ObjectInfo) + go func() { + defer close(objCh) + if opts.WithVersions { + return + } + objCh <- minio.ObjectInfo{Key: "FILE-NAME"} + }() + return objCh + }, + bucketName: "BUCKET-NAME", + showVersions: true, + expectedStatusCode: http.StatusOK, + expectedBodyContains: "FILE-NAME", + unexpectedInBody: []string{"Version ID"}, + }, + { + it: "collapses older versions by default with a toggle to expand them", + listObjectsFunc: func(_ context.Context, _ string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo { + objCh := make(chan minio.ObjectInfo) + go func() { + if opts.WithVersions { + objCh <- minio.ObjectInfo{Key: "FILE-NAME", VersionID: "v2-abcdefghijk", IsLatest: true} + objCh <- minio.ObjectInfo{Key: "FILE-NAME", VersionID: "v1-abcdefghijk", IsLatest: false} + } + close(objCh) + }() + return objCh + }, + bucketName: "BUCKET-NAME", + showVersions: true, + expectedStatusCode: http.StatusOK, + expectedBodyContains: `class="version-row" style="display: none;`, + }, + { + it: "does not hide folders or objects when the provider never sets IsLatest on versioned entries", + listObjectsFunc: func(_ context.Context, _ string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo { + objCh := make(chan minio.ObjectInfo) + go func() { + defer close(objCh) + if !opts.WithVersions { + return + } + // Folders synthesized from CommonPrefixes never carry + // version metadata, and some providers don't reliably + // set IsLatest on real objects either. + objCh <- minio.ObjectInfo{Key: "AFolder/"} + objCh <- minio.ObjectInfo{Key: "FILE-NAME", VersionID: "v1-abcdefghijk"} + }() + return objCh + }, + bucketName: "BUCKET-NAME", + showVersions: true, + expectedStatusCode: http.StatusOK, + expectedBodyContains: "AFolder", + unexpectedInBody: []string{`class="version-row" style="display: none;`}, + }, + { + it: "shows the metadata action when ShowMetadata is enabled", + listObjectsFunc: func(context.Context, string, minio.ListObjectsOptions) <-chan minio.ObjectInfo { + objCh := make(chan minio.ObjectInfo) + go func() { + objCh <- minio.ObjectInfo{Key: "FILE-NAME"} + close(objCh) + }() + return objCh + }, + bucketName: "BUCKET-NAME", + showMetadata: true, + expectedStatusCode: http.StatusOK, + expectedBodyContains: `onclick="handleOpenMetadataModal(`, + }, + { + it: "hides the metadata action when ShowMetadata is disabled", + listObjectsFunc: func(context.Context, string, minio.ListObjectsOptions) <-chan minio.ObjectInfo { + objCh := make(chan minio.ObjectInfo) + go func() { + objCh <- minio.ObjectInfo{Key: "FILE-NAME"} + close(objCh) + }() + return objCh + }, + bucketName: "BUCKET-NAME", + showMetadata: false, + expectedStatusCode: http.StatusOK, + expectedBodyContains: "FILE-NAME", + unexpectedInBody: []string{`onclick="handleOpenMetadataModal(`}, + }, } for _, tc := range cases { @@ -194,7 +342,7 @@ func TestHandleBucketView(t *testing.T) { templates := os.DirFS(filepath.Join("..", "..", "..", "web", "template")) r := mux.NewRouter() - r.PathPrefix("/buckets/").Handler(s3manager.HandleBucketView(s3, templates, true, true, tc.rootUrl)).Methods(http.MethodGet) + r.PathPrefix("/buckets/").Handler(s3manager.HandleBucketView(s3, templates, true, true, tc.rootUrl, tc.showVersions, tc.showMetadata)).Methods(http.MethodGet) ts := httptest.NewServer(r) defer ts.Close() @@ -210,6 +358,9 @@ func TestHandleBucketView(t *testing.T) { is.Equal(tc.expectedStatusCode, resp.StatusCode) // status code is.True(strings.Contains(string(body), tc.expectedBodyContains)) // body + for _, unexpected := range tc.unexpectedInBody { + is.True(!strings.Contains(string(body), unexpected)) + } // fmt.Println(string(body)) if tc.expectedStatusCode == http.StatusOK { diff --git a/internal/app/s3manager/get_object.go b/internal/app/s3manager/get_object.go index c0f1630..9921304 100644 --- a/internal/app/s3manager/get_object.go +++ b/internal/app/s3manager/get_object.go @@ -10,12 +10,18 @@ import ( ) // HandleGetObject downloads an object to the client. -func HandleGetObject(s3 S3, forceDownload bool) http.HandlerFunc { +func HandleGetObject(s3 S3, forceDownload, showVersions bool) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { bucketName := mux.Vars(r)["bucketName"] objectName := mux.Vars(r)["objectName"] + // Ignore versionId unless the versions feature is enabled, so + // disabling SHOW_VERSIONS also prevents access to old versions. + versionID := "" + if showVersions { + versionID = r.URL.Query().Get("versionId") + } - object, err := s3.GetObject(r.Context(), bucketName, objectName, minio.GetObjectOptions{}) + object, err := s3.GetObject(r.Context(), bucketName, objectName, minio.GetObjectOptions{VersionID: versionID}) if err != nil { handleHTTPError(w, fmt.Errorf("error getting object: %w", err)) return diff --git a/internal/app/s3manager/get_object_metadata.go b/internal/app/s3manager/get_object_metadata.go new file mode 100644 index 0000000..e3f59ed --- /dev/null +++ b/internal/app/s3manager/get_object_metadata.go @@ -0,0 +1,72 @@ +package s3manager + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "time" + + "github.com/gorilla/mux" + "github.com/minio/minio-go/v7" +) + +// objectMetadata is the JSON shape returned by HandleGetObjectMetadata. +type objectMetadata struct { + Key string `json:"key"` + VersionID string `json:"versionId,omitempty"` + Size int64 `json:"size"` + ContentType string `json:"contentType"` + ETag string `json:"etag"` + LastModified string `json:"lastModified"` + StorageClass string `json:"storageClass,omitempty"` + IsLatest bool `json:"isLatest,omitempty"` + UserMetadata map[string]string `json:"userMetadata"` +} + +// HandleGetObjectMetadata returns metadata for an object (optionally a specific version). +func HandleGetObjectMetadata(s3 S3) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + bucketName := mux.Vars(r)["bucketName"] + objectName := mux.Vars(r)["objectName"] + versionID := r.URL.Query().Get("versionId") + + info, err := s3.StatObject(r.Context(), bucketName, objectName, minio.StatObjectOptions{VersionID: versionID}) + if err != nil { + handleHTTPError(w, fmt.Errorf("error getting object metadata: %w", err)) + return + } + + userMetadata := info.UserMetadata + if len(userMetadata) == 0 { + // AWS S3 doesn't populate UserMetadata; derive it from the raw + // headers by stripping the x-amz-meta- prefix. + userMetadata = make(map[string]string) + for key, values := range info.Metadata { + lowerKey := strings.ToLower(key) + if !strings.HasPrefix(lowerKey, "x-amz-meta-") || len(values) == 0 { + continue + } + userMetadata[strings.TrimPrefix(lowerKey, "x-amz-meta-")] = values[0] + } + } + + response := objectMetadata{ + Key: info.Key, + VersionID: info.VersionID, + Size: info.Size, + ContentType: info.ContentType, + ETag: info.ETag, + LastModified: info.LastModified.Format(time.RFC3339), + StorageClass: info.StorageClass, + IsLatest: info.IsLatest, + UserMetadata: userMetadata, + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(response); err != nil { + handleHTTPError(w, fmt.Errorf("error encoding JSON: %w", err)) + return + } + } +} diff --git a/internal/app/s3manager/get_object_metadata_test.go b/internal/app/s3manager/get_object_metadata_test.go new file mode 100644 index 0000000..08679c6 --- /dev/null +++ b/internal/app/s3manager/get_object_metadata_test.go @@ -0,0 +1,143 @@ +package s3manager_test + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/cloudlena/s3manager/internal/app/s3manager" + "github.com/cloudlena/s3manager/internal/app/s3manager/mocks" + "github.com/gorilla/mux" + "github.com/matryer/is" + "github.com/minio/minio-go/v7" +) + +func TestHandleGetObjectMetadata(t *testing.T) { + t.Parallel() + + lastModified := time.Date(2026, 1, 2, 15, 4, 5, 0, time.UTC) + + cases := []struct { + it string + statObjectFunc func(context.Context, string, string, minio.StatObjectOptions) (minio.ObjectInfo, error) + queryString string + expectedStatusCode int + expectedBody map[string]any + expectedBodyError string + }{ + { + it: "returns metadata for the latest version", + statObjectFunc: func(_ context.Context, _, _ string, opts minio.StatObjectOptions) (minio.ObjectInfo, error) { + is := is.New(t) + is.Equal("", opts.VersionID) + return minio.ObjectInfo{ + Key: "OBJECT-NAME", + Size: 1234, + ContentType: "text/plain", + ETag: "abc123", + LastModified: lastModified, + StorageClass: "STANDARD", + UserMetadata: map[string]string{"foo": "bar"}, + }, nil + }, + expectedStatusCode: http.StatusOK, + expectedBody: map[string]any{ + "key": "OBJECT-NAME", + "size": float64(1234), + "contentType": "text/plain", + "etag": "abc123", + "storageClass": "STANDARD", + "userMetadata": map[string]any{"foo": "bar"}, + }, + }, + { + it: "passes the versionId query param through to StatObjectOptions", + statObjectFunc: func(_ context.Context, _, _ string, opts minio.StatObjectOptions) (minio.ObjectInfo, error) { + is := is.New(t) + is.Equal("VERSION-123", opts.VersionID) + return minio.ObjectInfo{ + Key: "OBJECT-NAME", + VersionID: "VERSION-123", + IsLatest: true, + LastModified: lastModified, + }, nil + }, + queryString: "?versionId=VERSION-123", + expectedStatusCode: http.StatusOK, + expectedBody: map[string]any{ + "key": "OBJECT-NAME", + "versionId": "VERSION-123", + "isLatest": true, + }, + }, + { + it: "derives user metadata from raw headers when UserMetadata is empty", + statObjectFunc: func(_ context.Context, _, _ string, _ minio.StatObjectOptions) (minio.ObjectInfo, error) { + return minio.ObjectInfo{ + Key: "OBJECT-NAME", + LastModified: lastModified, + Metadata: map[string][]string{ + "X-Amz-Meta-Foo": {"bar"}, + "Content-Type": {"text/plain"}, + }, + }, nil + }, + expectedStatusCode: http.StatusOK, + expectedBody: map[string]any{ + "userMetadata": map[string]any{"foo": "bar"}, + }, + }, + { + it: "returns error if there is an S3 error", + statObjectFunc: func(context.Context, string, string, minio.StatObjectOptions) (minio.ObjectInfo, error) { + return minio.ObjectInfo{}, errS3 + }, + expectedStatusCode: http.StatusInternalServerError, + expectedBodyError: "mocked s3 error", + }, + } + + for _, tc := range cases { + t.Run(tc.it, func(t *testing.T) { + t.Parallel() + is := is.New(t) + + s3 := &mocks.S3Mock{ + StatObjectFunc: tc.statObjectFunc, + } + + r := mux.NewRouter() + r.Handle("/api/buckets/{bucketName}/objects/{objectName}/metadata", s3manager.HandleGetObjectMetadata(s3)).Methods(http.MethodGet) + + ts := httptest.NewServer(r) + defer ts.Close() + + resp, err := http.Get(ts.URL + "/api/buckets/BUCKET-NAME/objects/OBJECT-NAME/metadata" + tc.queryString) + is.NoErr(err) + defer func() { + err = resp.Body.Close() + is.NoErr(err) + }() + + is.Equal(tc.expectedStatusCode, resp.StatusCode) + + if tc.expectedBodyError != "" { + rawBody, err := io.ReadAll(resp.Body) + is.NoErr(err) + is.True(strings.Contains(string(rawBody), tc.expectedBodyError)) + return + } + + var body map[string]any + is.NoErr(json.NewDecoder(resp.Body).Decode(&body)) + for key, expected := range tc.expectedBody { + is.Equal(expected, body[key]) + } + }) + } +} diff --git a/internal/app/s3manager/get_object_test.go b/internal/app/s3manager/get_object_test.go index 2fc2280..9bf027c 100644 --- a/internal/app/s3manager/get_object_test.go +++ b/internal/app/s3manager/get_object_test.go @@ -24,6 +24,8 @@ func TestHandleGetObject(t *testing.T) { getObjectFunc func(context.Context, string, string, minio.GetObjectOptions) (*minio.Object, error) bucketName string objectName string + queryString string + showVersions bool expectedStatusCode int expectedBodyContains string }{ @@ -37,6 +39,49 @@ func TestHandleGetObject(t *testing.T) { expectedStatusCode: http.StatusInternalServerError, expectedBodyContains: "mocked s3 error", }, + { + it: "leaves VersionID empty when no versionId query param is given", + getObjectFunc: func(_ context.Context, _, _ string, opts minio.GetObjectOptions) (*minio.Object, error) { + if opts.VersionID != "" { + return nil, fmt.Errorf("expected empty VersionID, got %q", opts.VersionID) + } + return nil, errS3 + }, + bucketName: "BUCKET-NAME", + objectName: "OBJECT-NAME", + expectedStatusCode: http.StatusInternalServerError, + expectedBodyContains: "mocked s3 error", + }, + { + it: "passes the versionId query param through to GetObjectOptions", + getObjectFunc: func(_ context.Context, _, _ string, opts minio.GetObjectOptions) (*minio.Object, error) { + if opts.VersionID != "VERSION-123" { + return nil, fmt.Errorf("expected VersionID %q, got %q", "VERSION-123", opts.VersionID) + } + return nil, errS3 + }, + bucketName: "BUCKET-NAME", + objectName: "OBJECT-NAME", + queryString: "?versionId=VERSION-123", + showVersions: true, + expectedStatusCode: http.StatusInternalServerError, + expectedBodyContains: "mocked s3 error", + }, + { + it: "ignores the versionId query param when showVersions is disabled", + getObjectFunc: func(_ context.Context, _, _ string, opts minio.GetObjectOptions) (*minio.Object, error) { + if opts.VersionID != "" { + return nil, fmt.Errorf("expected empty VersionID, got %q", opts.VersionID) + } + return nil, errS3 + }, + bucketName: "BUCKET-NAME", + objectName: "OBJECT-NAME", + queryString: "?versionId=VERSION-123", + showVersions: false, + expectedStatusCode: http.StatusInternalServerError, + expectedBodyContains: "mocked s3 error", + }, } for _, tc := range cases { @@ -49,12 +94,12 @@ func TestHandleGetObject(t *testing.T) { } r := mux.NewRouter() - r.Handle("/buckets/{bucketName}/objects/{objectName}", s3manager.HandleGetObject(s3, true)).Methods(http.MethodGet) + r.Handle("/buckets/{bucketName}/objects/{objectName}", s3manager.HandleGetObject(s3, true, tc.showVersions)).Methods(http.MethodGet) ts := httptest.NewServer(r) defer ts.Close() - resp, err := http.Get(fmt.Sprintf("%s/buckets/%s/objects/%s", ts.URL, tc.bucketName, tc.objectName)) + resp, err := http.Get(fmt.Sprintf("%s/buckets/%s/objects/%s%s", ts.URL, tc.bucketName, tc.objectName, tc.queryString)) is.NoErr(err) defer func() { err = resp.Body.Close() diff --git a/internal/app/s3manager/manager_handlers.go b/internal/app/s3manager/manager_handlers.go index 9337328..0b2c069 100644 --- a/internal/app/s3manager/manager_handlers.go +++ b/internal/app/s3manager/manager_handlers.go @@ -10,7 +10,6 @@ import ( "strings" "github.com/gorilla/mux" - "github.com/minio/minio-go/v7" ) // withInstance extracts the instance from the request, looks it up in the manager, @@ -98,7 +97,7 @@ func HandleBucketsViewWithManager(manager *MultiS3Manager, templates fs.FS, allo } // HandleBucketViewWithManager shows the details page of a bucket using MultiS3Manager. -func HandleBucketViewWithManager(manager *MultiS3Manager, templates fs.FS, allowDelete bool, listRecursive bool, rootURL string) http.HandlerFunc { +func HandleBucketViewWithManager(manager *MultiS3Manager, templates fs.FS, allowDelete bool, listRecursive bool, rootURL string, showVersions bool, showMetadata bool) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) instanceName := vars["instance"] @@ -113,7 +112,7 @@ func HandleBucketViewWithManager(manager *MultiS3Manager, templates fs.FS, allow instances := manager.GetAllInstances() // Create a modified handler that includes S3 instance data - handler := createBucketViewWithS3Data(s3, templates, allowDelete, listRecursive, rootURL, current, instances) + handler := createBucketViewWithS3Data(s3, templates, allowDelete, listRecursive, rootURL, current, instances, showVersions, showMetadata) handler(w, r) } } @@ -139,8 +138,8 @@ func HandleGenerateURLWithManager(manager *MultiS3Manager) http.HandlerFunc { } // HandleGetObjectWithManager downloads an object to the client using MultiS3Manager. -func HandleGetObjectWithManager(manager *MultiS3Manager, forceDownload bool) http.HandlerFunc { - return withInstance(manager, func(s3 S3) http.HandlerFunc { return HandleGetObject(s3, forceDownload) }) +func HandleGetObjectWithManager(manager *MultiS3Manager, forceDownload, showVersions bool) http.HandlerFunc { + return withInstance(manager, func(s3 S3) http.HandlerFunc { return HandleGetObject(s3, forceDownload, showVersions) }) } // HandleDeleteObjectWithManager deletes an object using MultiS3Manager. @@ -153,29 +152,37 @@ func HandleCheckPublicAccessWithManager(manager *MultiS3Manager) http.HandlerFun return withInstance(manager, HandleCheckPublicAccess) } +// HandleGetObjectMetadataWithManager retrieves object metadata using MultiS3Manager. +func HandleGetObjectMetadataWithManager(manager *MultiS3Manager) http.HandlerFunc { + return withInstance(manager, HandleGetObjectMetadata) +} + // createBucketViewWithS3Data creates a bucket view handler that includes S3 instance data -func createBucketViewWithS3Data(s3 S3, templates fs.FS, allowDelete bool, listRecursive bool, rootURL string, current *S3Instance, instances []*S3Instance) http.HandlerFunc { +func createBucketViewWithS3Data(s3 S3, templates fs.FS, allowDelete bool, listRecursive bool, rootURL string, current *S3Instance, instances []*S3Instance, showVersions bool, showMetadata bool) http.HandlerFunc { type pageData struct { - RootURL string - BucketName string - Objects []objectWithIcon - AllowDelete bool - Paths []string - CurrentPath string - Endpoint string - CurrentS3 *S3Instance - S3Instances []*S3Instance - HasError bool - ErrorMessage string - SortBy string - SortOrder string - Page int - PerPage int - TotalItems int - TotalPages int - HasPrevPage bool - HasNextPage bool - Search string + RootURL string + BucketName string + Objects []objectWithIcon + AllowDelete bool + Paths []string + CurrentPath string + Endpoint string + CurrentS3 *S3Instance + S3Instances []*S3Instance + HasError bool + ErrorMessage string + SortBy string + SortOrder string + Page int + PerPage int + TotalItems int + TotalPages int + HasPrevPage bool + HasNextPage bool + Search string + ShowVersions bool + VersionsUnavailable bool + ShowMetadata bool } return func(w http.ResponseWriter, r *http.Request) { @@ -227,43 +234,21 @@ func createBucketViewWithS3Data(s3 S3, templates fs.FS, allowDelete bool, listRe // Get search parameter search := strings.TrimSpace(r.URL.Query().Get("search")) - var objs []objectWithIcon hasError := false errorMessage := "" - opts := minio.ListObjectsOptions{ - Recursive: listRecursive, - Prefix: path, + objs, versionsShown, listErr := listObjectsForBucketView(r.Context(), s3, bucketName, path, listRecursive, showVersions) + if listErr != nil { + // Instead of returning HTTP error, show user-friendly message + hasError = true + errorMessage = friendlyListObjectsErrorMessage(listErr, bucketName, current.Name) + } else if versionsShown { + annotateVersionGroups(objs) } - objectCh := s3.ListObjects(r.Context(), bucketName, opts) - for object := range objectCh { - if object.Err != nil { - // Instead of returning HTTP error, show user-friendly message - hasError = true - if strings.Contains(object.Err.Error(), "AccessDenied") || strings.Contains(object.Err.Error(), "InvalidAccessKeyId") || strings.Contains(object.Err.Error(), "SignatureDoesNotMatch") { - errorMessage = fmt.Sprintf("Unable to access bucket '%s' on S3 instance '%s'. Please check the credentials and try switching to another instance.", bucketName, current.Name) - } else if strings.Contains(object.Err.Error(), ErrBucketDoesNotExist) { - errorMessage = fmt.Sprintf("Bucket '%s' does not exist on S3 instance '%s'. Please try switching to another instance or go back to the buckets list.", bucketName, current.Name) - } else { - errorMessage = fmt.Sprintf("Unable to list objects in bucket '%s' on S3 instance '%s'. Please try switching to another instance.", bucketName, current.Name) - } - break - } - sizeDisplay := FormatFileSize(object.Size) - - obj := objectWithIcon{ - Key: object.Key, - Size: object.Size, - SizeDisplay: sizeDisplay, - LastModified: object.LastModified, - Owner: object.Owner.DisplayName, - Icon: icon(object.Key), - IsFolder: strings.HasSuffix(object.Key, "/"), - DisplayName: strings.TrimSuffix(strings.TrimPrefix(object.Key, path), "/"), - } - objs = append(objs, obj) - } + // Only warn about unavailable versions when there is content to show; + // an empty bucket legitimately produces an empty versioned listing. + versionsUnavailable := showVersions && !versionsShown && !hasError && len(objs) > 0 // Filter objects based on search query if search != "" && !hasError { @@ -279,25 +264,14 @@ func createBucketViewWithS3Data(s3 S3, templates fs.FS, allowDelete bool, listRe objs = filteredObjs } - // Sort objects based on sortBy and sortOrder - if !hasError { - sortObjects(objs, sortBy, sortOrder) - } - - // Calculate pagination - totalItems := len(objs) + // Sort and paginate; versions of the same object stay together + totalItems := 0 totalPages := 1 - if !showAll { - totalPages = (totalItems + perPage - 1) / perPage - if totalPages == 0 { - totalPages = 1 - } - if page > totalPages { - page = totalPages - } + if !hasError { + objs, totalItems, totalPages, page = sortAndPaginateObjects(objs, sortBy, sortOrder, page, perPage, showAll, versionsShown) + } else { + page = 1 } - - // Paginate objects if showAll { // Show all items - no pagination perPage = totalItems @@ -305,44 +279,32 @@ func createBucketViewWithS3Data(s3 S3, templates fs.FS, allowDelete bool, listRe perPage = 1 // Avoid division by zero } page = 1 - } else { - // Apply pagination - start := (page - 1) * perPage - end := start + perPage - if start < 0 { - start = 0 - } - if end > totalItems { - end = totalItems - } - if start < totalItems && !hasError { - objs = objs[start:end] - } else if !hasError { - objs = []objectWithIcon{} - } } data := pageData{ - RootURL: rootURL, - BucketName: bucketName, - Objects: objs, - AllowDelete: allowDelete, - Paths: removeEmptyStrings(strings.Split(path, "/")), - CurrentPath: path, - Endpoint: s3.EndpointURL().String(), - CurrentS3: current, - S3Instances: instances, - HasError: hasError, - ErrorMessage: errorMessage, - SortBy: sortBy, - SortOrder: sortOrder, - Page: page, - PerPage: perPage, - TotalItems: totalItems, - TotalPages: totalPages, - HasPrevPage: page > 1, - HasNextPage: page < totalPages, - Search: search, + RootURL: rootURL, + BucketName: bucketName, + Objects: objs, + AllowDelete: allowDelete, + Paths: removeEmptyStrings(strings.Split(path, "/")), + CurrentPath: path, + Endpoint: s3.EndpointURL().String(), + CurrentS3: current, + S3Instances: instances, + HasError: hasError, + ErrorMessage: errorMessage, + SortBy: sortBy, + SortOrder: sortOrder, + Page: page, + PerPage: perPage, + TotalItems: totalItems, + TotalPages: totalPages, + HasPrevPage: page > 1, + HasNextPage: page < totalPages, + Search: search, + ShowVersions: versionsShown, + VersionsUnavailable: versionsUnavailable, + ShowMetadata: showMetadata, } funcMap := template.FuncMap{ diff --git a/internal/app/s3manager/manager_handlers_test.go b/internal/app/s3manager/manager_handlers_test.go index c2c4e6b..2690964 100644 --- a/internal/app/s3manager/manager_handlers_test.go +++ b/internal/app/s3manager/manager_handlers_test.go @@ -46,6 +46,7 @@ type stubS3 struct { listObjects func(context.Context, string, minio.ListObjectsOptions) <-chan minio.ObjectInfo endpointURL func() *url.URL presignedGetObject func(context.Context, string, string, time.Duration, url.Values) (*url.URL, error) + statObject func(context.Context, string, string, minio.StatObjectOptions) (minio.ObjectInfo, error) } func (s *stubS3) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) { @@ -87,6 +88,12 @@ func (s *stubS3) PresignedGetObject(ctx context.Context, bucket, object string, func (s *stubS3) PutObject(_ context.Context, _, _ string, _ io.Reader, _ int64, _ minio.PutObjectOptions) (minio.UploadInfo, error) { panic("PutObject not expected in this test") } +func (s *stubS3) StatObject(ctx context.Context, bucket, object string, opts minio.StatObjectOptions) (minio.ObjectInfo, error) { + if s.statObject != nil { + return s.statObject(ctx, bucket, object, opts) + } + panic("StatObject not expected in this test") +} var errManagerTest = errors.New("manager test error") @@ -706,6 +713,72 @@ func TestHandleCheckPublicAccessWithManager(t *testing.T) { } } +func TestHandleGetObjectMetadataWithManager(t *testing.T) { + t.Parallel() + + cases := []struct { + it string + instanceName string + expectedStatusCode int + expectedBodyContains string + }{ + { + it: "returns 404 for unknown instance", + instanceName: "unknown", + expectedStatusCode: http.StatusNotFound, + expectedBodyContains: "Instance not found", + }, + { + it: "returns metadata through the resolved instance", + instanceName: "primary", + expectedStatusCode: http.StatusOK, + expectedBodyContains: `"contentType":"text/plain"`, + }, + } + + for _, tc := range cases { + t.Run(tc.it, func(t *testing.T) { + t.Parallel() + is := is.New(t) + + s3mock := &stubS3{ + endpointURL: func() *url.URL { + u, _ := url.Parse("http://localhost:9000") + return u + }, + statObject: func(_ context.Context, _, _ string, _ minio.StatObjectOptions) (minio.ObjectInfo, error) { + return minio.ObjectInfo{ + Key: "test-object", + Size: 42, + ContentType: "text/plain", + }, nil + }, + } + manager := newTestMultiS3Manager([]*S3Instance{ + {ID: "1", Name: "primary", Client: s3mock}, + }) + + r := mux.NewRouter() + r.Handle("/{instance}/api/buckets/{bucketName}/objects/{objectName:.*}/metadata", HandleGetObjectMetadataWithManager(manager)) + + ts := httptest.NewServer(r) + defer ts.Close() + + resp, err := http.Get(ts.URL + "/" + tc.instanceName + "/api/buckets/test-bucket/objects/test-object/metadata") + is.NoErr(err) + defer func() { + err = resp.Body.Close() + is.NoErr(err) + }() + body, err := io.ReadAll(resp.Body) + is.NoErr(err) + + is.Equal(tc.expectedStatusCode, resp.StatusCode) + is.True(strings.Contains(string(body), tc.expectedBodyContains)) + }) + } +} + func TestHandleGetObjectWithManager(t *testing.T) { t.Parallel() is := is.New(t) @@ -715,7 +788,7 @@ func TestHandleGetObjectWithManager(t *testing.T) { }) r := mux.NewRouter() - r.Handle("/{instance}/api/buckets/{bucketName}/objects/{objectName}", HandleGetObjectWithManager(manager, true)).Methods(http.MethodGet) + r.Handle("/{instance}/api/buckets/{bucketName}/objects/{objectName}", HandleGetObjectWithManager(manager, true, false)).Methods(http.MethodGet) ts := httptest.NewServer(r) defer ts.Close() @@ -765,29 +838,242 @@ func TestHandleCreateObjectWithManager(t *testing.T) { func TestHandleBucketViewWithManager(t *testing.T) { t.Parallel() - is := is.New(t) - templates := os.DirFS(filepath.Join("..", "..", "..", "web", "template")) + versionedListObjects := func(_ context.Context, _ string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo { + ch := make(chan minio.ObjectInfo) + go func() { + defer close(ch) + if !opts.WithVersions { + return + } + ch <- minio.ObjectInfo{Key: "FILE-NAME", VersionID: "v2-abcdefghijk", IsLatest: true} + ch <- minio.ObjectInfo{Key: "FILE-NAME", VersionID: "v1-abcdefghijk", IsLatest: false} + }() + return ch + } - manager := newTestMultiS3Manager([]*S3Instance{ - {ID: "1", Name: "primary", Client: &stubS3{}}, - }) + cases := []struct { + it string + path string + client S3 + showVersions bool + showMetadata bool + expectedStatusCode int + expectedBodyContains []string + unexpectedInBody []string + }{ + { + it: "returns not found for an unknown instance", + path: "/unknown/buckets/test-bucket/", + client: &stubS3{}, + expectedStatusCode: http.StatusNotFound, + expectedBodyContains: []string{ + "Instance not found", + }, + }, + { + it: "does not show version columns when ShowVersions is disabled", + path: "/primary/buckets/test-bucket/", + client: &stubS3{ + listObjects: versionedListObjects, + endpointURL: func() *url.URL { u, _ := url.Parse("http://localhost:9000"); return u }, + }, + showVersions: false, + expectedStatusCode: http.StatusOK, + unexpectedInBody: []string{ + "Version ID", + "v1-abcdef", + "v2-abcdef", + }, + }, + { + it: "renders multiple versions when ShowVersions is enabled", + path: "/primary/buckets/test-bucket/", + client: &stubS3{ + listObjects: versionedListObjects, + endpointURL: func() *url.URL { u, _ := url.Parse("http://localhost:9000"); return u }, + }, + showVersions: true, + expectedStatusCode: http.StatusOK, + expectedBodyContains: []string{ + "Version ID", + "v1-abcdef", + "v2-abcdef", + "Latest", + }, + }, + { + it: "falls back to a normal listing and shows a notice when the provider rejects listing versions", + path: "/primary/buckets/test-bucket/", + client: &stubS3{ + listObjects: func(_ context.Context, _ string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo { + ch := make(chan minio.ObjectInfo) + go func() { + defer close(ch) + if opts.WithVersions { + ch <- minio.ObjectInfo{Err: errManagerTest} + return + } + ch <- minio.ObjectInfo{Key: "FILE-NAME"} + }() + return ch + }, + endpointURL: func() *url.URL { u, _ := url.Parse("http://localhost:9000"); return u }, + }, + showVersions: true, + expectedStatusCode: http.StatusOK, + expectedBodyContains: []string{ + "FILE-NAME", + "Object versions unavailable", + }, + unexpectedInBody: []string{ + "Version ID", + }, + }, + { + it: "surfaces the underlying S3 error when listing fails for a reason other than versioning", + path: "/primary/buckets/test-bucket/", + client: &stubS3{ + listObjects: func(_ context.Context, _ string, _ minio.ListObjectsOptions) <-chan minio.ObjectInfo { + ch := make(chan minio.ObjectInfo) + go func() { + defer close(ch) + ch <- minio.ObjectInfo{Err: errManagerTest} + }() + return ch + }, + endpointURL: func() *url.URL { u, _ := url.Parse("http://localhost:9000"); return u }, + }, + showVersions: false, + expectedStatusCode: http.StatusOK, + expectedBodyContains: []string{ + errManagerTest.Error(), + }, + }, + { + it: "falls back to a normal listing when the versioned listing succeeds but returns nothing", + path: "/primary/buckets/test-bucket/", + client: &stubS3{ + listObjects: func(_ context.Context, _ string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo { + ch := make(chan minio.ObjectInfo) + go func() { + defer close(ch) + if opts.WithVersions { + // Some S3-compatible providers silently return an + // empty result instead of erroring when versioned + // listing isn't supported. + return + } + ch <- minio.ObjectInfo{Key: "FILE-NAME"} + }() + return ch + }, + endpointURL: func() *url.URL { u, _ := url.Parse("http://localhost:9000"); return u }, + }, + showVersions: true, + expectedStatusCode: http.StatusOK, + expectedBodyContains: []string{ + "FILE-NAME", + }, + unexpectedInBody: []string{ + "Version ID", + }, + }, + { + it: "does not warn about unavailable versions for an empty bucket", + path: "/primary/buckets/test-bucket/", + client: &stubS3{ + listObjects: func(_ context.Context, _ string, _ minio.ListObjectsOptions) <-chan minio.ObjectInfo { + ch := make(chan minio.ObjectInfo) + close(ch) + return ch + }, + endpointURL: func() *url.URL { u, _ := url.Parse("http://localhost:9000"); return u }, + }, + showVersions: true, + expectedStatusCode: http.StatusOK, + unexpectedInBody: []string{ + "Object versions unavailable", + }, + }, + { + it: "shows the metadata action when ShowMetadata is enabled", + path: "/primary/buckets/test-bucket/", + client: &stubS3{ + listObjects: func(_ context.Context, _ string, _ minio.ListObjectsOptions) <-chan minio.ObjectInfo { + ch := make(chan minio.ObjectInfo) + go func() { + defer close(ch) + ch <- minio.ObjectInfo{Key: "FILE-NAME"} + }() + return ch + }, + endpointURL: func() *url.URL { u, _ := url.Parse("http://localhost:9000"); return u }, + }, + showMetadata: true, + expectedStatusCode: http.StatusOK, + expectedBodyContains: []string{ + `onclick="handleOpenMetadataModal(`, + }, + }, + { + it: "hides the metadata action when ShowMetadata is disabled", + path: "/primary/buckets/test-bucket/", + client: &stubS3{ + listObjects: func(_ context.Context, _ string, _ minio.ListObjectsOptions) <-chan minio.ObjectInfo { + ch := make(chan minio.ObjectInfo) + go func() { + defer close(ch) + ch <- minio.ObjectInfo{Key: "FILE-NAME"} + }() + return ch + }, + endpointURL: func() *url.URL { u, _ := url.Parse("http://localhost:9000"); return u }, + }, + showMetadata: false, + expectedStatusCode: http.StatusOK, + expectedBodyContains: []string{ + "FILE-NAME", + }, + unexpectedInBody: []string{ + `onclick="handleOpenMetadataModal(`, + }, + }, + } - r := mux.NewRouter() - r.PathPrefix("/{instance}/buckets/").Handler(HandleBucketViewWithManager(manager, templates, true, true, "")).Methods(http.MethodGet) + for _, tc := range cases { + t.Run(tc.it, func(t *testing.T) { + t.Parallel() + is := is.New(t) - ts := httptest.NewServer(r) - defer ts.Close() + templates := os.DirFS(filepath.Join("..", "..", "..", "web", "template")) - resp, err := http.Get(ts.URL + "/unknown/buckets/test-bucket/") - is.NoErr(err) - defer func() { - err = resp.Body.Close() - is.NoErr(err) - }() - body, err := io.ReadAll(resp.Body) - is.NoErr(err) + manager := newTestMultiS3Manager([]*S3Instance{ + {ID: "1", Name: "primary", Client: tc.client}, + }) - is.Equal(http.StatusNotFound, resp.StatusCode) - is.True(strings.Contains(string(body), "Instance not found")) + r := mux.NewRouter() + r.PathPrefix("/{instance}/buckets/").Handler(HandleBucketViewWithManager(manager, templates, true, true, "", tc.showVersions, tc.showMetadata)).Methods(http.MethodGet) + + ts := httptest.NewServer(r) + defer ts.Close() + + resp, err := http.Get(ts.URL + tc.path) + is.NoErr(err) + defer func() { + err = resp.Body.Close() + is.NoErr(err) + }() + body, err := io.ReadAll(resp.Body) + is.NoErr(err) + + is.Equal(tc.expectedStatusCode, resp.StatusCode) + for _, expected := range tc.expectedBodyContains { + is.True(strings.Contains(string(body), expected)) + } + for _, unexpected := range tc.unexpectedInBody { + is.True(!strings.Contains(string(body), unexpected)) + } + }) + } } diff --git a/internal/app/s3manager/mocks/s3.go b/internal/app/s3manager/mocks/s3.go index 4d3abe8..0123d8f 100644 --- a/internal/app/s3manager/mocks/s3.go +++ b/internal/app/s3manager/mocks/s3.go @@ -59,6 +59,9 @@ var _ s3manager.S3 = &S3Mock{} // SetBucketPolicyFunc: func(ctx context.Context, bucketName string, policy string) error { // panic("mock out the SetBucketPolicy method") // }, +// StatObjectFunc: func(ctx context.Context, bucketName string, objectName string, opts minio.StatObjectOptions) (minio.ObjectInfo, error) { +// panic("mock out the StatObject method") +// }, // } // // // use mockedS3 in code that requires s3manager.S3 @@ -102,6 +105,9 @@ type S3Mock struct { // SetBucketPolicyFunc mocks the SetBucketPolicy method. SetBucketPolicyFunc func(ctx context.Context, bucketName string, policy string) error + // StatObjectFunc mocks the StatObject method. + StatObjectFunc func(ctx context.Context, bucketName string, objectName string, opts minio.StatObjectOptions) (minio.ObjectInfo, error) + // calls tracks calls to the methods. calls struct { // EndpointURL holds details about calls to the EndpointURL method. @@ -214,6 +220,17 @@ type S3Mock struct { // Policy is the policy argument value. Policy string } + // StatObject holds details about calls to the StatObject method. + StatObject []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // BucketName is the bucketName argument value. + BucketName string + // ObjectName is the objectName argument value. + ObjectName string + // Opts is the opts argument value. + Opts minio.StatObjectOptions + } } lockEndpointURL sync.RWMutex lockGetBucketPolicy sync.RWMutex @@ -227,6 +244,7 @@ type S3Mock struct { lockRemoveObject sync.RWMutex lockRemoveObjects sync.RWMutex lockSetBucketPolicy sync.RWMutex + lockStatObject sync.RWMutex } // EndpointURL calls EndpointURLFunc. @@ -711,3 +729,47 @@ func (mock *S3Mock) SetBucketPolicyCalls() []struct { mock.lockSetBucketPolicy.RUnlock() return calls } + +// StatObject calls StatObjectFunc. +func (mock *S3Mock) StatObject(ctx context.Context, bucketName string, objectName string, opts minio.StatObjectOptions) (minio.ObjectInfo, error) { + if mock.StatObjectFunc == nil { + panic("S3Mock.StatObjectFunc: method is nil but S3.StatObject was just called") + } + callInfo := struct { + Ctx context.Context + BucketName string + ObjectName string + Opts minio.StatObjectOptions + }{ + Ctx: ctx, + BucketName: bucketName, + ObjectName: objectName, + Opts: opts, + } + mock.lockStatObject.Lock() + mock.calls.StatObject = append(mock.calls.StatObject, callInfo) + mock.lockStatObject.Unlock() + return mock.StatObjectFunc(ctx, bucketName, objectName, opts) +} + +// StatObjectCalls gets all the calls that were made to StatObject. +// Check the length with: +// +// len(mockedS3.StatObjectCalls()) +func (mock *S3Mock) StatObjectCalls() []struct { + Ctx context.Context + BucketName string + ObjectName string + Opts minio.StatObjectOptions +} { + var calls []struct { + Ctx context.Context + BucketName string + ObjectName string + Opts minio.StatObjectOptions + } + mock.lockStatObject.RLock() + calls = mock.calls.StatObject + mock.lockStatObject.RUnlock() + return calls +} diff --git a/internal/app/s3manager/s3.go b/internal/app/s3manager/s3.go index a063ca2..f96f02a 100644 --- a/internal/app/s3manager/s3.go +++ b/internal/app/s3manager/s3.go @@ -14,6 +14,7 @@ import ( // S3 is a client to interact with S3 storage. type S3 interface { GetObject(ctx context.Context, bucketName, objectName string, opts minio.GetObjectOptions) (*minio.Object, error) + StatObject(ctx context.Context, bucketName, objectName string, opts minio.StatObjectOptions) (minio.ObjectInfo, error) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) ListObjects(ctx context.Context, bucketName string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo MakeBucket(ctx context.Context, bucketName string, opts minio.MakeBucketOptions) error diff --git a/main.go b/main.go index a0c5c58..3146f04 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,8 @@ type configuration struct { AllowDelete bool ForceDownload bool ListRecursive bool + ShowVersions bool + ShowMetadata bool Port string Timeout int32 SseType string @@ -107,6 +109,12 @@ func parseConfiguration() configuration { listRecursive := viper.GetBool("LIST_RECURSIVE") + viper.SetDefault("SHOW_VERSIONS", false) + showVersions := viper.GetBool("SHOW_VERSIONS") + + viper.SetDefault("SHOW_METADATA", true) + showMetadata := viper.GetBool("SHOW_METADATA") + viper.SetDefault("PORT", "8080") port := viper.GetString("PORT") @@ -127,6 +135,8 @@ func parseConfiguration() configuration { AllowDelete: allowDelete, ForceDownload: forceDownload, ListRecursive: listRecursive, + ShowVersions: showVersions, + ShowMetadata: showMetadata, Port: port, Timeout: timeout, SseType: sseType, @@ -184,7 +194,7 @@ func main() { // S3 management endpoints (with instance in URL) r.Handle("/{instance}/buckets", s3manager.HandleBucketsViewWithManager(s3Manager, templates, configuration.AllowDelete, rootURL, configuration.BucketName)).Methods(http.MethodGet) - r.PathPrefix("/{instance}/buckets/").Handler(s3manager.HandleBucketViewWithManager(s3Manager, templates, configuration.AllowDelete, configuration.ListRecursive, rootURL)).Methods(http.MethodGet) + r.PathPrefix("/{instance}/buckets/").Handler(s3manager.HandleBucketViewWithManager(s3Manager, templates, configuration.AllowDelete, configuration.ListRecursive, rootURL, configuration.ShowVersions, configuration.ShowMetadata)).Methods(http.MethodGet) r.Handle("/{instance}/api/buckets", s3manager.HandleCreateBucketWithManager(s3Manager)).Methods(http.MethodPost) if configuration.AllowDelete { r.Handle("/{instance}/api/buckets/{bucketName}", s3manager.HandleDeleteBucketWithManager(s3Manager)).Methods(http.MethodDelete) @@ -192,7 +202,10 @@ func main() { r.Handle("/{instance}/api/buckets/{bucketName}/objects", s3manager.HandleCreateObjectWithManager(s3Manager, sseType)).Methods(http.MethodPost) r.Handle("/{instance}/api/buckets/{bucketName}/objects/{objectName:.*}/url", s3manager.HandleGenerateURLWithManager(s3Manager)).Methods(http.MethodGet) r.Handle("/{instance}/api/buckets/{bucketName}/objects/{objectName:.*}/public-access", s3manager.HandleCheckPublicAccessWithManager(s3Manager)).Methods(http.MethodGet) - r.Handle("/{instance}/api/buckets/{bucketName}/objects/{objectName:.*}", s3manager.HandleGetObjectWithManager(s3Manager, configuration.ForceDownload)).Methods(http.MethodGet) + if configuration.ShowMetadata { + r.Handle("/{instance}/api/buckets/{bucketName}/objects/{objectName:.*}/metadata", s3manager.HandleGetObjectMetadataWithManager(s3Manager)).Methods(http.MethodGet) + } + r.Handle("/{instance}/api/buckets/{bucketName}/objects/{objectName:.*}", s3manager.HandleGetObjectWithManager(s3Manager, configuration.ForceDownload, configuration.ShowVersions)).Methods(http.MethodGet) if configuration.AllowDelete { r.Handle("/{instance}/api/buckets/{bucketName}/objects/{objectName:.*}", s3manager.HandleDeleteObjectWithManager(s3Manager)).Methods(http.MethodDelete) r.Handle("/{instance}/api/buckets/{bucketName}/objects/bulk-delete", s3manager.HandleBulkDeleteObjectsWithManager(s3Manager)).Methods(http.MethodPost) diff --git a/web/template/bucket.html.tmpl b/web/template/bucket.html.tmpl index 37fb392..c0f993e 100644 --- a/web/template/bucket.html.tmpl +++ b/web/template/bucket.html.tmpl @@ -102,7 +102,20 @@ {{ else }} - + + {{ if .VersionsUnavailable }} +
+
+
+
+ Object versions unavailable +

This bucket does not support listing object versions (it may not have versioning enabled). Showing the latest version of each object instead.

+
+
+
+
+ {{ end }} +
@@ -192,15 +205,26 @@ {{ end }} {{ end }} + {{ if $.ShowVersions }} + Version ID + Latest + {{ end }} {{ range $index, $object := .Objects }} - + {{ $isCollapsedVersion := and $.ShowVersions (not $object.IsPrimaryVersion) }} + - {{ if not $object.IsFolder }} + {{- /* Bulk actions operate on the object key, so only the + primary row of a version group gets a checkbox. */}} + {{ if and (not $object.IsFolder) (not $isCollapsedVersion) }}
+ +