diff --git a/NEWS.md b/NEWS.md index 2c0718d3..14fa2a2b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,6 +7,8 @@ ## New features * Added `target_transform_resize` to manage bounding_box resizing in parallel to `transform_resize` for images (#337). +* Added article showcasing `model_fasterrcnn_resnet50_fpn()` with visualization utilities `draw_bounding_boxes()` and `vision_make_grid()` (@DerrickUnleashed, #301). + ## Bug fixes and improvements @@ -26,7 +28,9 @@ * Added article showcasing `model_fcn_resnet50()` with visualization utilities `draw_segmentation_masks()` and `vision_make_grid()` (@DerrickUnleashed, #281). * Added collection dataset catalog with `search_collection()`, `get_collection_catalog()`, and `list_collection_datasets()` functions for discovering and exploring collections (#271, @ANAMASGARD). * Added `target_transform_coco_masks()` and `target_transform_trimap_masks()` transformation functions for explicit segmentation mask generation (@ANAMASGARD). -* Added support for `connectivity` argument for drawing lines between keypoints in `draw_keypoints()` (@DerrickUnleashed, #303). + +* Added support for `connectivity` argument for drawing lines between keypoints in `draw_keypoints()` (@DerrickUnleashed #303) + ## New models diff --git a/R/models-faster_rcnn.R b/R/models-faster_rcnn.R index 04cfd36a..57f60665 100644 --- a/R/models-faster_rcnn.R +++ b/R/models-faster_rcnn.R @@ -975,18 +975,47 @@ mobilenet_v3_320_fpn_backbone <- function(pretrained = TRUE) { #' # ResNet backbone requires image normalization #' input <- image %>% #' transform_normalize(norm_mean, norm_std) -#' batch_normalized <- input$unsqueeze(1) # Add batch dimension (1, 3, H, W) +#' batch <- input$unsqueeze(1) # Add batch dimension (1, 3, H, W) +#' +#' # ResNet-50 FPN +#' model <- model_fasterrcnn_resnet50_fpn(pretrained = TRUE, score_thresh = 0.5, +#' nms_thresh = 0.8, detections_per_img = 3) +#' model$eval() +#' pred <- model(batch)$detections[[1]] +#' num_boxes <- as.integer(pred$boxes$size()[1]) +#' keep <- seq_len(min(5, num_boxes)) +#' boxes <- pred$boxes[keep, ]$view(c(-1, 4)) +#' labels <- coco_classes(as.integer(pred$labels[keep])) +#' if (num_boxes > 0) { +#' boxed <- draw_bounding_boxes(image, boxes, labels = labels) +#' tensor_image_browse(boxed) +#' } #' #' # ResNet-50 FPN V2 #' model <- model_fasterrcnn_resnet50_fpn_v2(pretrained = TRUE, , detections_per_img = 5 ) #' model$eval() -#' torch::with_no_grad({pred <- model(batch_normalized)$detections[[1]]}) -#' labels <- coco_classes(as.integer(pred$labels)) +#' pred <- model(batch)$detections[[1]] +#' num_boxes <- as.integer(pred$boxes$size()[1]) +#' keep <- seq_len(min(5, num_boxes)) +#' boxes <- pred$boxes[keep, ]$view(c(-1, 4)) +#' labels <- coco_classes(as.integer(pred$labels[keep])) +#' if (num_boxes > 0) { +#' boxed <- draw_bounding_boxes(img, boxes, labels = labels) +#' tensor_image_browse(boxed) +#' } #' -#' # Visualize boxes -#' labels <- coco_classes(as.integer(pred$labels)) -#' boxed <- draw_bounding_boxes(image, pred$boxes, labels = labels) -#' tensor_image_browse(boxed) +#' # MobileNet V3 Large FPN +#' model <- model_fasterrcnn_mobilenet_v3_large_fpn(pretrained = TRUE) +#' model$eval() +#' pred <- model(batch)$detections[[1]] +#' num_boxes <- as.integer(pred$boxes$size()[1]) +#' keep <- seq_len(min(5, num_boxes)) +#' boxes <- pred$boxes[keep, ]$view(c(-1, 4)) +#' labels <- coco_classes(as.integer(pred$labels[keep])) +#' if (num_boxes > 0) { +#' boxed <- draw_bounding_boxes(image, boxes, labels = labels) +#' tensor_image_browse(boxed) +#' } #' #' # MobileNet V3 Large 320 FPN #' batch <- image$unsqueeze(1) # Add batch dimension (1, 3, H, W) @@ -994,12 +1023,15 @@ mobilenet_v3_320_fpn_backbone <- function(pretrained = TRUE) { #' pretrained = TRUE, score_thresh = 0.02, nms_thresh = 0.8, detections_per_img = 5 #' ) #' model$eval() -#' torch::with_no_grad({pred <- model(batch)$detections[[1]]}) -#' -#' # Visualize boxes -#' labels <- coco_classes(as.integer(pred$labels)) -#' boxed <- draw_bounding_boxes(image, pred$boxes, labels = labels) -#' tensor_image_browse(boxed) +#' pred <- model(batch)$detections[[1]] +#' num_boxes <- as.integer(pred$boxes$size()[1]) +#' keep <- seq_len(min(5, num_boxes)) +#' boxes <- pred$boxes[keep, ]$view(c(-1, 4)) +#' labels <- coco_classes(as.integer(pred$labels[keep]))] +#' if (num_boxes > 0) { +#' boxed <- draw_bounding_boxes(image, boxes, labels = labels) +#' tensor_image_browse(boxed) +#' } #' } #' #' @family object_detection_model diff --git a/_pkgdown.yml b/_pkgdown.yml index 21aef695..19793758 100644 --- a/_pkgdown.yml +++ b/_pkgdown.yml @@ -31,6 +31,8 @@ navbar: href: articles/examples/fcnresnet.html - text: keypoints href: articles/examples/keypoints.html + - text: fasterrcnn + href: articles/examples/fasterrcnn.html reference: - title: Transforms diff --git a/man/model_fasterrcnn.Rd b/man/model_fasterrcnn.Rd index 5e01099a..ba84e25b 100644 --- a/man/model_fasterrcnn.Rd +++ b/man/model_fasterrcnn.Rd @@ -116,18 +116,47 @@ image <- magick_loader(url) \%>\% # ResNet backbone requires image normalization input <- image \%>\% transform_normalize(norm_mean, norm_std) -batch_normalized <- input$unsqueeze(1) # Add batch dimension (1, 3, H, W) +batch <- input$unsqueeze(1) # Add batch dimension (1, 3, H, W) + +# ResNet-50 FPN +model <- model_fasterrcnn_resnet50_fpn(pretrained = TRUE, score_thresh = 0.5, + nms_thresh = 0.8, detections_per_img = 3) +model$eval() +pred <- model(batch)$detections[[1]] +num_boxes <- as.integer(pred$boxes$size()[1]) +keep <- seq_len(min(5, num_boxes)) +boxes <- pred$boxes[keep, ]$view(c(-1, 4)) +labels <- coco_classes(as.integer(pred$labels[keep])) +if (num_boxes > 0) { + boxed <- draw_bounding_boxes(image, boxes, labels = labels) + tensor_image_browse(boxed) +} # ResNet-50 FPN V2 model <- model_fasterrcnn_resnet50_fpn_v2(pretrained = TRUE, , detections_per_img = 5 ) model$eval() -torch::with_no_grad({pred <- model(batch_normalized)$detections[[1]]}) -labels <- coco_classes(as.integer(pred$labels)) +pred <- model(batch)$detections[[1]] +num_boxes <- as.integer(pred$boxes$size()[1]) +keep <- seq_len(min(5, num_boxes)) +boxes <- pred$boxes[keep, ]$view(c(-1, 4)) +labels <- coco_classes(as.integer(pred$labels[keep])) +if (num_boxes > 0) { + boxed <- draw_bounding_boxes(img, boxes, labels = labels) + tensor_image_browse(boxed) +} -# Visualize boxes -labels <- coco_classes(as.integer(pred$labels)) -boxed <- draw_bounding_boxes(image, pred$boxes, labels = labels) -tensor_image_browse(boxed) +# MobileNet V3 Large FPN +model <- model_fasterrcnn_mobilenet_v3_large_fpn(pretrained = TRUE) +model$eval() +pred <- model(batch)$detections[[1]] +num_boxes <- as.integer(pred$boxes$size()[1]) +keep <- seq_len(min(5, num_boxes)) +boxes <- pred$boxes[keep, ]$view(c(-1, 4)) +labels <- coco_classes(as.integer(pred$labels[keep])) +if (num_boxes > 0) { + boxed <- draw_bounding_boxes(image, boxes, labels = labels) + tensor_image_browse(boxed) +} # MobileNet V3 Large 320 FPN batch <- image$unsqueeze(1) # Add batch dimension (1, 3, H, W) @@ -135,12 +164,15 @@ model <- model_fasterrcnn_mobilenet_v3_large_320_fpn( pretrained = TRUE, score_thresh = 0.02, nms_thresh = 0.8, detections_per_img = 5 ) model$eval() -torch::with_no_grad({pred <- model(batch)$detections[[1]]}) - -# Visualize boxes -labels <- coco_classes(as.integer(pred$labels)) -boxed <- draw_bounding_boxes(image, pred$boxes, labels = labels) -tensor_image_browse(boxed) +pred <- model(batch)$detections[[1]] +num_boxes <- as.integer(pred$boxes$size()[1]) +keep <- seq_len(min(5, num_boxes)) +boxes <- pred$boxes[keep, ]$view(c(-1, 4)) +labels <- coco_classes(as.integer(pred$labels[keep]))] +if (num_boxes > 0) { + boxed <- draw_bounding_boxes(image, boxes, labels = labels) + tensor_image_browse(boxed) +} } } diff --git a/vignettes/examples/faster-rcnn.R b/vignettes/examples/faster-rcnn.R new file mode 100644 index 00000000..adc10984 --- /dev/null +++ b/vignettes/examples/faster-rcnn.R @@ -0,0 +1,81 @@ +# Loading Images --------------------------------------------------- +library(torchvision) +library(torch) + +url1 <- "https://raw.githubusercontent.com/pytorch/vision/main/gallery/assets/dog1.jpg" +url2 <- "https://raw.githubusercontent.com/pytorch/vision/main/gallery/assets/dog2.jpg" + +dog1 <- base_loader(url1) %>% transform_to_tensor() +dog2 <- base_loader(url2) %>% transform_to_tensor() + + +# Visualizing a grid of images ------------------------------------- + + +dogs <- torch_stack(list(dog1, dog2)) +grid <- vision_make_grid(dogs, scale = TRUE, num_rows = 2) +tensor_image_browse(grid) + + +# Preprocessing the data ------------------------------------------- + + +norm_mean <- c(0.485, 0.456, 0.406) +norm_std <- c(0.229, 0.224, 0.225) + +dog1_prep <- dog1 %>% + transform_resize(c(800, 800)) %>% + transform_normalize(mean = norm_mean, std = norm_std) %>% + torch_tensor(dtype = torch_float32()) +dog2_prep <- dog2 %>% + transform_resize(c(800, 800)) %>% + transform_normalize(mean = norm_mean, std = norm_std) %>% + torch_tensor(dtype = torch_float32()) + +# make batch (2,3,800,800) +dog_batch <- torch_stack(list(dog1_prep, dog2_prep)) + + +# Loading Model ---------------------------------------------------- + + +model <- model_fasterrcnn_resnet50_fpn( + pretrained = TRUE, + score_thresh = 0.5, + nms_thresh = 0.8, + detections_per_img = 2 +) +model$eval() + +# run model +output <- model(dog_batch) + + +# Processing the Output -------------------------------------------- + +pred1 <- output$detections[[1]] +pred2 <- output$detections[[2]] + +pred1$boxes +pred1$labels +pred1$scores + + +# Visualizing the Output ------------------------------------------- + + +boxed1 <- draw_bounding_boxes( + dog1 %>% transform_resize(c(800, 800)), + boxes = pred1$boxes, + labels = coco_classes(as.integer(pred1$labels)) +) + +boxed2 <- draw_bounding_boxes( + dog2 %>% transform_resize(c(800, 800)), + boxes = pred2$boxes, + labels = coco_classes(as.integer(pred2$labels)) +) + +boxed <- torch_stack(list(boxed1, boxed2)) +grid <- vision_make_grid(boxed, scale = FALSE, num_rows = 2) +tensor_image_browse(grid) \ No newline at end of file diff --git a/vignettes/examples/faster-rcnn.Rmd b/vignettes/examples/faster-rcnn.Rmd new file mode 100644 index 00000000..4853275d --- /dev/null +++ b/vignettes/examples/faster-rcnn.Rmd @@ -0,0 +1,9 @@ +--- +title: "faster_rcnn" +type: docs +--- + +```{r, echo = FALSE} +knitr::opts_chunk$set(eval = FALSE) +knitr::spin_child(paste0(rmarkdown::metadata$title, ".R")) +```