diff --git a/CRAN-SUBMISSION b/CRAN-SUBMISSION index c699a2f4..1b92fd35 100644 --- a/CRAN-SUBMISSION +++ b/CRAN-SUBMISSION @@ -1,3 +1,3 @@ -Version: 0.3.0 -Date: 2025-04-09 19:42:35 UTC -SHA: 6fb4f4175e734d2257e419f80a93fd3466edd665 +Version: 0.4.1 +Date: 2026-05-31 15:33:08 UTC +SHA: 9d42bff0eb37133c5d332a1a963ba096dd2f9e7d diff --git a/DESCRIPTION b/DESCRIPTION index 4a02e23f..b53d8d1a 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -46,4 +46,4 @@ Config/testthat/edition: 3 Encoding: UTF-8 LazyData: true Roxygen: list(markdown = TRUE) -Config/roxygen2/version: 8.0.0 +Config/roxygen2/version: 8.1.0 diff --git a/R/gen_api.R b/R/gen_api.R index 561d4866..35aa518f 100644 --- a/R/gen_api.R +++ b/R/gen_api.R @@ -14,16 +14,17 @@ #' } #' gen_api <- function(..., + api_call_error_ignore = FALSE, use_cache = getOption("restatis.use_cache", TRUE)) { # Choose executing function based on cache option if (isTRUE(use_cache)) { - return(.gen_api_cached(...)) + return(.gen_api_cached(..., api_call_error_ignore = api_call_error_ignore)) } else { - return(.gen_api_core(...)) + return(.gen_api_core(..., api_call_error_ignore = api_call_error_ignore)) } @@ -44,6 +45,7 @@ gen_api <- function(..., #' .gen_api_core <- function(endpoint, database, + api_call_error_ignore, credential_list = NULL, ...) { @@ -102,67 +104,79 @@ gen_api <- function(..., } - #----------------------------------------------------------------------------- - - # First try to request with POST - # If POST errors, try GET - # This allows flexibility across different database instances - # However, GET is deprecated in many instances after V5 + # Catch API parameter values for ... + body_parameters <- list(...) - tryCatch( # tryCatch to try POST + # tryCatch to catch errors from POST request + tryCatch( expr = { - # Catch API parameter values for ... - body_parameters <- list(...) + # Perform API call with POST + .do_post(url = url, + body_parameters = body_parameters, + user_agent = user_agent, + endpoint = endpoint, + username = username, + password = password) + + }, error = function(e) { - # Check if there are any items in ... - if (length(body_parameters) > 0) { + if (isFALSE(api_call_error_ignore)) { - req <- httr2::request(url) %>% - httr2::req_body_form(!!!body_parameters) + stop(paste0("API call to database '", + database, + "' was unsuccessful (error message: '", + e$message, + "'). Check your specifications or try again later."), + call. = FALSE) } else { - # To make a request work with empty ... we need a fake body - req <- httr2::request(url) %>% - httr2::req_body_form(!!!list("foo" = "bar")) + return(list(message = e$message)) } - # Perform API call with POST - req %>% - httr2::req_user_agent(user_agent) %>% - httr2::req_url_path_append(endpoint) %>% - httr2::req_headers("Content-Type" = "application/x-www-form-urlencoded", - "username" = username, - "password" = password) %>% - httr2::req_retry(max_tries = 3) %>% - httr2::req_perform() + }) - }, error = function(e) { +} - tryCatch( # tryCatch to try GET +#------------------------------------------------------------------------------- - expr = { +# Helper to perform POST requests +.do_post <- function(url, + body_parameters, + user_agent, + endpoint, + username, + password) { - httr2::request(url) %>% - httr2::req_user_agent(user_agent) %>% - httr2::req_url_path_append(endpoint) %>% - httr2::req_url_query("username" = username, - "password" = password, ...) %>% - httr2::req_retry(max_tries = 3) %>% - httr2::req_perform() + # Check if there are any items in ... + if (length(body_parameters) > 0) { - }, error = function(e) { + req <- httr2::request(url) %>% + httr2::req_body_form(!!!body_parameters) - stop(paste0("The API call(s) have been tried with GET and POST methods, but were unsuccessful (error message: '", e$message, "'). Check your specifications or try again later."), - call. = FALSE) + } else { - }) + # To make a request work with empty ... we need a fake body + req <- httr2::request(url) %>% + httr2::req_body_form(!!!list("foo" = "bar")) - }) + } - #----------------------------------------------------------------------------- + req <- req %>% + httr2::req_user_agent(user_agent) %>% + httr2::req_url_path_append(endpoint) %>% + httr2::req_headers("Content-Type" = "application/x-www-form-urlencoded", + "username" = username, + "password" = password) %>% + httr2::req_retry(max_tries = 3) %>% + httr2::req_perform() + + req } + +#------------------------------------------------------------------------------- + diff --git a/R/gen_cube.R b/R/gen_cube.R index 7c2b3195..0d224e38 100644 --- a/R/gen_cube.R +++ b/R/gen_cube.R @@ -5,7 +5,7 @@ #' @param name Character string for a cube object. #' @param database Character string. Indicator if the GENESIS ('genesis'), regionalstatistik.de ('regio'), landesdatenbank.nrw.de ('nrw') or bildungsmonitoring.de ('bildung') database is called. #' @param credential_list A list containing the credentials for the databases to be accessed. If 'NULL' (default), the function will use the stored credentials from \code{gen_auth_get()}. -#' @param area Character string. The area in which the table is stored. +#' @param area Character string. The area in which the cube is stored. #' Possible values: #' \itemize{ #' \item \code{"public"}: cube in the public catalogue @@ -17,7 +17,18 @@ #' @param additionals Boolean. Should additional metadata be included? #' @param contents Character string. Names of required statistical specifications. #' @param startyear Four-digit integer. Only retrieve data from this year onward. +#' Note: Starting with {restatis} v0.4.2, no default values for the parameters ‘startyear’ and ‘endyear’ will be submitted anymore +#' (formerly 1900 to 2100 to fetch all available values). Thus, the resulting cube might contain a shorter time span than expected. +#' Instead, we encourage users to explicitly set the desired time span. Doing so, we follow the best practice to save API resources. +#' The new default of ‘NULL’ behaves as follows: Each database will always send data, as each cube is preconfigured with a standard time +#' span that varies for each statistic and each cube. If the user desires different time spans, she has to specify the values accordingly. #' @param endyear Four-digit integer. Only retrieve data up to this year. +#' Note: Starting with {restatis} v0.4.2, no default values for the parameters ‘startyear’ and ‘endyear’ will be submitted anymore +#' (formerly 1900 to 2100 to fetch all available values). Thus, the resulting cube might contain a shorter time span than expected. +#' Instead, we encourage users to explicitly set the desired time span. Doing so, we follow the best practice to save API resources. +#' The new default of ‘NULL’ behaves as follows: Each database will always send data, as each cube is preconfigured with a standard time +#' span that varies for each statistic and each cube. If the user desires different time spans, she has to specify the values accordingly. + #' @param timeslices Integer. Number of timeslices (cumulative to \code{startyear} or \code{endyear}). #' @param regionalvariable Character string. Code of the regional variable whose value #' is specified in \code{regionalkey} to filter the results. @@ -67,8 +78,8 @@ gen_cube <- function(name, values = TRUE, metadata = TRUE, additionals = FALSE, - startyear = 1900, - endyear = 2100, + startyear = NULL, + endyear = NULL, timeslices = NULL, contents = NULL, regionalvariable = NULL, @@ -88,7 +99,7 @@ gen_cube <- function(name, if (missing(database)) { - stop("It is mandatory to specifiy the 'database' parameter for 'gen_table()'.", + stop("It is mandatory to specifiy the 'database' parameter for 'gen_cube()'.", call. = FALSE) } diff --git a/R/gen_find.R b/R/gen_find.R index 81088bb9..055dc753 100644 --- a/R/gen_find.R +++ b/R/gen_find.R @@ -90,13 +90,24 @@ gen_find <- function(term = NULL, term = term, category = category, pagelength = pagelength, + api_call_error_ignore = error.ignore, ...) - results_json <- test_if_json(results_raw) + if (class(results_raw) != "httr2_response") { - empty_object <- test_if_error_find(results_json, para = error.ignore, verbose = verbose) + empty_object <- "HTTP_ERROR" - empty_object <- test_if_process_further(results_json, para = error.ignore, verbose = verbose) + warning(paste0("There has been a HTTPS error in the '", db, "' database (description: ", results_raw[["message"]], ").")) + + } else { + + results_json <- test_if_json(results_raw) + + empty_object <- test_if_error_find(results_json, para = error.ignore, verbose = verbose) + + empty_object <- test_if_process_further(results_json, para = error.ignore, verbose = verbose) + + } } @@ -136,6 +147,14 @@ gen_find <- function(term = NULL, return(list_resp) + } else if (empty_object == "HTTP_ERROR") { + + list_resp <- list("Output" = paste0("There has been a HTTPS error in the '", db, "' database (description: ", results_raw[["message"]], ").")) + + attr(list_resp, "Database") <- db + + return(list_resp) + } else if (empty_object == "DONE") { #--------------------------------------------------------------------------- diff --git a/R/gen_table.R b/R/gen_table.R index 767b5eef..2d392ac2 100644 --- a/R/gen_table.R +++ b/R/gen_table.R @@ -12,10 +12,22 @@ #' \item \code{"user"}: table in the user's account #' \item \code{"all"}: both of the above #' } +#' #' @param compress Boolean. Should empty rows and columns be discarded? #' @param transpose Boolean. Reshape the table between \code{"wide"} and \code{"long"} format. -#' @param startyear Four-digit integer. Only retrieve data from this year onward. -#' @param endyear Four-digit integer. Only retrieve data up to this year. +#' @param startyear Four-digit integer (valid range 1900 - 2100). Only retrieve data from this year onward. Default 'NULL' can help enable the 'timeslices' parameter (see vignettes for additional parameters). +#' Note: Starting with {restatis} v0.4.2, no default values for the parameters 'startyear' and 'endyear' will be submitted anymore +#' (formerly 1900 to 2100 to fetch all available values). Thus, the resulting table might contain a shorter time span than expected. +#' Instead, we encourage users to explicitly set the desired time span. Doing so, we follow the best practice to save API resources. +#' The new default of 'NULL' behaves as follows: Each database will always send data, as each table is preconfigured with a standard time +#' span that varies for each statistic and each table. If the user desires different time spans, she has to specify the values accordingly. +#' @param endyear Four-digit integer (valid range 1900 - 2100). Only retrieve data up to this year. Default 'NULL' can help enable the 'timeslices' parameter (see vignettes for additional parameters). +#' Note: Starting with {restatis} v0.4.2, no default values for the parameters 'startyear' and 'endyear' will be submitted anymore +#' (formerly 1900 to 2100 to fetch all available values). Thus, the resulting table might contain a shorter time span than expected. +#' Instead, we encourage users to explicitly set the desired time span. Doing so, we follow the best practice to save API resources. +#' The new default of 'NULL' behaves as follows: Each database will always send data, as each table is preconfigured with a standard time +#' span that varies for each statistic and each table. If the user desires different time spans, she has to specify the values accordingly. + #' @param regionalvariable Character string. Code of the regional variable whose value #' is specified in \code{regionalkey} to filter the results. #' @param regionalkey Character string. One or more regional keys. Multiple values can be @@ -69,8 +81,8 @@ gen_table <- function(name, area = c("all", "public", "user"), compress = FALSE, transpose = FALSE, - startyear = 1900, - endyear = 2100, + startyear = NULL, + endyear = NULL, regionalvariable = NULL, regionalkey = NULL, classifyingvariable1 = NULL, diff --git a/R/utils_api_calls.R b/R/utils_api_calls.R index b4c906c9..e12c69b7 100644 --- a/R/utils_api_calls.R +++ b/R/utils_api_calls.R @@ -46,6 +46,22 @@ test_if_okay <- function(input) { #' test_if_error_find <- function(input, para, verbose = NULL) { + if (is.null(input$Cubes) & + is.null(input$Statistics) & + is.null(input$Tables) & + is.null(input$Timeseries) & + is.null(input$Variables)) { + + if (!is.null(verbose) && isTRUE(verbose)) { + + message("No object found for your request. Artificial token is used.") + + } + + return(FALSE) + + } + if (input$Status$Code != 0 && isTRUE(para) && input$Status$Code != 22) { stop(input$Status$Content) diff --git a/R/utils_dataprocessing.R b/R/utils_dataprocessing.R index e06da226..9197b45d 100644 --- a/R/utils_dataprocessing.R +++ b/R/utils_dataprocessing.R @@ -8,13 +8,24 @@ #' param_check_year <- function(year) { - if (as.integer(year) < 1900 || as.integer(year) > 2100) { + if (!is.null(year) & !is.numeric(year)) { - stop("The parameter 'year' has been misspecified (>= 1900 or <= 2100).", + stop("Parameter 'year' has to be of value 'NULL' or of type 'numeric'.", call. = FALSE) } + if (!is.null(year) & is.integer(as.integer(year))) { + + if (as.integer(year) < 1900 || as.integer(year) > 2100) { + + stop("The parameter 'year' has been misspecified (>= 1900 or <= 2100).", + call. = FALSE) + + } + + } + } #------------------------------------------------------------------------------- diff --git a/README.Rmd b/README.Rmd index 08e0bf00..5ef87e31 100644 --- a/README.Rmd +++ b/README.Rmd @@ -69,6 +69,9 @@ abbreviation strings are the following: use literal lines of CSVs. This does not affect the functionality of the function, so for now you can safely ignore it. We monitor whether there will be a change on `{readr}`'s side or will implement a fix with upcoming updates. +- There are reports on the `regio` database occasionally returning incomplete or + corrupted tables. We are investigating (see issues, please amend if you encounter + the same problems). ## Installation diff --git a/README.md b/README.md index 6b701718..68c740fd 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ abbreviation strings are the following: functionality of the function, so for now you can safely ignore it. We monitor whether there will be a change on `{readr}`’s side or will implement a fix with upcoming updates. +- There are reports on the `regio` database occasionally returning + incomplete or corrupted tables. We are investigating (see issues, + please amend if you encounter the same problems). ## Installation diff --git a/man/dot-gen_api_core.Rd b/man/dot-gen_api_core.Rd index befc9f09..f6c79040 100644 --- a/man/dot-gen_api_core.Rd +++ b/man/dot-gen_api_core.Rd @@ -4,7 +4,13 @@ \alias{.gen_api_core} \title{Basic API request function} \usage{ -.gen_api_core(endpoint, database, credential_list = NULL, ...) +.gen_api_core( + endpoint, + database, + api_call_error_ignore, + credential_list = NULL, + ... +) } \arguments{ \item{endpoint}{Character string. The endpoint of the API that is to be queried.} diff --git a/man/gen_cube.Rd b/man/gen_cube.Rd index 648b5d8e..354b6817 100644 --- a/man/gen_cube.Rd +++ b/man/gen_cube.Rd @@ -12,8 +12,8 @@ gen_cube( values = TRUE, metadata = TRUE, additionals = FALSE, - startyear = 1900, - endyear = 2100, + startyear = NULL, + endyear = NULL, timeslices = NULL, contents = NULL, regionalvariable = NULL, @@ -36,7 +36,7 @@ gen_cube( \item{credential_list}{A list containing the credentials for the databases to be accessed. If 'NULL' (default), the function will use the stored credentials from \code{gen_auth_get()}.} -\item{area}{Character string. The area in which the table is stored. +\item{area}{Character string. The area in which the cube is stored. Possible values: \itemize{ \item \code{"public"}: cube in the public catalogue @@ -50,9 +50,19 @@ Possible values: \item{additionals}{Boolean. Should additional metadata be included?} -\item{startyear}{Four-digit integer. Only retrieve data from this year onward.} - -\item{endyear}{Four-digit integer. Only retrieve data up to this year.} +\item{startyear}{Four-digit integer. Only retrieve data from this year onward. +Note: Starting with {restatis} v0.4.2, no default values for the parameters ‘startyear’ and ‘endyear’ will be submitted anymore +(formerly 1900 to 2100 to fetch all available values). Thus, the resulting cube might contain a shorter time span than expected. +Instead, we encourage users to explicitly set the desired time span. Doing so, we follow the best practice to save API resources. +The new default of ‘NULL’ behaves as follows: Each database will always send data, as each cube is preconfigured with a standard time +span that varies for each statistic and each cube. If the user desires different time spans, she has to specify the values accordingly.} + +\item{endyear}{Four-digit integer. Only retrieve data up to this year. +Note: Starting with {restatis} v0.4.2, no default values for the parameters ‘startyear’ and ‘endyear’ will be submitted anymore +(formerly 1900 to 2100 to fetch all available values). Thus, the resulting cube might contain a shorter time span than expected. +Instead, we encourage users to explicitly set the desired time span. Doing so, we follow the best practice to save API resources. +The new default of ‘NULL’ behaves as follows: Each database will always send data, as each cube is preconfigured with a standard time +span that varies for each statistic and each cube. If the user desires different time spans, she has to specify the values accordingly.} \item{timeslices}{Integer. Number of timeslices (cumulative to \code{startyear} or \code{endyear}).} diff --git a/man/gen_table.Rd b/man/gen_table.Rd index a1950158..246de9e2 100644 --- a/man/gen_table.Rd +++ b/man/gen_table.Rd @@ -11,8 +11,8 @@ gen_table( area = c("all", "public", "user"), compress = FALSE, transpose = FALSE, - startyear = 1900, - endyear = 2100, + startyear = NULL, + endyear = NULL, regionalvariable = NULL, regionalkey = NULL, classifyingvariable1 = NULL, @@ -47,9 +47,19 @@ Possible values: \item{transpose}{Boolean. Reshape the table between \code{"wide"} and \code{"long"} format.} -\item{startyear}{Four-digit integer. Only retrieve data from this year onward.} - -\item{endyear}{Four-digit integer. Only retrieve data up to this year.} +\item{startyear}{Four-digit integer (valid range 1900 - 2100). Only retrieve data from this year onward. Default 'NULL' can help enable the 'timeslices' parameter (see vignettes for additional parameters). +Note: Starting with {restatis} v0.4.2, no default values for the parameters 'startyear' and 'endyear' will be submitted anymore +(formerly 1900 to 2100 to fetch all available values). Thus, the resulting table might contain a shorter time span than expected. +Instead, we encourage users to explicitly set the desired time span. Doing so, we follow the best practice to save API resources. +The new default of 'NULL' behaves as follows: Each database will always send data, as each table is preconfigured with a standard time +span that varies for each statistic and each table. If the user desires different time spans, she has to specify the values accordingly.} + +\item{endyear}{Four-digit integer (valid range 1900 - 2100). Only retrieve data up to this year. Default 'NULL' can help enable the 'timeslices' parameter (see vignettes for additional parameters). +Note: Starting with {restatis} v0.4.2, no default values for the parameters 'startyear' and 'endyear' will be submitted anymore +(formerly 1900 to 2100 to fetch all available values). Thus, the resulting table might contain a shorter time span than expected. +Instead, we encourage users to explicitly set the desired time span. Doing so, we follow the best practice to save API resources. +The new default of 'NULL' behaves as follows: Each database will always send data, as each table is preconfigured with a standard time +span that varies for each statistic and each table. If the user desires different time spans, she has to specify the values accordingly.} \item{regionalvariable}{Character string. Code of the regional variable whose value is specified in \code{regionalkey} to filter the results.}