diff --git a/.github/workflows/test-backend.yml b/.github/workflows/test-backend.yml index 94f6326d..331745bd 100644 --- a/.github/workflows/test-backend.yml +++ b/.github/workflows/test-backend.yml @@ -55,3 +55,39 @@ jobs: - name: Run tests run: | tests/test.sh + + sboltestrunner: + name: SBOLTestRunner + needs: build + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@v3 + name: Checkout source tree + - uses: actions/download-artifact@v4 + name: Download Docker image + with: + name: sbh3-image + - name: Import saved Docker image + run: | + cat sbh3.tar.gz | docker load + - name: Install Docker Compose + run: | + sudo apt-get update + sudo apt-get install -y docker-compose + - uses: actions/setup-java@v4 + name: Install Java + with: + distribution: temurin + java-version: '17' + - uses: actions/setup-python@v4 + name: Install Python + with: + python-version: '3.9' + - name: Install test dependencies + run: | + pip install -r tests/test_requirements.txt + - name: Run SBOLTestRunner + run: | + cd tests + bash ./sbolsuite.sh diff --git a/.gitignore b/.gitignore index 12807c1a..73162b40 100644 --- a/.gitignore +++ b/.gitignore @@ -171,4 +171,6 @@ tests/Compared/ tests/Emulated/ tests/Retrieved/ tests/SynBioHubRunner/ +tests/Timing/ +tests/sbol_testrunner_result tests/logs_from_test_suite/ diff --git a/backend/src/main/java/com/synbiohub/sbh3/controllers/ExposeController.java b/backend/src/main/java/com/synbiohub/sbh3/controllers/ExposeController.java new file mode 100644 index 00000000..d79c40c5 --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/controllers/ExposeController.java @@ -0,0 +1,40 @@ +package com.synbiohub.sbh3.controllers; + +import com.synbiohub.sbh3.submit.SubmitExposeRegistry; +import lombok.RequiredArgsConstructor; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RestController; + +import java.net.URLConnection; +import java.nio.file.Files; +import java.nio.file.Path; + +@RestController +@RequiredArgsConstructor +public class ExposeController { + + private final SubmitExposeRegistry exposeRegistry; + + @GetMapping("/expose/{token}") + public ResponseEntity exposeFile(@PathVariable String token) { + Path file = exposeRegistry.resolve(token); + if (file == null || !Files.isRegularFile(file)) { + return ResponseEntity.notFound().build(); + } + String filename = file.getFileName().toString(); + String contentType = URLConnection.guessContentTypeFromName(filename); + MediaType mediaType = contentType != null + ? MediaType.parseMediaType(contentType) + : MediaType.APPLICATION_OCTET_STREAM; + return ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "inline; filename=\"" + filename + "\"") + .contentType(mediaType) + .body(new FileSystemResource(file)); + } +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/controllers/SubmitController.java b/backend/src/main/java/com/synbiohub/sbh3/controllers/SubmitController.java index 8f46ecdd..14c4a1e7 100644 --- a/backend/src/main/java/com/synbiohub/sbh3/controllers/SubmitController.java +++ b/backend/src/main/java/com/synbiohub/sbh3/controllers/SubmitController.java @@ -1,15 +1,19 @@ package com.synbiohub.sbh3.controllers; import com.synbiohub.sbh3.services.SubmitService; +import com.synbiohub.sbh3.submit.SubmitPayload; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; import org.sbolstandard.core2.SBOLDocument; +import org.sbolstandard.core2.SBOLValidationException; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; +import java.io.IOException; import java.util.Map; @Tag(name = "Submissions", description = "Endpoints for creating and submitting registry objects") @@ -21,17 +25,17 @@ public class SubmitController { /** * Create a new collection or submit file contents into an existing collection. *

- * String form fields are bound via {@code allParams} (e.g. id, version, name, - * description, - * citations, overwrite_merge, rootCollections, plugin). The uploaded design - * file is bound - * separately because multipart files are not included in a - * {@code Map}. + * String form fields are bound via {@code allParams} (e.g. id, version, name, description, + * citations, overwrite_merge, rootCollections, plugin). The uploaded design file is bound + * separately because multipart files are not included in a {@code Map}. */ - @PostMapping(value = "/submit", consumes = MediaType.MULTIPART_FORM_DATA_VALUE, produces = "text/plain; charset=UTF-8") - @Operation(summary = "Submit object (stub)", description = "Temporary stub so the branch compiles.") - public ResponseEntity submit(@RequestParam Map allParams) { - return ResponseEntity.ok("Form submitted"); + @PostMapping(value = "/submit", + consumes = MediaType.MULTIPART_FORM_DATA_VALUE, + produces = "text/plain; charset=UTF-8") + public ResponseEntity submit( + @ModelAttribute SubmitPayload allParams, + @RequestPart(value = "file", required = false) MultipartFile file) throws IOException, SBOLValidationException { + return submitService.submit(allParams, file); } @Operation(summary = "Create new collection (Unimplemented)", description = "Currently an empty stub.", deprecated = true) diff --git a/backend/src/main/java/com/synbiohub/sbh3/dao/SparqlService.java b/backend/src/main/java/com/synbiohub/sbh3/dao/SparqlService.java index 16705a0b..fa98d2e6 100644 --- a/backend/src/main/java/com/synbiohub/sbh3/dao/SparqlService.java +++ b/backend/src/main/java/com/synbiohub/sbh3/dao/SparqlService.java @@ -1,5 +1,7 @@ package com.synbiohub.sbh3.dao; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.synbiohub.sbh3.repo.SparqlRepository; import com.synbiohub.sbh3.sparql.SPARQLQuery; import com.synbiohub.sbh3.utils.ConfigUtil; @@ -12,6 +14,7 @@ import org.springframework.stereotype.Service; import java.io.IOException; +import java.nio.file.Path; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -22,8 +25,85 @@ public class SparqlService { private final SparqlRepository sparqlRepository; + private final ObjectMapper objectMapper; private final JsonUtil jsonUtil; + /** + * Virtuoso DELETE templates return one row per batch; loop until nothing remains. + * Legacy {@code sparql.deleteStaggered}. + */ + public void deleteCollection(Map params, String graphUri) throws IOException { + deleteStaggered(sparqlRepository.REMOVE_COLLECTION_SPARQL, params, graphUri); + } + + public void delete(Map params, String graphUri) throws IOException { + deleteStaggered(sparqlRepository.REMOVE_SPARQL, params, graphUri); + } + + public void uploadGraphStore(String graphUri, Path file) throws IOException { + sparqlRepository.save(graphUri, file); + } + + public void uploadAttachment(Map params, String graphUri) throws IOException { + String query = new SPARQLQuery(sparqlRepository.ATTACHMENT_UPDATE_SPARQL).loadTemplate(params); + sparqlRepository.update(query, graphUri, false); + } + + public void update(String query, String graphUri, boolean jsonResults) throws IOException { + sparqlRepository.update(query, graphUri, jsonResults); + } + + private void deleteStaggered(String queryTemplate, Map params, String graphUri) throws IOException { + String query = new SPARQLQuery(queryTemplate).loadTemplate(params); + while (true) { + String raw = sparqlRepository.update(query, graphUri, true); + JsonNode bindings = objectMapper.readTree(raw).path("results").path("bindings"); + if (!bindings.isArray() || bindings.isEmpty()) { + break; + } + String msg = bindings.get(0).path("callret-0").path("value").asText(""); + if (msg.contains("nothing to do")) { + break; + } + } + } + + /** Maps existing {@code sbol:source} values (e.g. {@code file:foo.png}) to attachment URIs. */ + public Map loadAttachmentSources(String collectionUri, String graphUri) throws IOException { + String query = new SPARQLQuery(SparqlRepository.GET_ATTACHMENT_SOURCE_SPARQL) + .loadTemplate(Map.of("uri", collectionUri)); + String raw = sparqlRepository.getQuery(query, graphUri); + Map sources = new HashMap<>(); + JsonNode bindings = objectMapper.readTree(raw).path("results").path("bindings"); + if (!bindings.isArray()) { + return sources; + } + for (JsonNode row : bindings) { + String source = row.path("source").path("value").asText(null); + String attachment = row.path("attachment").path("value").asText(null); + if (source != null && attachment != null) { + sources.put(source, attachment); + } + } + return sources; + } + + public void attachUpload(Map params, String graphUri, boolean jsonResults) throws IOException { + String query = new SPARQLQuery(sparqlRepository.ATTACH_UPLOAD_SPARQL).loadTemplate(params); + update(query, graphUri, false); + } + + /** Replaces hash/size on an existing attachment when re-uploading the same {@code file:} source. */ + public void updateAttachment(String graphUri, String attachmentUri, String uploadHash, long size) + throws IOException { + String query = new SPARQLQuery(sparqlRepository.UPDATE_ATTACHMENT_SPARQL).loadTemplate(Map.of( + "attachmentURI", attachmentUri, + "attachmentSource", attachmentUri + "/download", + "hash", StringUtil.sparqlStringLiteral(uploadHash), + "size", StringUtil.sparqlStringLiteral(Long.toString(size)))); + update(query, graphUri, false); + } + public Map buildSearchQuery(Map allParams) { HashMap sparqlArgs = new HashMap<> (Map.of("from", "", "criteria", "", "limit", "", "offset", "")); diff --git a/backend/src/main/java/com/synbiohub/sbh3/dto/SubmissionDTO.java b/backend/src/main/java/com/synbiohub/sbh3/dto/SubmissionDTO.java deleted file mode 100644 index 1e804740..00000000 --- a/backend/src/main/java/com/synbiohub/sbh3/dto/SubmissionDTO.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.synbiohub.sbh3.dto; - -import lombok.Builder; -import lombok.Data; -import org.sbolstandard.core2.SBOLDocument; - -import java.io.File; -import java.util.Collection; -import java.util.List; - -@Data -@Builder -public class SubmissionDTO { - private String name; - private String description; - private String id; - private Integer version; - - private List citations; - private Collection files; - private Integer overwriteMerge; - private List plugins; - private Collection attachments; - private SBOLDocument sbolDocument; - // TODO: add rootcollections - -} diff --git a/backend/src/main/java/com/synbiohub/sbh3/dto/SubmitFileFormat.java b/backend/src/main/java/com/synbiohub/sbh3/dto/SubmitFileFormat.java new file mode 100644 index 00000000..41cc022d --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/dto/SubmitFileFormat.java @@ -0,0 +1,5 @@ +package com.synbiohub.sbh3.dto; + +public enum SubmitFileFormat { + SBOL, GENBANK, FASTA, GFF3, ATTACHMENT +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/repo/SparqlRepository.java b/backend/src/main/java/com/synbiohub/sbh3/repo/SparqlRepository.java index 6804e0b7..4e31cfdc 100644 --- a/backend/src/main/java/com/synbiohub/sbh3/repo/SparqlRepository.java +++ b/backend/src/main/java/com/synbiohub/sbh3/repo/SparqlRepository.java @@ -1,19 +1,43 @@ package com.synbiohub.sbh3.repo; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.synbiohub.sbh3.utils.ConfigUtil; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.io.entity.ByteArrayEntity; import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Component; import org.springframework.web.client.RestClient; +import org.springframework.web.server.ResponseStatusException; import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; @RequiredArgsConstructor @Component @Slf4j public class SparqlRepository { + // SPARQL templates used during prepare (overwrite) and upload (attachments). + public static final String REMOVE_COLLECTION_SPARQL = "src/main/java/com/synbiohub/sbh3/sparql/removeCollection.sparql"; + public static final String REMOVE_SPARQL = "src/main/java/com/synbiohub/sbh3/sparql/remove.sparql"; + public static final String GET_ATTACHMENT_SOURCE_SPARQL = "src/main/java/com/synbiohub/sbh3/sparql/GetAttachmentSourceFromTopLevel.sparql"; + public static final String ATTACH_UPLOAD_SPARQL = "src/main/java/com/synbiohub/sbh3/sparql/AttachUpload.sparql"; + public static final String UPDATE_ATTACHMENT_SPARQL = "src/main/java/com/synbiohub/sbh3/sparql/UpdateAttachment.sparql"; + public static final String ATTACHMENT_UPDATE_SPARQL = "src/main/java/com/synbiohub/sbh3/sparql/AttachmentUpdate.sparql"; public static final String SEARCH_SPARQL = "src/main/java/com/synbiohub/sbh3/sparql/search.sparql"; public static final String SEARCH_COUNT_SPARQL = "src/main/java/com/synbiohub/sbh3/sparql/searchCount.sparql"; public static final String COUNT_SPARQL = "src/main/java/com/synbiohub/sbh3/sparql/Count.sparql"; @@ -22,14 +46,20 @@ public class SparqlRepository { public static final String SHARED_VIEW_SPARQL = "src/main/java/com/synbiohub/sbh3/sparql/GetSharedCanView.sparql"; public static final String TOPLEVEL_METADATA_SPARQL = "src/main/java/com/synbiohub/sbh3/sparql/GetTopLevelMetadata.sparql"; + private final ObjectMapper objectMapper; private final RestClient restClient; /** - * Runs a read-only SPARQL query via POST (same parameters as , for large queries). + * Runs a read-only SPARQL query against {@code sparqlEndpoint} and returns JSON results. */ - public String executePostQuery(String url, String uri, String query) throws IOException { - return restClient.post() - .uri(url, uri, query) + public String getQuery(String query) throws IOException { + return getQuery(query, null); + } + + public String getQuery(String query, String defaultGraphUri) throws IOException { + String graphUri = resolveGraphUri(defaultGraphUri); + return restClient.get() + .uri(sparqlQueryUrl(), graphUri, query) .retrieve() .body(String.class); } @@ -41,6 +71,31 @@ public String executeReadQuery(String url, String uri, String query) { .body(String.class); } + /** + * Runs a read-only SPARQL query via POST (same parameters as {@link #getQuery}, for large queries). + */ + public String postQuery(String query) throws IOException { + return postQuery(query, null); + } + + public String postQuery(String query, String defaultGraphUri) throws IOException { + String graphUri = resolveGraphUri(defaultGraphUri); + return restClient.post() + .uri(sparqlQueryUrl(), graphUri, query) + .retrieve() + .body(String.class); + } + + /** + * Runs a read-only SPARQL query via POST (same parameters as {@link #executeReadQuery}, for large queries). + */ + public String executePostQuery(String url, String uri, String query) throws IOException { + return restClient.post() + .uri(url, uri, query) + .retrieve() + .body(String.class); + } + /** * POST JSON to an arbitrary URL. Replaces the legacy {@code com.synbiohub.sbh3.utils.RestClient}. */ @@ -59,4 +114,103 @@ public ResponseEntity postJson(String uri, Object requestBody, Class r .retrieve() .toEntity(responseType); } + + /** + * POST a SPARQL update to sparql-auth with digest auth (not preemptive basic auth). + * + * @param jsonResults when {@code true}, requests {@code application/sparql-results+json} + */ + public String update(String query, String graphUri, boolean jsonResults) throws IOException { + StringBuilder url = new StringBuilder(sparqlAuthEndpoint()); + url.append("?query=").append(URLEncoder.encode(query, StandardCharsets.UTF_8)); + url.append("&default-graph-uri=").append(URLEncoder.encode(graphUri, StandardCharsets.UTF_8)); + if (jsonResults) { + url.append("&format=") + .append(URLEncoder.encode("application/sparql-results+json", StandardCharsets.UTF_8)); + } + + try (CloseableHttpClient client = virtuosoDigestClient()) { + HttpPost post = new HttpPost(url.toString()); + return client.execute(post, response -> { + int code = response.getCode(); + if (code >= 300) { + throw new ResponseStatusException(HttpStatus.BAD_GATEWAY, + "SPARQL update failed (" + code + "): " + readResponseBody(response)); + } + return readResponseBody(response); + }); + } + } + + /** + * POST RDF/XML to the Virtuoso graph store. Uses digest auth (not preemptive basic auth), + * matching legacy {@code sparql.uploadSmallFile}. + */ + public void save(String graphUri, Path file) throws IOException { + String endpoint = ConfigUtil.get("graphStoreEndpoint").asText(); + String url = endpoint + + (endpoint.contains("?") ? "&" : "?") + + "graph-uri=" + URLEncoder.encode(graphUri, StandardCharsets.UTF_8); + + byte[] body = Files.readAllBytes(file); + try (CloseableHttpClient client = virtuosoDigestClient()) { + HttpPost post = new HttpPost(url); + post.setHeader(HttpHeaders.CONTENT_TYPE, "application/rdf+xml"); + post.setEntity(new ByteArrayEntity(body, ContentType.APPLICATION_XML)); + client.execute(post, response -> { + int code = response.getCode(); + if (code >= 300) { + throw new ResponseStatusException(HttpStatus.BAD_GATEWAY, + "Graph store upload failed (" + code + "): " + readResponseBody(response)); + } + return null; + }); + } + } + + /** Derives sparql-auth URL from config (explicit or inferred from sparqlEndpoint). */ + private String sparqlQueryUrl() throws IOException { + return ConfigUtil.get("sparqlEndpoint").asText() + + "?default-graph-uri={default-graph-uri}&query={query}&format=json&"; + } + + private String resolveGraphUri(String defaultGraphUri) throws IOException { + if (defaultGraphUri == null || defaultGraphUri.isBlank()) { + return ConfigUtil.get("defaultGraph").asText(); + } + return defaultGraphUri; + } + + private String sparqlAuthEndpoint() throws IOException { + JsonNode configured = ConfigUtil.get("sparqlAuthEndpoint"); + if (configured != null && !configured.isNull() && !configured.asText().isBlank()) { + return configured.asText(); + } + String base = ConfigUtil.get("sparqlEndpoint").asText(); + if (base.endsWith("-auth") || base.endsWith("-auth/")) { + return base; + } + return base.replaceAll("/sparql/?$", "/sparql-auth"); + } + + /** HttpClient configured for Virtuoso digest auth (waits for 401 challenge). */ + private static CloseableHttpClient virtuosoDigestClient() throws IOException { + BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials( + new AuthScope(null, -1), + new UsernamePasswordCredentials( + ConfigUtil.get("username").asText(), + ConfigUtil.get("password").asText().toCharArray())); + return HttpClients.custom() + .setDefaultCredentialsProvider(credsProvider) + .build(); + } + + private String readResponseBody(org.apache.hc.core5.http.ClassicHttpResponse response) + throws IOException { + if (response.getEntity() == null) { + return ""; + } + return new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8); + } } diff --git a/backend/src/main/java/com/synbiohub/sbh3/requests/SubmitRequest.java b/backend/src/main/java/com/synbiohub/sbh3/requests/SubmitRequest.java deleted file mode 100644 index 01cd1b75..00000000 --- a/backend/src/main/java/com/synbiohub/sbh3/requests/SubmitRequest.java +++ /dev/null @@ -1,25 +0,0 @@ -package com.synbiohub.sbh3.requests; - -import lombok.Builder; -import lombok.Getter; -import lombok.Setter; - -import java.io.File; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -@Getter -@Setter -@Builder -public class SubmitRequest { - private String name; - private String description; - private String id; - private String version; - - private String citations; - private List files; - private String overwriteMerge; - private String plugins; -} diff --git a/backend/src/main/java/com/synbiohub/sbh3/security/config/SecurityConfig.java b/backend/src/main/java/com/synbiohub/sbh3/security/config/SecurityConfig.java index 3f7db876..3ad2365f 100644 --- a/backend/src/main/java/com/synbiohub/sbh3/security/config/SecurityConfig.java +++ b/backend/src/main/java/com/synbiohub/sbh3/security/config/SecurityConfig.java @@ -70,7 +70,7 @@ public SecurityFilterChain securityFilterChain(HttpSecurity http) throws Excepti "/admin/theme", "/admin/registries", "/admin/logo", "/admin/plugins", "/v3/api-docs", "/v3/api-docs/**", "/swagger-ui/**", "/swagger-ui.html", "/openapi.yaml", "/error", - "/browse", "/rootCollections", "/root-collections", "/callPlugin", "/getSynBioHubVersion" + "/browse", "/rootCollections", "/root-collections", "/callPlugin", "/expose/**", "/getSynBioHubVersion" ).permitAll() .anyRequest().authenticated() .and() diff --git a/backend/src/main/java/com/synbiohub/sbh3/services/AttachmentService.java b/backend/src/main/java/com/synbiohub/sbh3/services/AttachmentService.java index 817e7e32..dd23ba3a 100644 --- a/backend/src/main/java/com/synbiohub/sbh3/services/AttachmentService.java +++ b/backend/src/main/java/com/synbiohub/sbh3/services/AttachmentService.java @@ -1,9 +1,153 @@ package com.synbiohub.sbh3.services; +import com.fasterxml.jackson.databind.JsonNode; +import com.synbiohub.sbh3.dao.SparqlService; +import com.synbiohub.sbh3.submit.SubmitPayload; +import com.synbiohub.sbh3.utils.ConfigUtil; +import com.synbiohub.sbh3.utils.FileUtil; +import com.synbiohub.sbh3.utils.StringUtil; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; +import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.UUID; +import java.util.zip.GZIPOutputStream; + @Service @RequiredArgsConstructor public class AttachmentService { + + private final SparqlService sparqlService; + private final SbolService sbolService; + + static final String UNKNOWN_ATTACHMENT_TYPE = "http://wiki.synbiohub.org/wiki/Terms/synbiohub#unknownAttachment"; + + // Code associated with submitting attachments + /** + * For each non-SBOL attachment: gzip+hash to {@code ./uploads/}, insert or update triples, + * then rewrite {@code file:filename} placeholders to real attachment URIs. + */ + public void uploadAttachments(SubmitPayload payload, String graphUri) throws IOException { + String collectionUri = payload.getCollectionUri(); + String baseUri = attachmentBaseUri(payload); + Map existingSources = sparqlService.loadAttachmentSources(collectionUri, graphUri); + + for (Path attachmentPath : payload.getAttachmentFiles()) { + String attachmentType = attachmentTypeFromExtension(attachmentPath); + if (attachmentType.toLowerCase(Locale.ROOT).contains("sbol")) { + continue; + } + + AttachmentService.UploadInfo upload = createAttachmentUpload(attachmentPath); + String filename = attachmentPath.getFileName().toString(); + String fileKey = "file:" + filename; // placeholder source URI from SBOL during readSbol + + if (existingSources.containsKey(fileKey)) { + sparqlService.updateAttachment(graphUri, existingSources.get(fileKey), upload.hash(), upload.size()); + continue; + } + + String attachmentUri = addAttachmentToTopLevel( + graphUri, + baseUri, + collectionUri, + filename, + upload.hash(), + upload.size(), + attachmentType, + payload.getCreatedBy()); + + sparqlService.uploadAttachment(Map.of( + "oldUri", fileKey, + "newUri", attachmentUri), graphUri); + } + } + + /** {@code databasePrefix + user//} — parent URI for new attachments. */ + private String attachmentBaseUri(SubmitPayload payload) throws IOException { + String databasePrefix = ConfigUtil.get("databasePrefix").asText(); + String username = URLEncoder.encode(payload.getCreatedBy(), StandardCharsets.UTF_8); + String collectionId = payload.getId(); + if (collectionId == null) { + collectionId = payload.collectionDisplayId().replace("_collection", ""); + } + return databasePrefix + "user/" + username + "/" + collectionId; + } + + /** Inserts attachment triples and links them to the root collection ({@code AttachUpload.sparql}). */ + private String addAttachmentToTopLevel(String graphUri, String baseUri, String topLevelUri, + String name, String uploadHash, long size, String attachmentType, + String owner) throws IOException { + String displayId = "attachment_" + UUID.randomUUID().toString().replace("-", ""); + String persistentIdentity = baseUri + "/" + displayId; + String version = "1"; + String attachmentUri = persistentIdentity + "/" + version; + String collectionUri = baseUri + baseUri.substring(baseUri.lastIndexOf('/')) + + "_collection/" + version; + String ownedBy = ConfigUtil.get("databasePrefix").asText() + + "user/" + URLEncoder.encode(owner, StandardCharsets.UTF_8); + + Map templateParams = new HashMap<>(); + templateParams.put("collectionUri", collectionUri); + templateParams.put("topLevel", topLevelUri); + templateParams.put("attachmentURI", attachmentUri); + templateParams.put("attachmentSource", attachmentUri + "/download"); + templateParams.put("persistentIdentity", persistentIdentity); + templateParams.put("displayId", StringUtil.sparqlStringLiteral(displayId)); + templateParams.put("version", StringUtil.sparqlStringLiteral(version)); + templateParams.put("name", StringUtil.sparqlStringLiteral(name)); + templateParams.put("description", StringUtil.sparqlStringLiteral("")); + templateParams.put("hash", StringUtil.sparqlStringLiteral(uploadHash)); + templateParams.put("size", StringUtil.sparqlStringLiteral(Long.toString(size))); + templateParams.put("type", attachmentType); + templateParams.put("ownedBy", ownedBy); + sparqlService.attachUpload(templateParams, graphUri, false); + return attachmentUri; + } + + private String attachmentTypeFromExtension(Path file) throws IOException { + String ext = FileUtil.fileExtension(file); + JsonNode mapping = ConfigUtil.get("fileExtensionToAttachmentType"); + if (mapping != null && mapping.has(ext)) { + return mapping.get(ext).asText(); + } + return UNKNOWN_ATTACHMENT_TYPE; + } + + /** + * Content-addressed storage: SHA-1 hash, gzip to {@code uploads//.gz}. + * Skips write when the file already exists (legacy {@code uploads.createUpload}). + */ + private AttachmentService.UploadInfo createAttachmentUpload(Path file) throws IOException { + byte[] raw = Files.readAllBytes(file); + try { + MessageDigest digest = MessageDigest.getInstance("SHA-1"); + String hash = sbolService.sha1Hex(digest.digest(raw)); + Path dest = uploadPath(hash); + if (!Files.exists(dest)) { + Files.createDirectories(dest.getParent()); + try (GZIPOutputStream gzip = new GZIPOutputStream(Files.newOutputStream(dest))) { + gzip.write(raw); + } + } + return new AttachmentService.UploadInfo(hash, raw.length); + } catch (NoSuchAlgorithmException e) { + throw new IOException("SHA-1 not available", e); + } + } + + private static Path uploadPath(String hash) { + return Path.of("uploads", hash.substring(0, 2), hash.substring(2) + ".gz"); + } + + private record UploadInfo(String hash, long size) {} } \ No newline at end of file diff --git a/backend/src/main/java/com/synbiohub/sbh3/services/CitationService.java b/backend/src/main/java/com/synbiohub/sbh3/services/CitationService.java new file mode 100644 index 00000000..41666ed3 --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/services/CitationService.java @@ -0,0 +1,32 @@ +package com.synbiohub.sbh3.services; + +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +@Service +public class CitationService { + + public List parseCitationPubmedIds(String citations) { + if (citations == null || citations.isBlank()) { + return new ArrayList<>(); + } + return Arrays.stream(citations.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .map(s -> { + try { + return Integer.parseInt(s); + } catch (NumberFormatException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "Invalid citation (expected PubMed ID): " + s); + } + }) + .collect(Collectors.toList()); + } +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/services/CollectionService.java b/backend/src/main/java/com/synbiohub/sbh3/services/CollectionService.java new file mode 100644 index 00000000..6b7e603f --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/services/CollectionService.java @@ -0,0 +1,280 @@ +package com.synbiohub.sbh3.services; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.synbiohub.sbh3.dao.SparqlService; +import com.synbiohub.sbh3.repo.SparqlRepository; +import com.synbiohub.sbh3.submit.SubmitPayload; +import com.synbiohub.sbh3.submit.SubmitRootCollectionMetadata; +import com.synbiohub.sbh3.utils.ConfigUtil; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.sbolstandard.core2.Collection; +import org.sbolstandard.core2.SBOLDocument; +import org.sbolstandard.core2.SBOLValidationException; +import org.sbolstandard.core2.TopLevel; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; +import com.fasterxml.jackson.databind.JsonNode; + +import java.io.IOException; +import java.net.URI; +import java.util.regex.Pattern; + +@Service +@Slf4j +@RequiredArgsConstructor +public class CollectionService { + + static final Pattern COLLECTION_ID_PATTERN = Pattern.compile("^[a-zA-Z0-9_]+$"); + private final ObjectMapper mapper; + private final UserService userService; + private final SparqlService sparqlService; + private final SparqlRepository sparqlRepository; + +// ------------------------------------------------------------------------- +// sanitize — resolve collection URI, check existence, enforce overwrite_merge +// ------------------------------------------------------------------------- + + /** + * Two submission modes: + *

+ * {@code overwrite_merge} modes (legacy submit form): + * 0 = new version, 1 = overwrite in place, 2 = merge, 3 = merge and replace remote duplicates. + */ + public void sanitize(SubmitPayload payload) throws IOException { + boolean submittingToExisting = payload.getCollectionUri() != null && payload.getId() == null; + boolean creatingNew = payload.getId() != null; + + if (!submittingToExisting && !creatingNew) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "Provide either rootCollections (existing collection) or id (new collection)."); + } + + if (submittingToExisting) { + if (payload.getOverwriteMerge() == null) { + payload.setOverwriteMerge("2"); + } + if (payload.getUploadedFilePath() == null) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "file is required when submitting into an existing collection."); + } + } else { + resolveNewCollectionUri(payload); + } + + String graphUri = graphUriForCollection(payload.getCollectionUri(), payload); + boolean metadataExists = collectionExists(payload.getCollectionUri(), graphUri); + payload.setExistingCollection(metadataExists + ? loadExistingCollection(payload.getCollectionUri(), graphUri) + : null); + + applyCollectionExistenceRules(payload, metadataExists, creatingNew); + } + + /** + * Resolves {@link SubmitPayload#getCollectionUri()} for a new collection from {@code id} and {@code version}. + */ + private void resolveNewCollectionUri(SubmitPayload payload) { + if (!COLLECTION_ID_PATTERN.matcher(payload.getId()).matches()) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "id must contain only alphanumeric characters and underscores."); + } + if (payload.getVersion() == null) { + payload.setVersion("1"); + } + payload.setCollectionUri(uriPrefix(payload) + payload.collectionDisplayId() + "/" + payload.getVersion()); + } + + /** + * Public collections live in {@code defaultGraph}; private user collections use the + * submitter's named graph URI. + */ + public String graphUriForCollection(String collectionUri, SubmitPayload payload) throws IOException { + String publicGraph = ConfigUtil.get("defaultGraph").asText(); + if (collectionUri.startsWith(publicGraph) || collectionUri.contains("/public/")) { + return publicGraph; + } + return userService.getUserByUsername(payload.getCreatedBy()).getGraphUri(); + } + + public boolean collectionExists(String collectionUri, String graphUri) throws IOException { + String query = "ASK { <" + collectionUri + "> a . }"; + String raw = sparqlRepository.getQuery(query, graphUri); + return mapper.readTree(raw).path("boolean").asBoolean(false); + } + + public SubmitRootCollectionMetadata loadExistingCollection(String collectionUri, String graphUri) + throws IOException { + String sparql = sparqlService.getTopLevelMetadataSPARQL(collectionUri); + String raw = sparqlRepository.getQuery(sparql, graphUri); + JsonNode bindings = mapper.readTree(raw).path("results").path("bindings"); + if (!bindings.isArray() || bindings.isEmpty()) { + return SubmitRootCollectionMetadata.builder().build(); + } + JsonNode row = bindings.get(0); + return SubmitRootCollectionMetadata.builder() + .name(textValue(row, "name")) + .description(textValue(row, "description")) + .displayId(textValue(row, "displayId")) + .version(textValue(row, "version")) + .build(); + } + + private static String textValue(JsonNode binding, String key) { + JsonNode node = binding.path(key); + return node.isMissingNode() || node.isNull() ? null : node.path("value").asText(null); + } + + /** + * Legacy submit rules after collection metadata presence is known. + *

+ * If metadata is missing, merge modes 2/3 are rejected. If metadata exists and mode is 0, + * the submission is rejected (id+version already taken). Modes 2/3 copy name/description + * from the store into the payload. + */ + public void applyCollectionExistenceRules(SubmitPayload payload, boolean metadataExists, + boolean creatingNew) { + if (!metadataExists) { + if (payload.getOverwriteMerge() != null) { + int mode = parseOverwriteMerge(payload.getOverwriteMerge()); + if (mode == 2 || mode == 3) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "Submission id and version do not exist"); + } + } + payload.setOverwriteMerge("0"); + if (creatingNew) { + requireNonBlank(payload.getName(), "name"); + requireNonBlank(payload.getDescription(), "description"); + } + return; + } + + int mode = parseOverwriteMerge( + payload.getOverwriteMerge() != null ? payload.getOverwriteMerge() : "0"); + + if (mode == 2 || mode == 3) { + fillPayloadFromExistingCollection(payload); + return; + } + if (mode == 1) { + return; + } + throw new ResponseStatusException(HttpStatus.CONFLICT, + "Submission id and version already in use"); + } + + private int parseOverwriteMerge(String raw) { + try { + return Integer.parseInt(raw.trim()); + } catch (NumberFormatException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid overwrite_merge: " + raw); + } + } + + private void requireNonBlank(String value, String field) { + if (value == null || value.isBlank()) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, field + " is required."); + } + } + + /** Copies stored collection metadata into the payload when merging (modes 2/3). */ + public void fillPayloadFromExistingCollection(SubmitPayload payload) { + SubmitRootCollectionMetadata existing = payload.getExistingCollection(); + if (existing == null) { + return; + } + if (existing.getName() != null) { + payload.setName(existing.getName()); + } + if (existing.getDescription() != null) { + payload.setDescription(existing.getDescription()); + } + if (existing.getVersion() != null) { + payload.setVersion(existing.getVersion()); + } + if (existing.getDisplayId() != null) { + String displayId = existing.getDisplayId(); + if (displayId.endsWith("_collection")) { + payload.setId(displayId.substring(0, displayId.length() - "_collection".length())); + } + } + } + + /** + * URI prefix for new objects in this submission + * (e.g. {@code https://synbiohub.org/user/alice/myproject/}). + */ + public String uriPrefix(SubmitPayload payload) { + String graphUri = userService.getUserByUsername(payload.getCreatedBy()).getGraphUri(); + if (payload.getCreatedBy() == null || graphUri == null || payload.getId() == null) { + if (payload.getCollectionUri() == null) { + return null; + } + return uriPrefixFromCollectionUri(payload, payload.getCollectionUri()); + } + String base = graphUri.endsWith("/") ? graphUri : graphUri + "/"; + return base + payload.getId() + "/"; + } + + /** + * Derives the object URI prefix from a root collection identity URI by removing + * {@code {displayId}/{version}}. + */ + public String uriPrefixFromCollectionUri(SubmitPayload payload, String collectionUri) { + int lastSlash = collectionUri.lastIndexOf('/'); + if (lastSlash <= 0) { + return null; + } + int prevSlash = collectionUri.lastIndexOf('/', lastSlash - 1); + if (prevSlash <= 0) { + return null; + } + return collectionUri.substring(0, prevSlash + 1); + } + + public Collection getRootCollection(SBOLDocument doc, String displayId, String version) throws SBOLValidationException { + org.sbolstandard.core2.Collection root = doc.getCollection(displayId, version); + if (root == null) { + root = doc.createCollection(displayId, version); + log.debug("New root collection: {}", root.getIdentity()); + } + return root; + } + + /** + * Adds all top-levels as collection members. Re-adds registry objects stripped earlier. + */ + public void populateCollectionMembership(SubmitPayload payload) { + String displayId = payload.collectionDisplayId(); + if (displayId == null) { + return; + } + String version = payload.getVersion() != null ? payload.getVersion() : "1"; + SBOLDocument doc = payload.getSbolDocument(); + org.sbolstandard.core2.Collection rootCollection = doc.getCollection(displayId, version); + if (rootCollection == null) { + return; + } + + URI rootIdentity = rootCollection.getIdentity(); + try { + for (TopLevel topLevel : doc.getTopLevels()) { + if (rootIdentity.equals(topLevel.getIdentity())) { + continue; + } + rootCollection.addMember(topLevel.getIdentity()); + } + for (String uri : payload.getUrisFoundInSynBioHub()) { + rootCollection.addMember(URI.create(uri)); + } + } catch (SBOLValidationException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "Failed to populate collection membership: " + e.getMessage()); + } + } +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/services/DownloadService.java b/backend/src/main/java/com/synbiohub/sbh3/services/DownloadService.java index 5276e45d..e06acbb9 100644 --- a/backend/src/main/java/com/synbiohub/sbh3/services/DownloadService.java +++ b/backend/src/main/java/com/synbiohub/sbh3/services/DownloadService.java @@ -6,6 +6,7 @@ import com.synbiohub.sbh3.dao.SparqlService; import com.synbiohub.sbh3.sparql.SPARQLQuery; import com.synbiohub.sbh3.utils.ConfigUtil; +import com.synbiohub.sbh3.utils.SbolWriterLegacyPrefixRewriter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.jena.datatypes.RDFDatatype; @@ -19,12 +20,19 @@ import org.apache.jena.riot.RDFFormat; import org.apache.jena.riot.RiotException; import org.apache.jena.vocabulary.RDF; +import org.sbolstandard.core2.Annotation; import org.sbolstandard.core2.ComponentDefinition; +import org.sbolstandard.core2.GenericTopLevel; +import org.sbolstandard.core2.SBOLConversionException; import org.sbolstandard.core2.SBOLDocument; import org.sbolstandard.core2.SBOLReader; +import org.sbolstandard.core2.SBOLValidationException; +import org.sbolstandard.core2.SBOLWriter; import org.sbolstandard.core2.Sequence; +import org.sbolstandard.core2.TopLevel; import org.springframework.stereotype.Service; +import javax.xml.namespace.QName; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -129,24 +137,91 @@ public String resolveLatestVersionedUri(String persistentIdentityUri) throws IOE } /** - * Recursive SBOL2 closure as RDF/XML with legacy SynBioHub1 namespace layout ({@code sbh}, {@code igem}, - * {@code dcterms}, etc.). The merged Jena model is written with {@link RDFFormat#RDFXML} and stable prefixes; - * {@link SBOLWriter} is not used because it emits generic {@code ns*} prefixes and splits iGEM IRIs into - * fragment namespaces that fail download regression tests. + * Recursive SBOL2 closure as RDF/XML matching SynBioHub1 {@code RDFToSBOLJob}: + * CONSTRUCT → Jena model → {@link SBOLReader} (via {@link RDFFormat#RDFXML_PLAIN} so + * {@code sbol:Sequence} stays link-style) → {@link #inlineNestedAnnotations} for + * {@code sbh:topLevel}-owned {@link GenericTopLevel}s → {@link SBOLWriter} → + * {@link SbolWriterLegacyPrefixRewriter} so root prefixes match SBH1 ({@code sbh}, + * {@code igem}, …) instead of {@code ns*} / fragment-split xmlns that fail download + * validator equality. */ public byte[] getSbol2RdfXmlBytes(String topLevelUri) throws IOException { Model model = getRecursiveModel(topLevelUri); if (model.isEmpty()) { return null; } - applyLegacySynbiohubRdfXmlPrefixes(model); - var out = new ByteArrayOutputStream(); - RDFDataMgr.write(out, model, RDFFormat.RDFXML); - return postProcessLegacySbolRdfXml(out.toByteArray()); + SBOLDocument doc = readModelAsSbolDocument(model); + if (doc == null) { + throw new IOException("Failed to read SBOL document for " + topLevelUri); + } + try { + prepareDocumentLikeSbh1(doc); + var out = new ByteArrayOutputStream(); + SBOLWriter.write(doc, out); + return SbolWriterLegacyPrefixRewriter.rewrite(out.toByteArray()); + } catch (SBOLValidationException | SBOLConversionException e) { + throw new IOException("Failed to serialize SBOL for " + topLevelUri, e); + } + } + + private static final String SBH_NS = "http://wiki.synbiohub.org/wiki/Terms/synbiohub#"; + + /** + * SynBioHub1 {@code RDFToSBOLJob} prep: touch Collection members, then restore nested + * annotation objects owned via {@code sbh:topLevel} before {@link SBOLWriter}. + */ + private static void prepareDocumentLikeSbh1(SBOLDocument doc) throws SBOLValidationException { + for (TopLevel topLevel : new ArrayList<>(doc.getTopLevels())) { + if (topLevel instanceof org.sbolstandard.core2.Collection) { + ((org.sbolstandard.core2.Collection) topLevel).getMembers(); + } + } + for (TopLevel topLevel : new ArrayList<>(doc.getTopLevels())) { + inlineNestedAnnotations(doc, topLevel, topLevel.getAnnotations()); + } + } + + /** + * Port of SynBioHub1 {@code RDFToSBOLJob.inlineNestedAnnotations}: when a URI annotation + * points at a {@link GenericTopLevel} whose {@code sbh:topLevel} is this top level, nest + * that GTL under the annotation and remove it as a top-level. Does not nest {@link Sequence} + * (Sequences are never GenericTopLevels with {@code sbh:topLevel}). + */ + private static void inlineNestedAnnotations(SBOLDocument doc, TopLevel topLevel, List annotations) + throws SBOLValidationException { + if (annotations == null) { + return; + } + for (Annotation annotation : annotations) { + if (!annotation.isURIValue()) { + continue; + } + URI genericTopLevelURI = annotation.getURIValue(); + GenericTopLevel genericTopLevel = doc.getGenericTopLevel(genericTopLevelURI); + if (genericTopLevel == null || genericTopLevelURI.equals(topLevel.getIdentity())) { + continue; + } + URI ownerTopLevelUri = null; + for (Annotation annotation2 : genericTopLevel.getAnnotations()) { + if (SBH_NS.equals(annotation2.getQName().getNamespaceURI()) + && "topLevel".equals(annotation2.getQName().getLocalPart()) + && annotation2.isURIValue()) { + ownerTopLevelUri = annotation2.getURIValue(); + break; + } + } + if (ownerTopLevelUri != null && ownerTopLevelUri.equals(topLevel.getIdentity())) { + annotation.setAnnotations(genericTopLevel.getAnnotations()); + annotation.setNestedIdentity(genericTopLevel.getIdentity()); + annotation.setNestedQName(genericTopLevel.getRDFType()); + doc.removeGenericTopLevel(genericTopLevel); + inlineNestedAnnotations(doc, topLevel, annotation.getAnnotations()); + } + } } /** - * Prefix map aligned with SynBioHub1 RDF/XML root for {@code /sbol} (see legacy public igem downloads). + * Prefix map aligned with SynBioHub1 RDF/XML root (used by {@code /sbolnr} Jena re-serialize). */ private static void applyLegacySynbiohubRdfXmlPrefixes(Model model) { List toClear = new ArrayList<>(); @@ -265,13 +340,13 @@ public String getMetadata(String uri) throws IOException { } /** - * Non-recursive SBOL RDF/XML for /sbolnr, using the same legacy namespace layout as {@link #getSbol2RdfXmlBytes}. + * Non-recursive SBOL RDF/XML for /sbolnr with legacy SynBioHub1-style namespace layout. *

* Virtuoso RDF/XML (and Turtle / N-Triples / SPARQL-JSON fallback from * {@link #readConstructResponseIntoModel}) is parsed into a Jena model, then written with - * {@link RDFFormat#RDFXML} and {@link #applyLegacySynbiohubRdfXmlPrefixes} so output matches SynBioHub1-style - * {@code sbh}/{@code igem} prefixes instead of auto-generated {@code ns*} names. If RDF/XML from Virtuoso cannot - * be parsed, the raw body is returned after {@link #postProcessLegacySbolRdfXml} only. + * {@link RDFFormat#RDFXML_PLAIN} and {@link #applyLegacySynbiohubRdfXmlPrefixes}. If RDF/XML from + * Virtuoso cannot be parsed, the raw body is returned after {@link #postProcessLegacySbolRdfXml} only. + * (Recursive {@code /sbol} uses {@link SBOLWriter} instead; see {@link #getSbol2RdfXmlBytes}.) */ public byte[] getSBOLNonRecursiveRdfXmlBytes(String uri) throws IOException { URI uriClass; @@ -295,7 +370,7 @@ public byte[] getSBOLNonRecursiveRdfXmlBytes(String uri) throws IOException { if (!model.isEmpty()) { applyLegacySynbiohubRdfXmlPrefixes(model); var out = new ByteArrayOutputStream(); - RDFDataMgr.write(out, model, RDFFormat.RDFXML); + RDFDataMgr.write(out, model, RDFFormat.RDFXML_PLAIN); return postProcessLegacySbolRdfXml(out.toByteArray()); } } catch (Exception e) { @@ -309,7 +384,7 @@ public byte[] getSBOLNonRecursiveRdfXmlBytes(String uri) throws IOException { readConstructResponseIntoModel(model, raw); applyLegacySynbiohubRdfXmlPrefixes(model); var out = new ByteArrayOutputStream(); - RDFDataMgr.write(out, model, RDFFormat.RDFXML); + RDFDataMgr.write(out, model, RDFFormat.RDFXML_PLAIN); return postProcessLegacySbolRdfXml(out.toByteArray()); } @@ -454,19 +529,34 @@ public Model getRecursiveModel(String uri) throws IOException { return model; } - public SBOLDocument getSBOLRecursive(String uri) throws IOException { - var results = getRecursiveModel(uri); - - // Write RDF Model to byte stream - SBOLDocument document = null; + /** + * Write the Jena model with {@link RDFFormat#RDFXML_PLAIN} (flat Identifiable links) and parse with + * {@link SBOLReader}. Plain RDF/XML avoids nested {@code sbol:Sequence} blocks that crash stock readers. + */ + private SBOLDocument readModelAsSbolDocument(Model model) { + if (model == null || model.isEmpty()) { + return null; + } var modelOutput = new ByteArrayOutputStream(); - RDFDataMgr.write(modelOutput, results, RDFFormat.RDFXML_PLAIN); - // Use libSBOLj to serialize and return + RDFDataMgr.write(modelOutput, model, RDFFormat.RDFXML_PLAIN); try { - document = SBOLReader.read(new ByteArrayInputStream(modelOutput.toByteArray())); + return SBOLReader.read(new ByteArrayInputStream(modelOutput.toByteArray())); } catch (Exception e) { - log.error("Error reading RDF to SBOL Document!"); - e.printStackTrace(); + log.error("Error reading RDF to SBOL Document!", e); + return null; + } + } + + public SBOLDocument getSBOLRecursive(String uri) throws IOException { + Model results = getRecursiveModel(uri); + SBOLDocument document = readModelAsSbolDocument(results); + if (document == null) { + return null; + } + try { + prepareDocumentLikeSbh1(document); + } catch (SBOLValidationException e) { + throw new IOException("Failed to inline nested annotations for " + uri, e); } return document; } diff --git a/backend/src/main/java/com/synbiohub/sbh3/services/PluginService.java b/backend/src/main/java/com/synbiohub/sbh3/services/PluginService.java index f28a6ba6..5ea3bc0b 100644 --- a/backend/src/main/java/com/synbiohub/sbh3/services/PluginService.java +++ b/backend/src/main/java/com/synbiohub/sbh3/services/PluginService.java @@ -243,4 +243,102 @@ public JsonNode buildType(String type) { node.put("type", type); return objectMapper.convertValue(node, JsonNode.class); } + + /** + * Resolves a submit plugin config entry by array index or by {@code name}. + */ + public String resolveSubmitPluginName(String pluginRef) throws IOException { + if (pluginRef == null || pluginRef.isBlank() || "default".equalsIgnoreCase(pluginRef)) { + return null; + } + JsonNode submitPlugins = ConfigUtil.get("plugins").get("submit"); + if (submitPlugins == null || !submitPlugins.isArray()) { + throw new IllegalStateException("No submit plugins are configured."); + } + if (pluginRef.matches("\\d+")) { + int index = Integer.parseInt(pluginRef); + if (index < 0 || index >= submitPlugins.size()) { + throw new IllegalArgumentException("Invalid submit plugin index: " + pluginRef); + } + String name = submitPlugins.get(index).path("name").asText(null); + if (name == null || name.isBlank()) { + throw new IllegalStateException("Submit plugin at index " + index + " has no name."); + } + return name; + } + for (JsonNode plugin : submitPlugins) { + if (pluginRef.equals(plugin.path("name").asText())) { + return pluginRef; + } + } + throw new IllegalArgumentException("Unknown submit plugin: " + pluginRef); + } + + public void checkSubmitPluginStatus(String pluginName) throws IOException { + String pluginUrl = requireSubmitPluginUrl(pluginName); + try { + HttpURLConnection connection = openConnection(pluginUrl + "status", "GET", 5000); + int code = connection.getResponseCode(); + if (code < 200 || code >= 300) { + throw new IOException("HTTP " + code); + } + } catch (IOException e) { + throw new IOException("The plugin " + pluginName + + " status endpoint is not responding. Check that the plugin is active and running.", e); + } + } + + public JsonNode evaluateSubmitPlugin(String pluginName, JsonNode evaluateManifest) throws IOException { + String pluginUrl = requireSubmitPluginUrl(pluginName); + try { + HttpURLConnection connection = openConnection(pluginUrl + "evaluate", "POST", 10000); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setRequestProperty("Accept", "application/json"); + connection.setDoOutput(true); + connection.getOutputStream().write(mapper.writeValueAsBytes(evaluateManifest)); + + int code = connection.getResponseCode(); + byte[] responseBody = readResponse( + code >= 400 ? connection.getErrorStream() : connection.getInputStream()); + if (code < 200 || code >= 300) { + throw new IOException(new String(responseBody, StandardCharsets.UTF_8)); + } + return mapper.readTree(responseBody); + } catch (IOException e) { + throw new IOException("The plugin " + pluginName + + " evaluate endpoint is not responding. Check that the plugin is active and running.", e); + } + } + + public byte[] runSubmitPlugin(String pluginName, JsonNode runRequest) throws IOException { + String pluginUrl = requireSubmitPluginUrl(pluginName); + try { + HttpURLConnection connection = openConnection(pluginUrl + "run", "POST", 120000); + connection.setRequestProperty("Content-Type", "application/json"); + connection.setDoOutput(true); + connection.getOutputStream().write(mapper.writeValueAsBytes(runRequest)); + + int code = connection.getResponseCode(); + byte[] responseBody = readResponse( + code >= 400 ? connection.getErrorStream() : connection.getInputStream()); + if (code < 200 || code >= 300) { + throw new IOException(new String(responseBody, StandardCharsets.UTF_8)); + } + return responseBody; + } catch (IOException e) { + throw new IOException("The plugin " + pluginName + + " run endpoint is not responding. Check that the plugin is active and running.", e); + } + } + + private String requireSubmitPluginUrl(String pluginName) throws IOException { + String pluginUrl = findPluginUrl(pluginName, "submit"); + if (pluginUrl == null) { + throw new IllegalArgumentException("Submit plugin not found: " + pluginName); + } + if (!pluginUrl.endsWith("/")) { + pluginUrl = pluginUrl + "/"; + } + return pluginUrl; + } } diff --git a/backend/src/main/java/com/synbiohub/sbh3/services/SbolService.java b/backend/src/main/java/com/synbiohub/sbh3/services/SbolService.java new file mode 100644 index 00000000..5b7701b7 --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/services/SbolService.java @@ -0,0 +1,558 @@ +package com.synbiohub.sbh3.services; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.synbiohub.sbh3.submit.SubmitPayload; +import com.synbiohub.sbh3.utils.ConfigUtil; +import com.synbiohub.sbh3.utils.FileUtil; +import com.synbiohub.sbh3.utils.StringUtil; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.sbolstandard.core2.*; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.server.ResponseStatusException; +import org.synbiohub.frontend.SynBioHubException; +import org.synbiohub.frontend.SynBioHubFrontend; + +import javax.xml.namespace.QName; +import java.io.*; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.*; +import java.util.regex.Pattern; + +@Service +@RequiredArgsConstructor +@Slf4j +public class SbolService { + + private final CollectionService collectionService; + private final UserService userService; + private final ObjectMapper objectMapper; + + /** + * Office uploads are passed through to SBOL validation (SynBioHub Excel plugin + * path). + */ + static final Set OFFICE_EXTENSIONS = Set.of( + "xlsx", "xls", "docx", "doc", "pptx", "ppt"); + static final QName SBH_MUTABLE_DESCRIPTION = new QName("http://wiki.synbiohub.org/wiki/Terms/synbiohub#", + "mutableDescription", "sbh"); + static final QName SBH_MUTABLE_NOTES = new QName("http://wiki.synbiohub.org/wiki/Terms/synbiohub#", "mutableNotes", + "sbh"); + static final QName SBH_MUTABLE_PROVENANCE = new QName("http://wiki.synbiohub.org/wiki/Terms/synbiohub#", + "mutableProvenance", "sbh"); + static final QName DC_CREATOR = new QName("http://purl.org/dc/elements/1.1/", "creator", "dc"); + static final QName DCTERMS_CREATED = new QName("http://purl.org/dc/terms/", "created", "dcterms"); + static final QName OBO_PUBMED = new QName("http://purl.obolibrary.org/obo/", "OBI_0001617", "obo"); + // SynBioHub-specific annotation QNames used when annotating submitted objects. + static final QName SBH_OWNED_BY = new QName("http://wiki.synbiohub.org/wiki/Terms/synbiohub#", "ownedBy", "sbh"); + static final QName SBH_TOP_LEVEL = new QName("http://wiki.synbiohub.org/wiki/Terms/synbiohub#", "topLevel", "sbh"); + /** + * Rewrites {@code img src="/user/.../"} paths in mutable HTML to the public + * collection path. + */ + static final Pattern MUTABLE_IMG_USER_PATH = Pattern.compile("img src=\\\"/user/[^/]*/[^/]*/"); + + // ------------------------------------------------------------------------- + // readSbol — validate/convert input, build composite SBOLDocument (PrepareSubmissionJob) + // ------------------------------------------------------------------------- + + /** + * SBOL preparation pipeline. Produces {@code payload.resultFilePath} (serialized XML). + */ + public void readSbol(SubmitPayload payload, Map webOfRegistriesMap, String ownedByUri, String authToken) throws IOException, SBOLValidationException { + setupReadSbol(payload); + classifySubmitInput(payload); + mergeValidatedSbolFiles(payload, webOfRegistriesMap); + prepareRootCollection(payload, ownedByUri); + mergeIntoExistingCollection(payload, authToken, webOfRegistriesMap); // modes 2/3 only + fixMutableAnnotations(payload); + collectionService.populateCollectionMembership(payload); + if (ConfigUtil.get("useSBOLExplorer").asBoolean(false)) { + incrementallyUpdateSbolExplorer(payload, ownedByUri); + } + finishReadSbol(payload); + } + + /** + * Initializes per-submission lists and an empty composite SBOLDocument. + */ + private void setupReadSbol(SubmitPayload payload) { + payload.setAttachmentFiles(new ArrayList<>()); + payload.setSbolFiles(new ArrayList<>()); + payload.setUrisFoundInSynBioHub(new HashSet<>()); + + SBOLDocument doc = new SBOLDocument(); + String uriPrefix = collectionService.uriPrefix(payload); + if (uriPrefix != null) { + doc.setDefaultURIprefix(uriPrefix); + } + payload.setSbolDocument(doc); + payload.setReadSbolStartedAtMs(System.currentTimeMillis()); + log.debug("readSbol setup: uriPrefix={}", uriPrefix); + } + + // --- readSbol: classify uploaded file(s) into sbolFiles / attachmentFiles --- + + /** + * Unpacks or classifies the uploaded file. + */ + private void classifySubmitInput(SubmitPayload payload) throws IOException { + if (payload.getUploadedFilePath() == null) { + return; + } + Path input = Path.of(payload.getUploadedFilePath()); + if (!Files.isRegularFile(input)) { + return; + } + + String ext = FileUtil.fileExtension(input); + if (OFFICE_EXTENSIONS.contains(ext)) { + payload.getSbolFiles().add(input); + return; + } + if ("omex".equals(ext) || FileUtil.isCombineArchive(input)) { + FileUtil.extractSubmitArchive(input, payload, true); + return; + } + if ("zip".equals(ext)) { + // Generic zip: unpack and classify; not flagged as COMBINE. + FileUtil.extractSubmitArchive(input, payload, false); + return; + } + + FileUtil.classifySubmitFile(input, payload); + } + + /** + * Validates each SBOL input file, strips registry objects, rewrites URIs to the submission + * prefix, and merges into the composite document on {@code payload}. + */ + private void mergeValidatedSbolFiles(SubmitPayload payload, Map webOfRegistriesMap) throws IOException { + String databasePrefix = ConfigUtil.get("databasePrefix").asText(); + String uriPrefix = collectionService.uriPrefix(payload); + String version = payload.getVersion() != null ? payload.getVersion() : "1"; + + boolean requireComplete = ConfigUtil.get("requireComplete").asBoolean(false); + boolean requireCompliant = ConfigUtil.get("requireCompliant").asBoolean(false); + boolean enforceBestPractices = ConfigUtil.get("requireBestPractice").asBoolean(false); + List registryPrefixes = new ArrayList<>(webOfRegistriesMap.keySet()); + + StringBuilder readLog = new StringBuilder(); + SBOLDocument composite = payload.getSbolDocument(); + + for (Path file : payload.getSbolFiles()) { + String filename = file.toString(); + String defaultDisplayId = StringUtil.fixDisplayId(StringUtil.displayIdFromFilename(filename)); + + ByteArrayOutputStream logOut = new ByteArrayOutputStream(); + ByteArrayOutputStream errorOut = new ByteArrayOutputStream(); + SBOLDocument individual = SBOLValidate.validate( + new PrintStream(logOut), + new PrintStream(errorOut), + filename, + "http://dummy.org/", + defaultDisplayId, + requireComplete, + requireCompliant, + enforceBestPractices, + false, + "1", + true, + "", + "", + filename, + "", + false, + false, + false, + false, + false, + null, + false, + true, + false); + + readLog.append("[").append(filename).append(" log] \n") + .append(logOut.toString(StandardCharsets.UTF_8)).append("\n"); + + String errorLog = errorOut.toString(StandardCharsets.UTF_8); + if (errorLog.startsWith("File is empty")) { + // Empty Office/plugin placeholder — treat as blank document. + individual = new SBOLDocument(); + errorLog = ""; + } else if (!errorLog.isEmpty()) { + failReadSbol(readLog.toString(), errorLog); + } + + // Registry URIs are removed from the document but remembered for collection membership. + stripRegistryTopLevels(individual, registryPrefixes, payload); + + try { + individual = rewriteIndividualUris(individual, uriPrefix, version, databasePrefix); + composite.createCopy(individual); + } catch (Exception e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + failReadSbol(readLog.toString(), sw.toString()); + } + } + if (!readLog.isEmpty()) { + log.debug("readSbol validate log:\n{}", readLog); + } + } + + /** + * Creates or updates the root {@code Collection} and annotates every top-level with + * ownedBy, topLevel, citations, and creator metadata. + */ + private void prepareRootCollection (SubmitPayload payload, String ownedByUri) throws IOException, SBOLValidationException { + SBOLDocument doc = payload.getSbolDocument(); + String version = payload.getVersion() != null ? payload.getVersion() : "1"; + + String displayId = payload.collectionDisplayId(); + if (displayId == null) { + return; + } + org.sbolstandard.core2.Collection root = collectionService.getRootCollection(doc, displayId, version); + applySubmitRootCollection(payload, doc, root, ownedByUri); + } + + /** + * Merge modes 2/3: for objects whose URIs already exist in a configured registry, + * fetch from remote SynBioHub and either reject (different content), replace (mode 3), + * or keep as collection members (identical content). + */ + private void mergeIntoExistingCollection(SubmitPayload payload, String authToken, Map registries) throws IOException { + String mode = payload.getOverwriteMerge(); + if (mode == null || "0".equals(mode) || "1".equals(mode)) { + return; + } + + SBOLDocument doc = payload.getSbolDocument(); + String rootCollectionIdentity = payload.getCollectionUri(); + String version = payload.getVersion() != null ? payload.getVersion() : "1"; + org.sbolstandard.core2.Collection rootCollection = null; + if (payload.collectionDisplayId() != null) { + rootCollection = doc.getCollection(payload.collectionDisplayId(), version); + } + + String shareLinkSalt = ConfigUtil.get("shareLinkSalt").asText(""); + + for (TopLevel topLevel : new ArrayList<>(doc.getTopLevels())) { + String identity = topLevel.getIdentity().toString(); + + if (rootCollectionIdentity != null && identity.equals(rootCollectionIdentity)) { + // Existing collection metadata is preserved from the store during merge. + stripExistingRootCollectionMetadata(topLevel); + continue; + } + + for (Map.Entry entry : registries.entrySet()) { + String registry = entry.getKey(); + if (!identity.startsWith(registry)) { + continue; + } + + String fetchUri = registryFetchUri(identity, registry, shareLinkSalt); + SynBioHubFrontend sbh = new SynBioHubFrontend(entry.getValue(), registry); + if (authToken != null) { + sbh.setUser(authToken); + } + + SBOLDocument remoteDoc; + try { + remoteDoc = sbh.getSBOL(URI.create(fetchUri)); + } catch (SynBioHubException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage()); + } + + if (remoteDoc == null) { + break; + } + + TopLevel remote = remoteDoc.getTopLevel(topLevel.getIdentity()); + if (remote == null) { + break; + } + + if (!topLevel.equals(remote)) { + if ("3".equals(mode)) { + try { + sbh.replaceSBOL(URI.create(fetchUri)); + } catch (SynBioHubException e) { + log.warn("replaceSBOL failed for {}", identity, e); + } + } else { + throw new ResponseStatusException(HttpStatus.CONFLICT, + "Submission terminated.\nA submission with this id already exists," + + " and it includes an object: " + identity + + " that is already in this repository and has different content"); + } + } else { + log.debug("Found duplicate registry object, keeping as member: {}", identity); + try { + doc.removeTopLevel(topLevel); + if (rootCollection != null) { + rootCollection.addMember(topLevel.getIdentity()); + } + } catch (SBOLValidationException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "Failed to merge duplicate object: " + e.getMessage()); + } + } + break; + } + } + } + + /** Rewrites user-scoped image paths in sbh:mutable* HTML to public collection URLs. */ + private void fixMutableAnnotations(SubmitPayload payload) { + String displayId = payload.collectionDisplayId(); + if (displayId == null) { + return; + } + String publicPrefix = "img src=\"/public/" + displayId.replace("_collection", "") + "/"; + SBOLDocument doc = payload.getSbolDocument(); + for (TopLevel topLevel : doc.getTopLevels()) { + rewriteMutableAnnotation(topLevel, SBH_MUTABLE_DESCRIPTION, publicPrefix); + rewriteMutableAnnotation(topLevel, SBH_MUTABLE_NOTES, publicPrefix); + rewriteMutableAnnotation(topLevel, SBH_MUTABLE_PROVENANCE, publicPrefix); + } + } + + /** Notifies SBOL Explorer of added top-levels after a successful submit (when enabled). */ + private void incrementallyUpdateSbolExplorer(SubmitPayload payload, String ownedByUri) throws IOException { + ObjectNode body = objectMapper.createObjectNode(); + body.set("partsToRemove", objectMapper.createArrayNode()); + ArrayNode partsToAdd = objectMapper.createArrayNode(); + for (TopLevel topLevel : payload.getSbolDocument().getTopLevels()) { + ObjectNode part = objectMapper.createObjectNode(); + part.put("subject", topLevel.getIdentity().toString()); + part.put("displayId", topLevel.getDisplayId()); + part.put("version", topLevel.getVersion()); + part.put("name", topLevel.getName()); + part.put("description", topLevel.getDescription()); + part.put("type", "TODO"); + part.put("graph", ownedByUri); + partsToAdd.add(part); + } + body.set("partsToAdd", partsToAdd); + + String endpoint = ConfigUtil.get("SBOLExplorerEndpoint").asText(); + if (!endpoint.endsWith("/")) { + endpoint += "/"; + } + HttpHeaders headers = new HttpHeaders(); + headers.setContentType(MediaType.APPLICATION_JSON); + try { + new RestTemplate().postForEntity(endpoint + "incrementalupdate", + new HttpEntity<>(body, headers), String.class); + } catch (Exception e) { + log.warn("SBOLExplorer /incrementalupdate failed", e); + } + } + + /** Writes the composite document to a temp XML file ({@code resultFilename} in legacy Node). */ + private void finishReadSbol(SubmitPayload payload) throws IOException { + Path resultFile = Files.createTempFile("sbh_convert_validate", ".xml"); + try { + SBOLWriter.write(payload.getSbolDocument(), resultFile.toFile()); + } catch (SBOLConversionException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "Failed to write SBOL output: " + e.getMessage()); + } + payload.setResultFilePath(resultFile.toAbsolutePath().toString()); + + long started = payload.getReadSbolStartedAtMs() != null + ? payload.getReadSbolStartedAtMs() + : System.currentTimeMillis(); + log.info("readSbol total time (sec): {}", (System.currentTimeMillis() - started) / 1000.0); + } + + private void failReadSbol(String readLog, String errorLog) { + log.debug("readSbol failed:\n{}\n{}", readLog, errorLog); + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, errorLog); + } + + /** + * Removes top-levels that belong to a configured web-of-registries prefix. + * Their URIs are recorded so they can be re-added as collection members later. + */ + private void stripRegistryTopLevels(SBOLDocument individual, + List registryPrefixes, + SubmitPayload payload) { + for (TopLevel topLevel : new ArrayList<>(individual.getTopLevels())) { + String identity = topLevel.getIdentity().toString(); + for (String registry : registryPrefixes) { + if (identity.startsWith(registry)) { + log.debug("Found and removed registry object: {}", identity); + payload.getUrisFoundInSynBioHub().add(identity); + try { + individual.removeTopLevel(topLevel); + } catch (SBOLValidationException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "Failed to remove registry object: " + e.getMessage()); + } + break; + } + } + } + } + + /** Assigns submission URI prefix/version and fixes attachment download sources after rewrite. */ + private SBOLDocument rewriteIndividualUris(SBOLDocument individual, String uriPrefix, String version, + String databasePrefix) throws SBOLValidationException { + individual.setDefaultURIprefix("http://dummy.org/"); + if (uriPrefix == null) { + return individual; + } + if (individual.getTopLevels().isEmpty()) { + individual.setDefaultURIprefix(uriPrefix); + return individual; + } + individual = individual.changeURIPrefixVersion(uriPrefix, null, version); + individual.setDefaultURIprefix(uriPrefix); + for (Attachment attachment : individual.getAttachments()) { + String source = attachment.getSource().toString(); + if (source.startsWith(databasePrefix) && source.endsWith("/download")) { + attachment.setSource(URI.create(attachment.getIdentity().toString() + "/download")); + } + } + return individual; + } + + /** Sets name, description, creator, citations, ownedBy, and topLevel on all identified objects. */ + private void applySubmitRootCollection(SubmitPayload payload, SBOLDocument doc, + org.sbolstandard.core2.Collection root, + String ownedByUri) { + try { + String creator = userService.getUserByUsername(payload.getCreatedBy()).getName(); + if (creator != null) { + root.createAnnotation(DC_CREATOR, creator); + } + root.createAnnotation(DCTERMS_CREATED, + ZonedDateTime.now().format(DateTimeFormatter.ISO_INSTANT)); + if (payload.getName() != null) { + root.setName(payload.getName()); + } + if (payload.getDescription() != null) { + root.setDescription(payload.getDescription()); + } + new IdentifiedVisitor() { + @Override + public void visit(Identified identified, TopLevel topLevel) { + try { + propagateTopLevelInNestedAnnotations(topLevel, identified.getAnnotations()); + for (Integer pubmedId : payload.getCitationPubmedIds()) { + identified.createAnnotation(OBO_PUBMED, String.valueOf(pubmedId)); + } + Annotation ownedBy = identified.getAnnotation(SBH_OWNED_BY); + if (ownedBy != null) { + identified.removeAnnotation(ownedBy); + } + identified.createAnnotation(SBH_OWNED_BY, new URI(ownedByUri)); + Annotation topLevelAnn = identified.getAnnotation(SBH_TOP_LEVEL); + if (topLevelAnn != null) { + identified.removeAnnotation(topLevelAnn); + } + identified.createAnnotation(SBH_TOP_LEVEL, topLevel.getIdentity()); + } catch (SBOLValidationException | URISyntaxException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "Failed to annotate object: " + e.getMessage()); + } + } + }.visitDocument(doc); + } catch (SBOLValidationException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "Failed to prepare root collection: " + e.getMessage()); + } + } + + /** Clears name/description/creator on the existing root collection so merge keeps store metadata. */ + private void stripExistingRootCollectionMetadata(TopLevel topLevel) { + topLevel.unsetDescription(); + topLevel.unsetName(); + topLevel.clearWasDerivedFroms(); + Annotation creator = topLevel.getAnnotation(DC_CREATOR); + if (creator != null) { + topLevel.removeAnnotation(creator); + } + } + + /** Share token for fetching private user objects from remote registries during merge. */ + private String registryFetchUri(String identity, String registry, String shareLinkSalt) { + if (identity.startsWith(registry + "/user/")) { + return identity + "/" + privateShareToken(identity, shareLinkSalt) + "/share"; + } + return identity; + } + + private void rewriteMutableAnnotation(TopLevel topLevel, QName qname, String publicPrefix) { + Annotation annotation = topLevel.getAnnotation(qname); + if (annotation == null || !annotation.isStringValue()) { + return; + } + String value = annotation.getStringValue(); + String updated = MUTABLE_IMG_USER_PATH.matcher(value).replaceAll(publicPrefix); + if (value.equals(updated)) { + return; + } + try { + topLevel.removeAnnotation(annotation); + topLevel.createAnnotation(qname, updated); + } catch (SBOLValidationException e) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "Failed to rewrite mutable annotation: " + e.getMessage()); + } + } + + private String privateShareToken(String identity, String shareLinkSalt) { + try { + MessageDigest md = MessageDigest.getInstance("SHA-1"); + String innerHex = sha1Hex(md.digest((identity + "/edit").getBytes(StandardCharsets.UTF_8))); + md.reset(); + String payload = "synbiohub_" + innerHex + (shareLinkSalt == null ? "" : shareLinkSalt); + return sha1Hex(md.digest(payload.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-1 not available", e); + } + } + + public String sha1Hex(byte[] digest) { + StringBuilder sb = new StringBuilder(digest.length * 2); + for (byte b : digest) { + sb.append(String.format(Locale.US, "%02x", b & 0xff)); + } + return sb.toString(); + } + + /** Ensures sbh:topLevel is set on nested annotation trees (e.g. mutable HTML blocks). */ + private void propagateTopLevelInNestedAnnotations(TopLevel topLevel, List annotations) + throws SBOLValidationException { + for (Annotation annotation : annotations) { + if (!annotation.isNestedAnnotations()) { + continue; + } + List nested = new ArrayList<>(annotation.getAnnotations()); + propagateTopLevelInNestedAnnotations(topLevel, nested); + nested.removeIf(a -> SBH_TOP_LEVEL.equals(a.getQName())); + nested.add(new Annotation(SBH_TOP_LEVEL, topLevel.getIdentity())); + annotation.setAnnotations(nested); + } + } +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/services/SubmitService.java b/backend/src/main/java/com/synbiohub/sbh3/services/SubmitService.java index 75d2551f..4c30fa7a 100644 --- a/backend/src/main/java/com/synbiohub/sbh3/services/SubmitService.java +++ b/backend/src/main/java/com/synbiohub/sbh3/services/SubmitService.java @@ -1,155 +1,194 @@ package com.synbiohub.sbh3.services; -import com.synbiohub.sbh3.dto.SubmissionDTO; -import com.synbiohub.sbh3.requests.SubmitRequest; -import com.synbiohub.sbh3.utils.ObjectMapperUtils; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.synbiohub.sbh3.dao.SparqlService; +import com.synbiohub.sbh3.security.model.AuthCodes; +import com.synbiohub.sbh3.security.repo.AuthRepository; +import com.synbiohub.sbh3.submit.SubmitPayload; +import com.synbiohub.sbh3.submit.SubmitPluginService; +import com.synbiohub.sbh3.utils.ConfigUtil; +import com.synbiohub.sbh3.utils.FileUtil; import lombok.RequiredArgsConstructor; -import org.sbolstandard.core2.*; +import lombok.extern.slf4j.Slf4j; +import org.sbolstandard.core2.SBOLValidationException; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.stereotype.Service; -import org.apache.commons.io.FilenameUtils; - -import java.io.File; -import java.util.*; -import java.util.Collection; - +import org.springframework.web.client.RestTemplate; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.server.ResponseStatusException; + +import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Orchestrates {@code POST /submit}: multipart form → Virtuoso graph upload. + *

+ * Pipeline (legacy {@code lib/views/submit.js} + {@code PrepareSubmissionJob}): + * 

+ *   parse → sanitize → submit plugin → readSbol → prepare → upload
+ *
+ * State is carried in a single mutable {@link SubmitPayload}. + */ @Service @RequiredArgsConstructor - +@Slf4j public class SubmitService { - public SubmissionDTO createSubmissionDTO(SubmitRequest submitRequest) throws SBOLValidationException { - if (Integer.parseInt(submitRequest.getOverwriteMerge()) <= 1) { - return createSubmissionWithNewCollection(submitRequest); - } else { - return createSubmissionWithoutNewCollection(submitRequest); - } - } - - public SubmissionDTO createSubmissionWithNewCollection(SubmitRequest submitRequest) throws SBOLValidationException { - String id = parseID(submitRequest); - String name = parseName(submitRequest); - String description = parseDescription(submitRequest); - int version = parseVersion(submitRequest); - List citations = parseCitations(submitRequest); - Collection allFiles = new ArrayList<>(); - Collection attachmentFiles = new ArrayList<>(); - String fileType = FilenameUtils.getExtension(submitRequest.getFiles().get(0).getAbsolutePath()); // currently, only one file can be submitted at a time, may change in the future - if (fileType.equalsIgnoreCase(".xlsx")) { - //send to excel2sbol plugin - } - if ((fileType.equalsIgnoreCase(".omex")) || fileType.equals(".zip")) { - // parse out files and keep sbol ones but send other ones to become attachment files - } - - allFiles.forEach(file -> { - if (checkForSBOL(file)) { - verifyFile(file); - } else { - attachmentFiles.add(file); - allFiles.remove(file); - } - }); - SBOLDocument doc = new SBOLDocument(); - String URIPrefix = createURIPrefix(""); //TODO: get URI prefix from submitRequest - doc.setDefaultURIprefix(URIPrefix); - org.sbolstandard.core2.Collection rootCollection = null; - try { - rootCollection = doc.createCollection(URIPrefix, id, String.valueOf(version)); //not sure what to do with rootCollection here - } catch (Exception e) { - System.out.println("This exact collection already exists."); - return null; - } - updateAnnotations(doc); - updateSBOLExplorer(doc); - - if (submitRequest.getOverwriteMerge().equalsIgnoreCase("1")) { - //TODO get existing collection and members to delete for overwrite for removal later + private final UserService userService; + private final AuthRepository authRepository; + private final SearchService searchService; + private final SubmitPluginService submitPluginService; + private final CitationService citationService; + private final CollectionService collectionService; + private final SbolService sbolService; + private final SparqlService sparqlService; + private final AttachmentService attachmentService; + private final ObjectMapper objectMapper; + + /** + * Main submit entry point. Each step mutates {@code payload} in place. + */ + public ResponseEntity submit(SubmitPayload allParams, MultipartFile file) throws IOException, SBOLValidationException { + SubmitPayload payload = parse(allParams, file); + collectionService.sanitize(payload); + submitPluginService.applySubmitPlugin(payload); // optional transform of uploaded file + sbolService.readSbol(payload, webOfRegistriesMap(), + ConfigUtil.get("databasePrefix").asText() + "user/" + payload.getCreatedBy(), + resolveUserAuthToken(payload)); + prepare(payload); // only runs for overwrite_merge == 1 + upload(payload); + return successResponse(); + } + + // ------------------------------------------------------------------------- + // parse — multipart form fields + temp file + authenticated user context + // ------------------------------------------------------------------------- + + /** + * Reads form parameters and persists the uploaded file to a temp path. + * Does not validate business rules. + */ + private SubmitPayload parse(SubmitPayload payload, MultipartFile file) throws IOException { + //TODO: why are we creating a temp file here? + if (file != null && !file.isEmpty()) { + String suffix = FileUtil.sanitizeFilename(file.getOriginalFilename()); + Path temp = Files.createTempFile("sbh-submit-", "-" + suffix); + file.transferTo(temp); + payload.setUploadedFilePath(temp.toAbsolutePath().toString()); } - SubmissionDTO submissionDTO = ObjectMapperUtils.createNewSubmission(name, description, id, version, citations, allFiles, Integer.parseInt(submitRequest.getOverwriteMerge()), new ArrayList<>(), attachmentFiles, doc); - uploadToVirtuoso(submissionDTO); - uploadAttachmentFiles(submissionDTO); - - // TODO remove existing collection and members for overwrite - - return submissionDTO; // this will change, - } - - public SubmissionDTO createSubmissionWithoutNewCollection(SubmitRequest submitRequest) { - return null; - } - - public SubmitRequest createSubmitRequest(Map allParams) { //maps the request params to the fields listed above - return null; - } - - private String parseID(SubmitRequest submitRequest) { - return submitRequest.getId(); - } - - private String parseName(SubmitRequest submitRequest) { - return submitRequest.getName(); - } - private String parseDescription(SubmitRequest submitRequest) { - return submitRequest.getDescription(); + Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + payload.setCreatedBy(auth.getName()); + payload.setCitationPubmedIds(citationService.parseCitationPubmedIds(payload.getCitations())); + return payload; } - private int parseVersion(SubmitRequest submitRequest) { - return Integer.parseInt(submitRequest.getVersion()); + private String resolveUserAuthToken(SubmitPayload payload) { + // TODO: create authService + return authRepository.findByName(payload.getCreatedBy()) + .map(AuthCodes::getAuth) + .orElse(null); } - private List parseCitations(SubmitRequest submitRequest) { - return handleCitations(submitRequest.getCitations()); - } - - - public List handleCitations(String citations) { //change the inputted string of citations into a list of citations - return null; - } - - - public void verifyFile(File file) { - + private static Map webOfRegistriesMap() throws IOException { + JsonNode node = ConfigUtil.get("webOfRegistries"); + Map map = new LinkedHashMap<>(); + if (node != null && node.isObject()) { + node.fields().forEachRemaining(entry -> map.put(entry.getKey(), entry.getValue().asText())); + } + return map; } - public Boolean checkForSBOL(File file) { - // if is in sbol, then return true - // else - // if file is not in sbol, check to see if it can be converted - // it can be converted, then go to convertToSBOL - // else, return false - return true; - } + // ------------------------------------------------------------------------- + // prepare — delete existing triples before overwrite (overwrite_merge == 1) + // ------------------------------------------------------------------------- - public void convertToSBOL() { + /** + * Overwrite mode: stagger-delete all objects under the collection URI prefix, then remove + * the collection itself. Mirrors {@code submit.js} after prepareSubmission succeeds. + */ + private void prepare(SubmitPayload payload) throws IOException { + if (!"1".equals(payload.getOverwriteMerge())) { + return; + } + String collectionUri = payload.getCollectionUri(); + String uriPrefix = collectionService.uriPrefixFromCollectionUri(payload, collectionUri); + if (collectionUri == null || uriPrefix == null) { + return; + } - } + String graphUri = collectionService.graphUriForCollection(collectionUri, payload); + Map templateParams = Map.of( + "collection", collectionUri, + "uriPrefix", uriPrefix); - public void SBOL2ToSBOL3() { - // to be implemented in the future - } + log.debug("prepare overwrite: removing {}", uriPrefix); + sparqlService.deleteCollection(templateParams, graphUri); + sparqlService.delete(Map.of("uri", collectionUri), graphUri); - public String createURIPrefix(String str) { - return ""; + if (ConfigUtil.get("useSBOLExplorer").asBoolean(false)) { + notifyExplorerRemoveCollection(collectionUri, uriPrefix); + } } - public void updateAnnotations(SBOLDocument sbolDocument) { - + private void notifyExplorerRemoveCollection(String collectionUri, String uriPrefix) throws IOException { + // TODO: refactor this method in the future, currently not being called + String endpoint = ConfigUtil.get("SBOLExplorerEndpoint").asText(); + if (!endpoint.endsWith("/")) { + endpoint += "/"; + } + String url = endpoint + "incrementalremovecollection?subject=" + + URLEncoder.encode(collectionUri, StandardCharsets.UTF_8) + + "&uriPrefix=" + URLEncoder.encode(uriPrefix, StandardCharsets.UTF_8); + try { + new RestTemplate().getForEntity(url, String.class); + } catch (Exception e) { + log.warn("SBOLExplorer incrementalremovecollection failed", e); + } } - public void updateSBOLExplorer(SBOLDocument sbolDocument) { - - } + // ------------------------------------------------------------------------- + // upload — graph store RDF POST + attachment files + temp cleanup + // ------------------------------------------------------------------------- - public void checkObject(SBOLDocument sbolDocument) { //check if the submission object already exists in the collection, only for OM = 2,3 + /** + * Posts prepared SBOL XML to Virtuoso, stores attachment binaries, rewrites {@code file:} + * sources in the graph, then deletes temp files. + */ + private void upload(SubmitPayload payload) throws IOException { + String graphUri = collectionService.graphUriForCollection(payload.getCollectionUri(), payload); + String resultPath = payload.getResultFilePath(); + if (resultPath == null || resultPath.isBlank()) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "No prepared SBOL file to upload."); + } - } + log.debug("upload: posting RDF to graph {}", graphUri); + sparqlService.uploadGraphStore(graphUri, Path.of(resultPath)); - public void uploadToVirtuoso(SubmissionDTO submissionDTO) { + if (!payload.getAttachmentFiles().isEmpty()) { + attachmentService.uploadAttachments(payload, graphUri); + } + FileUtil.cleanupSubmitTemp(payload); } - public void uploadAttachmentFiles(SubmissionDTO submissionDTO) { + // ------------------------------------------------------------------------- + // response / small helpers + // ------------------------------------------------------------------------- + private ResponseEntity successResponse() { + return ResponseEntity.ok() + .contentType(MediaType.parseMediaType("text/plain; charset=UTF-8")) + .body("Submission successful"); } -} +} \ No newline at end of file diff --git a/backend/src/main/java/com/synbiohub/sbh3/services/UserService.java b/backend/src/main/java/com/synbiohub/sbh3/services/UserService.java index 621a661d..b0931478 100644 --- a/backend/src/main/java/com/synbiohub/sbh3/services/UserService.java +++ b/backend/src/main/java/com/synbiohub/sbh3/services/UserService.java @@ -15,6 +15,7 @@ import com.synbiohub.sbh3.security.model.User; import com.synbiohub.sbh3.security.repo.AuthRepository; import com.synbiohub.sbh3.security.repo.UserRepository; +import com.synbiohub.sbh3.repo.SparqlRepository; import com.synbiohub.sbh3.sparql.SPARQLQuery; import com.synbiohub.sbh3.utils.ConfigUtil; import jakarta.servlet.http.HttpServletRequest; @@ -272,7 +273,7 @@ private void sendPasswordResetEmail(String apiKey, String fromAddress, String to payload.put("subject", "SynBioHub password reset"); payload.put("text", body); - ResponseEntity response = sparqlRepository.postJson( + ResponseEntity response = sparqlRepository.postJson( "https://api.resend.com/emails", payload, String.class, headers); if (!response.getStatusCode().is2xxSuccessful()) { throw new IOException("Resend rejected password reset email with status " diff --git a/backend/src/main/java/com/synbiohub/sbh3/sparql/AttachUpload.sparql b/backend/src/main/java/com/synbiohub/sbh3/sparql/AttachUpload.sparql new file mode 100644 index 00000000..afd8dcf7 --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/sparql/AttachUpload.sparql @@ -0,0 +1,23 @@ +PREFIX dcterms: +PREFIX sbh: +PREFIX sbol: + +INSERT { + <$topLevel> sbol:attachment <$attachmentURI> . + + <$collectionUri> sbol:member <$attachmentURI> . + + <$attachmentURI> a sbol:Attachment ; + dcterms:title $name ; + dcterms:description $description ; + sbol:displayId $displayId ; + sbol:persistentIdentity <$persistentIdentity> ; + sbol:version $version ; + sbh:ownedBy <$ownedBy> ; + sbh:topLevel <$attachmentURI> ; + sbol:source <$attachmentSource> ; + sbol:hash $hash ; + sbol:size $size ; + sbol:format <$type> . + +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/sparql/AttachmentUpdate.sparql b/backend/src/main/java/com/synbiohub/sbh3/sparql/AttachmentUpdate.sparql new file mode 100644 index 00000000..e4f12b3f --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/sparql/AttachmentUpdate.sparql @@ -0,0 +1,9 @@ +DELETE { + ?s ?p <$oldUri> +} +INSERT { + ?s ?p <$newUri> +} +WHERE { + ?s ?p <$oldUri> +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/sparql/GetAttachmentSourceFromTopLevel.sparql b/backend/src/main/java/com/synbiohub/sbh3/sparql/GetAttachmentSourceFromTopLevel.sparql new file mode 100644 index 00000000..135fadb4 --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/sparql/GetAttachmentSourceFromTopLevel.sparql @@ -0,0 +1,11 @@ +PREFIX sbol: +PREFIX sbh: + +SELECT + ?source + ?attachment +WHERE { + <$uri> sbol:member ?attachment . + ?attachment a sbol:Attachment . + ?attachment sbol:source ?source . +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/sparql/UpdateAttachment.sparql b/backend/src/main/java/com/synbiohub/sbh3/sparql/UpdateAttachment.sparql new file mode 100644 index 00000000..788084e8 --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/sparql/UpdateAttachment.sparql @@ -0,0 +1,21 @@ +PREFIX dcterms: +PREFIX sbh: +PREFIX sbol: + +DELETE WHERE { + <$attachmentURI> sbol:source ?source . +} +; +DELETE WHERE { + <$attachmentURI> sbol:hash ?hash . +} +; +DELETE WHERE { + <$attachmentURI> sbol:size ?size . +} +; +INSERT DATA { + <$attachmentURI> sbol:source <$attachmentSource> . + <$attachmentURI> sbol:hash $hash . + <$attachmentURI> sbol:size $size . +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/sparql/remove.sparql b/backend/src/main/java/com/synbiohub/sbh3/sparql/remove.sparql new file mode 100644 index 00000000..3e40102f --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/sparql/remove.sparql @@ -0,0 +1,8 @@ +PREFIX sbh: + +DELETE { + ?s ?p ?o +} WHERE { + ?s ?p ?o . + ?s sbh:topLevel <$uri> +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/sparql/removeCollection.sparql b/backend/src/main/java/com/synbiohub/sbh3/sparql/removeCollection.sparql new file mode 100644 index 00000000..186a5b42 --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/sparql/removeCollection.sparql @@ -0,0 +1,11 @@ +PREFIX sbh: +PREFIX sbol2: + +DELETE { + ?s ?p ?o +} WHERE { + ?s ?p ?o . + <$collection> sbol2:member ?member . + ?s sbh:topLevel ?member . + FILTER(STRSTARTS(str(?s), '$uriPrefix')) +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/submit/SubmitExposeRegistry.java b/backend/src/main/java/com/synbiohub/sbh3/submit/SubmitExposeRegistry.java new file mode 100644 index 00000000..869baa40 --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/submit/SubmitExposeRegistry.java @@ -0,0 +1,37 @@ +package com.synbiohub.sbh3.submit; + +import com.synbiohub.sbh3.utils.ConfigUtil; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Single-use URLs under {@code /expose/{token}} so submit plugins can fetch uploaded files. + */ +@Component +public class SubmitExposeRegistry { + + private final Map exposed = new ConcurrentHashMap<>(); + + public String register(Path file) throws IOException { + String token = UUID.randomUUID().toString(); + exposed.put(token, file.toAbsolutePath()); + String base = ConfigUtil.get("instanceUrl").asText(); + if (!base.endsWith("/")) { + base = base + "/"; + } + return base + "expose/" + token; + } + + public Path resolve(String token) { + return exposed.get(token); + } + + public void revoke(String token) { + exposed.remove(token); + } +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/submit/SubmitPayload.java b/backend/src/main/java/com/synbiohub/sbh3/submit/SubmitPayload.java new file mode 100644 index 00000000..5f476b08 --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/submit/SubmitPayload.java @@ -0,0 +1,89 @@ +package com.synbiohub.sbh3.submit; + +import lombok.Data; +import org.sbolstandard.core2.SBOLDocument; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Single mutable carrier for the entire submit pipeline (parse → sanitize → SBOL → upload). + * Fields are populated incrementally; unset values remain {@code null} until their step runs. + *

+ * Derived values ({@link #collectionDisplayId()}, {@link #uriPrefix()}) are + * computed from other fields rather than stored separately. + */ +@Data +public class SubmitPayload { + + // --- Request / parsed --- + + /** User-defined collection identifier (alphanumeric and underscores). */ + private String id; + + private String name; + private String description; + + /** Version string (e.g. {@code "1"}). */ + private String version; + + /** Raw comma-separated PubMed IDs from the form; consumed by {@code sanitize}. */ + private String citations; + + /** {@code overwrite_merge} form value: 0–3. */ + private String overwriteMerge; + + /** Submit plugin id (e.g. {@code default}). */ + private String plugin; + + /** + * Target root collection identity URI. + * From {@code rootCollections} when submitting into an existing collection, or computed + * during sanitization for new collections. + */ + private String collectionUri; + + /** Path to the uploaded file on disk (temp file). */ + private String uploadedFilePath; + + private String createdBy; + + // --- Sanitized / resolved --- + + /** Parsed PubMed citation ids (from the form {@code citations} field). */ + private List citationPubmedIds = new ArrayList<>(); + + /** Store snapshot when collection metadata exists in the triple store. */ + private SubmitRootCollectionMetadata existingCollection; + + // --- readSbol --- + + /** Composite SBOL document built during {@code readSbol}. */ + private SBOLDocument sbolDocument; + + private List attachmentFiles = new ArrayList<>(); + private List sbolFiles = new ArrayList<>(); + private Set urisFoundInSynBioHub = new HashSet<>(); + + /** {@link System#currentTimeMillis()} when {@code readSbol} setup started. */ + private Long readSbolStartedAtMs; + + /** Serialized SBOL XML from {@code readSbol} (legacy {@code resultFilename}). */ + private String resultFilePath; + + /** Temp unpack directory when archive extraction ran. */ + private String extractDirPath; + + // --- Derived accessors --- + + public String collectionDisplayId() { + return id == null ? null : id + "_collection"; + } + + public void setOverwrite_merge(String overwriteMerge) { + this.overwriteMerge = overwriteMerge; + } +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/submit/SubmitPluginService.java b/backend/src/main/java/com/synbiohub/sbh3/submit/SubmitPluginService.java new file mode 100644 index 00000000..ad731d08 --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/submit/SubmitPluginService.java @@ -0,0 +1,169 @@ +package com.synbiohub.sbh3.submit; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.synbiohub.sbh3.services.PluginService; +import com.synbiohub.sbh3.utils.ConfigUtil; +import lombok.RequiredArgsConstructor; +import org.apache.commons.io.FilenameUtils; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Service; +import org.springframework.web.server.ResponseStatusException; + +import java.io.IOException; +import java.net.URLConnection; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +/** + * Runs the submit-plugin evaluate/run flow and replaces {@link SubmitPayload#getUploadedFilePath()} + * with converted SBOL from the plugin zip response. + */ +@Service +@RequiredArgsConstructor +public class SubmitPluginService { + + private final PluginService pluginService; + private final SubmitExposeRegistry exposeRegistry; + private final ObjectMapper mapper; + + public void applySubmitPlugin(SubmitPayload payload) throws IOException { + if (payload.getUploadedFilePath() == null || isDefaultPlugin(payload.getPlugin())) { + return; + } + + String pluginName = pluginService.resolveSubmitPluginName(payload.getPlugin()); + pluginService.checkSubmitPluginStatus(pluginName); + + Path inputFile = Path.of(payload.getUploadedFilePath()); + String filename = inputFile.getFileName().toString(); + String mimeType = URLConnection.guessContentTypeFromName(filename); + if (mimeType == null) { + mimeType = "application/octet-stream"; + } + + String exposeUrl = exposeRegistry.register(inputFile); + String exposeToken = exposeUrl.substring(exposeUrl.lastIndexOf('/') + 1); + + try { + ObjectNode evaluateManifest = buildPluginManifest(exposeUrl, filename, mimeType, false); + JsonNode evaluateResponse = pluginService.evaluateSubmitPlugin(pluginName, evaluateManifest); + if (!pluginCanConvert(evaluateResponse, filename)) { + throw new ResponseStatusException(HttpStatus.BAD_REQUEST, + "The plugin " + pluginName + " requires a different file type."); + } + + ObjectNode runRequest = buildPluginManifest(exposeUrl, filename, mimeType, true); + byte[] zipBytes = pluginService.runSubmitPlugin(pluginName, runRequest); + payload.setUploadedFilePath(extractConvertedSbolPath(zipBytes, inputFile.getParent())); + } finally { + exposeRegistry.revoke(exposeToken); + } + } + + private static boolean isDefaultPlugin(String plugin) { + return plugin == null || plugin.isBlank() || "default".equalsIgnoreCase(plugin); + } + + private ObjectNode buildPluginManifest(String exposeUrl, String filename, String mimeType, + boolean includeInstanceUrl) throws IOException { + ObjectNode file = mapper.createObjectNode(); + file.put("url", exposeUrl); + file.put("filename", filename); + file.put("type", mimeType); + + ArrayNode files = mapper.createArrayNode(); + files.add(file); + + ObjectNode manifest = mapper.createObjectNode(); + manifest.set("files", files); + if (includeInstanceUrl) { + manifest.put("instanceUrl", ConfigUtil.get("instanceUrl").asText()); + } + + ObjectNode root = mapper.createObjectNode(); + root.set("manifest", manifest); + return root; + } + + private boolean pluginCanConvert(JsonNode evaluateResponse, String filename) { + JsonNode manifest = evaluateResponse.path("manifest"); + if (!manifest.isArray()) { + return false; + } + for (JsonNode entry : manifest) { + if (filename.equals(entry.path("filename").asText()) + && entry.path("requirement").asInt(0) == 2) { + return true; + } + } + return false; + } + + private String extractConvertedSbolPath(byte[] zipBytes, Path tempDir) throws IOException { + Path extractDir = Files.createTempDirectory(tempDir, "sbh-plugin-out-"); + Path manifestPath = null; + List resultFilenames = new ArrayList<>(); + + try (ZipInputStream zis = new ZipInputStream(new java.io.ByteArrayInputStream(zipBytes))) { + ZipEntry entry; + while ((entry = zis.getNextEntry()) != null) { + if (entry.isDirectory()) { + continue; + } + Path out = extractDir.resolve(entry.getName()).normalize(); + if (!out.startsWith(extractDir)) { + throw new IOException("Invalid zip entry path"); + } + Files.createDirectories(out.getParent()); + Files.copy(zis, out); + if ("manifest.json".equalsIgnoreCase(entry.getName())) { + manifestPath = out; + } + zis.closeEntry(); + } + } + + if (manifestPath != null && Files.exists(manifestPath)) { + JsonNode manifest = mapper.readTree(manifestPath.toFile()); + for (JsonNode result : manifest.path("results")) { + resultFilenames.add(result.path("filename").asText()); + } + } + + for (String resultName : resultFilenames) { + Path candidate = extractDir.resolve(resultName); + if (Files.isRegularFile(candidate)) { + return candidate.toAbsolutePath().toString(); + } + } + + try (var stream = Files.list(extractDir)) { + return stream + .filter(Files::isRegularFile) + .filter(p -> !"manifest.json".equalsIgnoreCase(p.getFileName().toString())) + .filter(p -> isSbolLike(p.getFileName().toString())) + .findFirst() + .map(p -> p.toAbsolutePath().toString()) + .orElseThrow(() -> new ResponseStatusException(HttpStatus.BAD_GATEWAY, + "Submit plugin did not return a convertible SBOL file.")); + } + } + + private static boolean isSbolLike(String name) { + String ext = FilenameUtils.getExtension(name); + if (ext == null) { + return false; + } + return switch (ext.toLowerCase()) { + case "xml", "sbol", "rdf" -> true; + default -> name.contains(".converted"); + }; + } +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/submit/SubmitRootCollectionMetadata.java b/backend/src/main/java/com/synbiohub/sbh3/submit/SubmitRootCollectionMetadata.java new file mode 100644 index 00000000..956b1c33 --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/submit/SubmitRootCollectionMetadata.java @@ -0,0 +1,20 @@ +package com.synbiohub.sbh3.submit; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Metadata for a root collection that already exists in the triple store. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class SubmitRootCollectionMetadata { + private String name; + private String description; + private String displayId; + private String version; +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/utils/FileUtil.java b/backend/src/main/java/com/synbiohub/sbh3/utils/FileUtil.java new file mode 100644 index 00000000..73060a9b --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/utils/FileUtil.java @@ -0,0 +1,156 @@ +package com.synbiohub.sbh3.utils; + +import com.synbiohub.sbh3.dto.SubmitFileFormat; +import com.synbiohub.sbh3.submit.SubmitPayload; +import org.apache.commons.io.FilenameUtils; +import org.sbolstandard.core2.SBOLReader; +import org.springframework.http.HttpStatus; +import org.springframework.web.server.ResponseStatusException; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +public interface FileUtil { + + static String fileExtension(Path file) { + String ext = FilenameUtils.getExtension(file.getFileName().toString()); + return ext == null ? "" : ext.toLowerCase(); + } + + /** COMBINE archives contain a manifest.xml entry. */ + static boolean isCombineArchive(Path file) throws IOException { + if (!"zip".equals(fileExtension(file)) && !"omex".equals(fileExtension(file))) { + return false; + } + try (ZipFile zip = new ZipFile(file.toFile())) { + return zip.getEntry("manifest.xml") != null + || zip.stream().anyMatch(e -> e.getName().endsWith("manifest.xml")); + } + } + + /** Extracts a zip/omex archive and classifies each entry; sets {@code extractDirPath}. */ + static void extractSubmitArchive(Path archive, SubmitPayload payload, boolean combine) + throws IOException { + Path dest = Files.createTempDirectory("sbh-submit-unpack-"); + payload.setExtractDirPath(dest.toAbsolutePath().toString()); + try (ZipFile zip = new ZipFile(archive.toFile())) { + var entries = zip.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + if (entry.isDirectory()) { + continue; + } + Path out = dest.resolve(entry.getName()).normalize(); + if (!out.startsWith(dest)) { + continue; // zip-slip guard + } + Files.createDirectories(out.getParent()); + try (InputStream in = zip.getInputStream(entry)) { + Files.copy(in, out); + } + classifySubmitFile(out, payload); + } + } + if (combine && payload.getSbolFiles().isEmpty()) { + payload.getSbolFiles().add(archive); + } + } + + /** Routes a single file into sbolFiles or attachmentFiles. */ + static void classifySubmitFile(Path file, SubmitPayload payload) throws IOException { + switch (guessSubmitFileFormat(file)) { + case SBOL, GENBANK, FASTA, GFF3 -> payload.getSbolFiles().add(file); + case ATTACHMENT -> payload.getAttachmentFiles().add(file); + } + } + + /** Sniffs file content and extension; anything unrecognized becomes an attachment. */ + static SubmitFileFormat guessSubmitFileFormat(Path file) throws IOException { + String path = file.toAbsolutePath().toString(); + if (SBOLReader.isGenBankFile(path)) { + return SubmitFileFormat.GENBANK; + } + if (SBOLReader.isFastaFile(path)) { + return SubmitFileFormat.FASTA; + } + if (SBOLReader.isGFF3File(path)) { + return SubmitFileFormat.GFF3; + } + + String head = readFileHead(file, 4096).toLowerCase(); + if (head.contains("sbols.org") || head.contains("sbol.org") || head.contains(" b.compareTo(a)) + .forEach(path -> { + try { + Files.deleteIfExists(path); + } catch (IOException e) { + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, + "Failed to delete " + path, e); + } + }); + } + } +} diff --git a/backend/src/main/java/com/synbiohub/sbh3/utils/ObjectMapperUtils.java b/backend/src/main/java/com/synbiohub/sbh3/utils/ObjectMapperUtils.java deleted file mode 100644 index ebb01871..00000000 --- a/backend/src/main/java/com/synbiohub/sbh3/utils/ObjectMapperUtils.java +++ /dev/null @@ -1,59 +0,0 @@ -package com.synbiohub.sbh3.utils; - -import com.synbiohub.sbh3.dto.SubmissionDTO; -import com.synbiohub.sbh3.requests.SubmitRequest; -import org.modelmapper.ModelMapper; -import org.sbolstandard.core2.SBOLDocument; - -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; - -public class ObjectMapperUtils { - private static final ModelMapper modelMapper = new ModelMapper(); - - private ObjectMapperUtils(){} - - public static D map(final T entity, Class outClass) { return modelMapper.map(entity, outClass); } - - public static List mapAll(final Collection entityList, Class outClass) { - return entityList.stream() - .map(entity -> map(entity, outClass)) - .collect(Collectors.toList()); - } - - public static D map(final S source, D destination) { - modelMapper.map(source, destination); - return destination; - } - -// public static SubmissionDTO createNewSubmission(SubmitRequest request) { -// return SubmissionDTO.builder() -// .id(request.getId()) -// .version(Integer.parseInt(request.getVersion())) -// .name(request.getName()) -// .description(request.getDescription()) -// .citations(Arrays.stream(request.getCitations().split(",")) -// .map(Integer::parseInt) -// .collect(Collectors.toList())) -// .files(request.getFiles()) -// .build(); -// } - - public static SubmissionDTO createNewSubmission(String name, String description, String id, int version, List citations, Collection allFiles, int overwriteMerge, ArrayList plugins, Collection attachmentFiles, SBOLDocument sbolDocument) { - return SubmissionDTO.builder() - .id(id) - .version(version) - .name(name) - .description(description) - .citations(citations) - .files(allFiles) - .plugins(plugins) - .attachments(attachmentFiles) - .sbolDocument(sbolDocument) - .build(); - } -} diff --git a/backend/src/main/java/com/synbiohub/sbh3/utils/SbolWriterLegacyPrefixRewriter.java b/backend/src/main/java/com/synbiohub/sbh3/utils/SbolWriterLegacyPrefixRewriter.java new file mode 100644 index 00000000..0c43a2ba --- /dev/null +++ b/backend/src/main/java/com/synbiohub/sbh3/utils/SbolWriterLegacyPrefixRewriter.java @@ -0,0 +1,277 @@ +package com.synbiohub.sbh3.utils; + +import org.w3c.dom.Attr; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; +import java.io.ByteArrayInputStream; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Rewrites {@link org.sbolstandard.core2.SBOLWriter} RDF/XML so root namespaces match + * SynBioHub1's legacy layout ({@code sbh}, {@code igem}, {@code dcterms}, …) instead of + * Jena/SBOLWriter {@code ns*} prefixes and fragment-split xmlns declarations that fail + * SBOL validator equality / download regression tests. + *

+ * Element nesting from SBOLWriter is preserved; only prefixes / xmlns decls are changed. + * Non-legacy namespaces that are still used by elements/attributes (e.g. partsregistry + * {@code j.0}) are kept so the XML stays well-formed. + */ +public final class SbolWriterLegacyPrefixRewriter { + + private static final String RDF_NS = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"; + private static final String XMLNS_NS = "http://www.w3.org/2000/xmlns/"; + + /** Exact namespace URI → preferred legacy prefix (declaration order on rdf:RDF). */ + private static final Map LEGACY_URI_TO_PREFIX = new LinkedHashMap<>(); + + static { + LEGACY_URI_TO_PREFIX.put(RDF_NS, "rdf"); + LEGACY_URI_TO_PREFIX.put("http://purl.org/dc/terms/", "dcterms"); + LEGACY_URI_TO_PREFIX.put("http://www.w3.org/ns/prov#", "prov"); + LEGACY_URI_TO_PREFIX.put("http://sbols.org/v2#", "sbol"); + LEGACY_URI_TO_PREFIX.put("http://www.w3.org/2001/XMLSchema#dateTime/", "xsd"); + LEGACY_URI_TO_PREFIX.put("http://www.ontology-of-units-of-measure.org/resource/om-2/", "om"); + LEGACY_URI_TO_PREFIX.put("http://synbiohub.org#", "synbiohub"); + LEGACY_URI_TO_PREFIX.put("http://wiki.synbiohub.org/wiki/Terms/synbiohub#", "sbh"); + LEGACY_URI_TO_PREFIX.put("http://www.sybio.ncl.ac.uk#", "sybio"); + LEGACY_URI_TO_PREFIX.put("http://www.w3.org/2000/01/rdf-schema#", "rdfs"); + LEGACY_URI_TO_PREFIX.put("http://www.ncbi.nlm.nih.gov#", "ncbi"); + LEGACY_URI_TO_PREFIX.put("http://wiki.synbiohub.org/wiki/Terms/igem#", "igem"); + LEGACY_URI_TO_PREFIX.put("http://www.ncbi.nlm.nih.gov/genbank#", "genbank"); + LEGACY_URI_TO_PREFIX.put("http://sbols.org/genBankConversion#", "gbconv"); + LEGACY_URI_TO_PREFIX.put("http://purl.org/dc/elements/1.1/", "dc"); + LEGACY_URI_TO_PREFIX.put("http://purl.obolibrary.org/obo/", "obo"); + } + + /** Also accept Jena's bare XSD NS and rewrite to SBH1's dateTime/ variant. */ + private static final String XSD_BARE = "http://www.w3.org/2001/XMLSchema#"; + private static final String XSD_LEGACY = "http://www.w3.org/2001/XMLSchema#dateTime/"; + + private SbolWriterLegacyPrefixRewriter() { + } + + /** + * Rewrite SBOLWriter RDF/XML bytes to legacy SynBioHub1 namespace layout. + * Returns {@code xml} unchanged on parse/serialize failure. + */ + public static byte[] rewrite(byte[] xml) { + if (xml == null || xml.length == 0) { + return xml; + } + try { + String in = new String(xml, StandardCharsets.UTF_8); + String out = rewrite(in); + return out.getBytes(StandardCharsets.UTF_8); + } catch (Exception e) { + return xml; + } + } + + public static String rewrite(String xml) throws Exception { + if (xml == null || xml.isBlank()) { + return xml; + } + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + dbf.setNamespaceAware(true); + Document doc = dbf.newDocumentBuilder() + .parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))); + Element root = doc.getDocumentElement(); + if (root == null || !"RDF".equals(root.getLocalName())) { + return xml; + } + + Map oldPrefixToUri = collectXmlns(root); + Map oldPrefixToNewPrefix = new LinkedHashMap<>(); + + for (Map.Entry e : oldPrefixToUri.entrySet()) { + String oldPrefix = e.getKey(); + String uri = normalizeUri(e.getValue()); + String legacy = LEGACY_URI_TO_PREFIX.get(uri); + if (legacy != null && !legacy.equals(oldPrefix)) { + oldPrefixToNewPrefix.put(oldPrefix, legacy); + } + } + + renamePrefixedNodes(root, oldPrefixToNewPrefix); + + // Prefixes still used by elements/attrs after remap (keeps partsregistry / custom NS). + Map usedPrefixToUri = collectUsedPrefixes(root); + + // Rebuild rdf:RDF xmlns: legacy set first, then any still-used non-legacy prefixes. + stripAllXmlns(root); + Set declared = new LinkedHashSet<>(); + for (Map.Entry e : LEGACY_URI_TO_PREFIX.entrySet()) { + root.setAttributeNS(XMLNS_NS, "xmlns:" + e.getValue(), e.getKey()); + declared.add(e.getValue()); + } + for (Map.Entry e : usedPrefixToUri.entrySet()) { + String prefix = e.getKey(); + if (prefix == null || prefix.isEmpty() || declared.contains(prefix)) { + continue; + } + if ("xml".equals(prefix) || "xmlns".equals(prefix)) { + continue; + } + String uri = normalizeUri(e.getValue()); + // Skip if this URI already has a legacy prefix declared. + if (LEGACY_URI_TO_PREFIX.containsKey(uri)) { + continue; + } + root.setAttributeNS(XMLNS_NS, "xmlns:" + prefix, uri); + declared.add(prefix); + } + + String serialized = serialize(doc); + serialized = normalizeXmlDeclaration(serialized); + return serialized; + } + + private static String normalizeUri(String uri) { + if (XSD_BARE.equals(uri)) { + return XSD_LEGACY; + } + return uri; + } + + private static Map collectXmlns(Element root) { + Map map = new LinkedHashMap<>(); + NamedNodeMap attrs = root.getAttributes(); + for (int i = 0; i < attrs.getLength(); i++) { + Attr a = (Attr) attrs.item(i); + String name = a.getName(); + if ("xmlns".equals(name)) { + map.put("", a.getValue()); + } else if (name.startsWith("xmlns:")) { + map.put(name.substring("xmlns:".length()), a.getValue()); + } + } + return map; + } + + private static Map collectUsedPrefixes(Element root) { + Map used = new LinkedHashMap<>(); + collectUsedPrefixesRecursive(root, used); + return used; + } + + private static void collectUsedPrefixesRecursive(Element el, Map used) { + String prefix = el.getPrefix(); + if (prefix != null && !prefix.isEmpty() && el.getNamespaceURI() != null) { + used.putIfAbsent(prefix, el.getNamespaceURI()); + } + NamedNodeMap attrs = el.getAttributes(); + for (int i = 0; i < attrs.getLength(); i++) { + Attr a = (Attr) attrs.item(i); + String ap = a.getPrefix(); + if (ap != null && !ap.isEmpty() && !"xmlns".equals(ap) && a.getNamespaceURI() != null) { + used.putIfAbsent(ap, a.getNamespaceURI()); + } + } + NodeList children = el.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + Node n = children.item(i); + if (n.getNodeType() == Node.ELEMENT_NODE) { + collectUsedPrefixesRecursive((Element) n, used); + } + } + } + + private static void stripAllXmlns(Element root) { + List toRemove = new ArrayList<>(); + NamedNodeMap attrs = root.getAttributes(); + for (int i = 0; i < attrs.getLength(); i++) { + Attr a = (Attr) attrs.item(i); + String name = a.getName(); + if ("xmlns".equals(name) || name.startsWith("xmlns:")) { + toRemove.add(a); + } + } + for (Attr a : toRemove) { + root.removeAttributeNode(a); + } + } + + private static void renamePrefixedNodes(Element el, Map oldToNew) { + String prefix = el.getPrefix(); + if (prefix != null && oldToNew.containsKey(prefix)) { + String newPrefix = oldToNew.get(prefix); + String ns = el.getNamespaceURI(); + if (XSD_BARE.equals(ns)) { + ns = XSD_LEGACY; + } + el.getOwnerDocument().renameNode(el, ns, newPrefix + ":" + el.getLocalName()); + } + + NamedNodeMap attrs = el.getAttributes(); + // Copy list; renameNode can mutate the live map. + List attrList = new ArrayList<>(); + for (int i = 0; i < attrs.getLength(); i++) { + attrList.add((Attr) attrs.item(i)); + } + for (Attr a : attrList) { + String ap = a.getPrefix(); + if (ap != null && oldToNew.containsKey(ap) && !"xmlns".equals(ap)) { + String newPrefix = oldToNew.get(ap); + String ns = a.getNamespaceURI(); + if (XSD_BARE.equals(ns)) { + ns = XSD_LEGACY; + } + el.getOwnerDocument().renameNode(a, ns, newPrefix + ":" + a.getLocalName()); + } + } + + NodeList children = el.getChildNodes(); + for (int i = 0; i < children.getLength(); i++) { + Node n = children.item(i); + if (n.getNodeType() == Node.ELEMENT_NODE) { + renamePrefixedNodes((Element) n, oldToNew); + } + } + } + + private static String serialize(Document doc) throws Exception { + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer t = tf.newTransformer(); + t.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "no"); + t.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); + t.setOutputProperty(OutputKeys.INDENT, "yes"); + t.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2"); + StringWriter sw = new StringWriter(); + t.transform(new DOMSource(doc), new StreamResult(sw)); + return sw.toString(); + } + + private static String normalizeXmlDeclaration(String s) { + if (s.startsWith("")) { + return s; + } + if (s.startsWith("")) { + return "" + + s.substring("".length()); + } + if (s.startsWith("")) { + return "" + s.substring("".length()); + } + if (s.startsWith("")) { + return "" + s.substring("".length()); + } + return s; + } +} diff --git a/backend/src/main/resources/application.yml b/backend/src/main/resources/application.yml index c39df1d8..7d62acbc 100644 --- a/backend/src/main/resources/application.yml +++ b/backend/src/main/resources/application.yml @@ -4,6 +4,10 @@ server: relaxed-path-chars: ['<','>'] relaxed-query-chars: ['<','>'] spring: + servlet: + multipart: + max-file-size: 50MB + max-request-size: 50MB datasource: driver-class-name: org.h2.Driver url: jdbc:h2:./data/sbhdb diff --git a/tests/Emulator_Settings.txt b/tests/Emulator_Settings.txt new file mode 100644 index 00000000..c48ea658 --- /dev/null +++ b/tests/Emulator_Settings.txt @@ -0,0 +1,15 @@ +{ + 'url':'http://localhost:6789', + 'prefix':'https://synbiohub.org', + 'email':'test@user.synbiohub', + 'pass':'test', + 'user':'testuser', + 'creator':'Test User', + 'id':'Tester_1', + 'version':'1', + 'name':'Round_Trip_Tester', + 'desc':'This is a simple roundtrip test', + 'topLevel':'https://synbiohub.org/user/testuser/Tester_1/Tester_1_collection/1', + 'complete':true, + 'create_defaults':true +} diff --git a/tests/SBOLTestRunner b/tests/SBOLTestRunner index 31a780fb..46b16db6 160000 --- a/tests/SBOLTestRunner +++ b/tests/SBOLTestRunner @@ -1 +1 @@ -Subproject commit 31a780fb72ac1f143d67e0bc4ab9f28a2b48d6de +Subproject commit 46b16db60ce73670ada409659c7634402af4f7ba diff --git a/tests/print_error_log.py b/tests/print_error_log.py new file mode 100644 index 00000000..2f4f0813 --- /dev/null +++ b/tests/print_error_log.py @@ -0,0 +1,17 @@ +try: + from test_functions import get_end_of_error_log +except ImportError: + get_end_of_error_log = None + + +def print_entire_log(): + if get_end_of_error_log is None: + print("Could not import get_end_of_error_log; skipping error log dump.") + return + try: + print(get_end_of_error_log(-1)) + except Exception as e: + print(f"Could not print error log: {e}") + + +print_entire_log() diff --git a/tests/run_sboltestrunner.sh b/tests/run_sboltestrunner.sh new file mode 100644 index 00000000..8da82645 --- /dev/null +++ b/tests/run_sboltestrunner.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -e + +source ./testutil.sh + +message "SBOLTestRunner" + +if [ ! -f SBOLTestRunner/pom.xml ]; then + message "SBOLTestRunner/pom.xml missing; run sbolsuite.sh (or clone SBOLTestRunner) first" + exit 1 +fi + +message "pulling mehersam/SynBioHubRunner" +if [ -d SynBioHubRunner/.git ] || [ -f SynBioHubRunner/pom.xml ]; then + (cd SynBioHubRunner && git pull --ff-only 2>/dev/null || true) +else + rm -rf SynBioHubRunner + git clone --recurse-submodules https://github.com/mehersam/SynBioHubRunner +fi + +message "Setting up SynBioHubRunner" +cp Emulator_Settings.txt SynBioHubRunner/src/resources/Emulator_Settings.txt +(cd SynBioHubRunner && mvn package) + +message "Building TestRunner" +(cd SBOLTestRunner && mvn package) + +JAR="SBOLTestRunner/target/SBOLTestRunner-0.0.1-SNAPSHOT-withDependencies.jar" +EMU_JAR="SynBioHubRunner/target/SBHEmulator-0.0.1-SNAPSHOT-withDependencies.jar" +if [ ! -f "$JAR" ] || [ ! -f "$EMU_JAR" ]; then + message "Expected jars missing after mvn package" + exit 1 +fi + +message "Running SBOLTestRunner" + +rm -rf Timing Emulated Retrieved Compared +mkdir Timing Emulated Retrieved Compared + +set +e +java -jar "$JAR" "java -jar $EMU_JAR" "Compared/" "Retrieved/" "-e" "Emulated/" | tee sbol_testrunner_result +exitcode=${PIPESTATUS[0]} +set -e + +rm -f sbol_testrunner_result + +if [ $exitcode -ne 0 ]; then + python3 print_error_log.py "$@" || true + message "Exiting with code $exitcode." + exit $exitcode +fi diff --git a/tests/sbolsuite.sh b/tests/sbolsuite.sh index 4dcfe953..581eeef0 100755 --- a/tests/sbolsuite.sh +++ b/tests/sbolsuite.sh @@ -1,22 +1,54 @@ #!/bin/bash -cd tests +set -e + source ./testutil.sh message "Running synbiohub test suite." -# Clone the SBOLTestRunner for necessary files +# SBOLTestRunner is often an empty gitlink/submodule checkout in CI. +# Ensure a real clone with pom.xml + nested SBOLTestSuite submodule. +ensure_sboltestrunner() { + if [ -f SBOLTestRunner/pom.xml ]; then + message "SBOLTestRunner already present" + ( + cd SBOLTestRunner + git pull --ff-only 2>/dev/null || true + git submodule update --init --recursive 2>/dev/null || true + ) + return + fi + + message "SBOLTestRunner missing or empty; cloning mehersam/SBOLTestRunner" + rm -rf SBOLTestRunner + git clone --recurse-submodules https://github.com/mehersam/SBOLTestRunner +} + message "pulling mehersam/SBOLTestRunner" -if cd SBOLTestRunner; then - git pull; - cd ..; +ensure_sboltestrunner + +if [ ! -f SBOLTestRunner/pom.xml ]; then + message "SBOLTestRunner/pom.xml still missing after clone" + exit 1 +fi + +# Use repo-root .venv so setup doesn't hit Homebrew's externally-managed Python. +REPO_ROOT="$(cd .. && pwd)" +if [ -x "$REPO_ROOT/.venv/bin/python3" ]; then + PYTHON="$REPO_ROOT/.venv/bin/python3" else - git clone --recurse-submodules https://github.com/mehersam/SBOLTestRunner; + message "Creating .venv in repo root" + python3 -m venv "$REPO_ROOT/.venv" + PYTHON="$REPO_ROOT/.venv/bin/python3" fi +message "Installing Python test deps" +"$PYTHON" -m pip install -q requests requests_html beautifulsoup4 lxml lxml_html_clean bash ./start_containers.sh -python3 -c "from first_time_setup import TestSetup; ts = TestSetup(); ts.test_post()" + +message "Running first-time setup" +"$PYTHON" -c "from first_time_setup import TestSetup; ts = TestSetup(); ts.test_post()" bash ./run_sboltestrunner.sh exitcode=$? @@ -27,5 +59,4 @@ fi bash ./stop_containers.sh - message "finished running tests" diff --git a/tests/start_containers_persist.sh b/tests/start_containers_persist.sh index f3dee25a..b1f977a7 100644 --- a/tests/start_containers_persist.sh +++ b/tests/start_containers_persist.sh @@ -2,13 +2,33 @@ source ./testutil.sh +COMPOSE_FILE="docker-compose.yml" +PROJECT="testsuiteproject" + +# Prefer Compose V2 (`docker compose`); fall back to legacy `docker-compose`. +if docker compose version >/dev/null 2>&1; then + COMPOSE=(docker compose) +else + COMPOSE=(docker-compose) +fi + +wait_http() { + local url="$1" + local label="$2" + message "Waiting for ${label} (${url})" + until curl -s -o /dev/null -w "%{http_code}" "$url" | grep -Eq '200|301|302|401|404'; do + sleep 3 + message "Waiting for ${label}..." + done +} + message "Starting SynBioHub from Containers" -docker-compose -f docker-compose.yml -p testsuiteproject --compatibility up -d -while [[ "$(docker inspect testsuiteproject_synbiohub_1 | jq .[0].State.Health.Status)" != "\"healthy\"" ]] -do - sleep 5 - message "Waiting for synbiohub container to be healthy." -done +"${COMPOSE[@]}" -f "$COMPOSE_FILE" -p "$PROJECT" up -d -message "Started successfully" +# synbiohubbackend has no healthcheck; wait until SBH3 HTTP answers (401 on / is OK). +wait_http "http://localhost:6789/" "synbiohubbackend" + +# SBH1 is used by first_time_setup; wait until it answers too. +wait_http "http://localhost:7777/" "synbiohub" +message "Started successfully" diff --git a/tests/stop_containers.sh b/tests/stop_containers.sh index 16fa4aae..ca01dd50 100644 --- a/tests/stop_containers.sh +++ b/tests/stop_containers.sh @@ -2,16 +2,14 @@ source ./testutil.sh -# stop the containers -message "Stopping containers" -docker stop testsuiteproject_synbiohub_1 -docker stop testsuiteproject_explorer_1 -docker stop testsuiteproject_autoheal_1 -docker stop testsuiteproject_virtuoso_1 -docker stop testsuiteproject_virtuoso3_1 -docker stop testsuiteproject_synbiohubbackend_1 -docker stop testsuiteproject_synbiohubFrontend_1 - - +COMPOSE_FILE="docker-compose.yml" +PROJECT="testsuiteproject" +if docker compose version >/dev/null 2>&1; then + COMPOSE=(docker compose) +else + COMPOSE=(docker-compose) +fi +message "Stopping containers" +"${COMPOSE[@]}" -f "$COMPOSE_FILE" -p "$PROJECT" stop diff --git a/tests/test.sh b/tests/test.sh index bd5dfd7e..0f0c5e7a 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -10,13 +10,17 @@ message "pulling backend image" #docker pull synbiohub/sbh3backend:snapshot -# Clone the SBOLTestRunner for necessary files +# Clone the SBOLTestRunner for necessary files (empty gitlink/submodule is common in CI) message "pulling mehersam/SBOLTestRunner" -if cd SBOLTestRunner; then - git pull; - cd ..; +if [ -f SBOLTestRunner/pom.xml ]; then + ( + cd SBOLTestRunner + git pull --ff-only 2>/dev/null || true + git submodule update --init --recursive 2>/dev/null || true + ) else - git clone --recurse-submodules https://github.com/mehersam/SBOLTestRunner; + rm -rf SBOLTestRunner + git clone --recurse-submodules https://github.com/mehersam/SBOLTestRunner fi #clone libSBOLj diff --git a/tests/test_functions.py b/tests/test_functions.py index 7ea3eab7..1d087824 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -369,13 +369,15 @@ def file_diff(sbh1requestcontent, sbh3requestcontent, request, requesttype): changelist.append(c) changelist.append("\n") - changelist.append("\n Here is the last 50 lines of the synbiohub error log: \n") - changelist.append(get_end_of_error_log(50)) - - if numofchanges>0: + if numofchanges > 0: + changelist.append("\n Here is the last 50 lines of the synbiohub error log: \n") + try: + changelist.append(get_end_of_error_log(50)) + except Exception as e: + changelist.append(f"(Could not fetch error log: {e})\n") + print(''.join(changelist)) return 0 - else: - return 1 + return 1 def login_with(data, valid, headers = {'Accept':'text/plain'}): resultSBH1 = post_request("login", 1, data, headers, [], files = None) @@ -472,20 +474,23 @@ def show_test_results(): test_state.show_test_results() def run_bash(command): - process = subprocess.Popen(command.split(), stdout=subprocess.PIPE) + process = subprocess.Popen(command.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = process.communicate() + return process.returncode, output, error def file_tail(filename, length): return os.popen('tail -n ' + str(length) +' '+filename).read() def get_end_of_error_log(num_of_lines): copy_docker_log() + if not os.path.isdir("./logs_from_test_suite"): + return "No logs_from_test_suite directory (docker log copy failed).\n" directory = os.listdir("./logs_from_test_suite") for filename in directory: if filename[len(filename)-5:] == "error": return file_tail("./logs_from_test_suite/" + filename, num_of_lines) - raise Exception("Could not find error log") + return "Could not find error log in logs_from_test_suite.\n" def copy_docker_log(): @@ -495,7 +500,23 @@ def copy_docker_log(): if os.path.isdir("docker_logs"): shutil.rmtree("./docker_logs") - run_bash("docker cp testsuiteproject_synbiohub_1:/mnt/data/logs .") - run_bash("mv ./logs ./logs_from_test_suite") + if os.path.isdir("./logs"): + shutil.rmtree("./logs") + + # Prefer Compose v2 hyphen names; fall back to legacy underscore names. + containers = [ + "testsuiteproject-synbiohub-1", + "testsuiteproject_synbiohub_1", + ] + for container in containers: + rc, _, _ = run_bash(f"docker cp {container}:/mnt/data/logs .") + if rc == 0 and os.path.isdir("./logs"): + run_bash("mv ./logs ./logs_from_test_suite") + return + + # Last resort: docker compose cp from the test project. + rc, _, _ = run_bash("docker compose -p testsuiteproject cp synbiohub:/mnt/data/logs ./logs") + if rc == 0 and os.path.isdir("./logs"): + run_bash("mv ./logs ./logs_from_test_suite") diff --git a/tests/testcleanup.sh b/tests/testcleanup.sh index 02348cb2..f425fb08 100755 --- a/tests/testcleanup.sh +++ b/tests/testcleanup.sh @@ -1,23 +1,13 @@ #!/bin/bash -docker kill testsuiteproject_synbiohub_1 -docker kill testsuiteproject_synbiohubFrontend -docker kill testsuiteproject_synbiohubbackend -docker kill testsuiteproject_explorer_1 -docker kill testsuiteproject_elasticsearch_1 -docker kill testsuiteproject_autoheal_1 -docker kill testsuiteproject_virtuoso_1 +COMPOSE_FILE="docker-compose.yml" +PROJECT="testsuiteproject" -docker rm testsuiteproject_synbiohub_1 -docker rm testsuiteproject_synbiohubFrontend -docker rm testsuiteproject_synbiohubbackend -docker rm testsuiteproject_explorer_1 -docker rm testsuiteproject_elasticsearch_1 -docker rm testsuiteproject_autoheal_1 -docker rm testsuiteproject_virtuoso_1 +if docker compose version >/dev/null 2>&1; then + COMPOSE=(docker compose) +else + COMPOSE=(docker-compose) +fi - -docker volume rm testsuiteproject_esdata -docker volume rm testsuiteproject_explorer -docker volume rm testsuiteproject_sbh -docker volume rm testsuiteproject_virtuoso-db +# Tear down Compose v2 project (hyphenated container names) and volumes. +"${COMPOSE[@]}" -f "$COMPOSE_FILE" -p "$PROJECT" down -v --remove-orphans 2>/dev/null || true