From 8e3c3f0c638e3e17bec60bc168817fa71d79f4f2 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Thu, 5 Jun 2025 10:12:14 -0400 Subject: [PATCH 001/105] Create LICENSE --- java/cuvs-lucene/LICENSE | 201 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 java/cuvs-lucene/LICENSE diff --git a/java/cuvs-lucene/LICENSE b/java/cuvs-lucene/LICENSE new file mode 100644 index 0000000000..1a89b9054d --- /dev/null +++ b/java/cuvs-lucene/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2020 NVIDIA Corporation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 8ec0201c29c7cb9732058a94f7ee67f04f5d5155 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 2 Jul 2025 16:26:09 -0400 Subject: [PATCH 002/105] Initial commit --- java/cuvs-lucene/LICENSE | 29 +- java/cuvs-lucene/README.md | 26 + java/cuvs-lucene/pom.xml | 118 +++++ .../sandbox/vectorsearch/CuVSCodec.java | 56 ++ .../sandbox/vectorsearch/CuVSFieldWriter.java | 80 +++ .../sandbox/vectorsearch/CuVSIndex.java | 119 +++++ .../vectorsearch/CuVSKnnFloatVectorQuery.java | 53 ++ .../sandbox/vectorsearch/CuVSSegmentFile.java | 63 +++ .../vectorsearch/CuVSVectorsFormat.java | 144 +++++ .../vectorsearch/CuVSVectorsReader.java | 487 +++++++++++++++++ .../vectorsearch/CuVSVectorsWriter.java | 495 ++++++++++++++++++ .../vectorsearch/FilterCuVSProvider.java | 72 +++ .../FilterCuVSServiceProvider.java | 28 + .../vectorsearch/IndexInputInputStream.java | 60 +++ .../vectorsearch/IndexOutputOutputStream.java | 70 +++ .../vectorsearch/PerLeafCuVSKnnCollector.java | 96 ++++ .../sandbox/vectorsearch/package-info.java | 19 + .../org.apache.lucene.codecs.KnnVectorsFormat | 18 + .../cuvs-lucene/src/main/resources/log4j2.xml | 14 + .../lucene/sandbox/vectorsearch/TestCuVS.java | 208 ++++++++ .../vectorsearch/TestCuVSVectorsFormat.java | 131 +++++ .../TestIndexOutputOutputStream.java | 102 ++++ 22 files changed, 2486 insertions(+), 2 deletions(-) create mode 100644 java/cuvs-lucene/README.md create mode 100644 java/cuvs-lucene/pom.xml create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsFormat.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java create mode 100644 java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java create mode 100644 java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat create mode 100644 java/cuvs-lucene/src/main/resources/log4j2.xml create mode 100644 java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java create mode 100644 java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java create mode 100644 java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java diff --git a/java/cuvs-lucene/LICENSE b/java/cuvs-lucene/LICENSE index 1a89b9054d..534e44ea18 100644 --- a/java/cuvs-lucene/LICENSE +++ b/java/cuvs-lucene/LICENSE @@ -178,7 +178,7 @@ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2020 NVIDIA Corporation + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -199,3 +199,28 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +========================================================================== +The following license applies to the JQuery JavaScript library +-------------------------------------------------------------------------- +Copyright (c) 2010 John Resig, http://jquery.com/ + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/java/cuvs-lucene/README.md b/java/cuvs-lucene/README.md new file mode 100644 index 0000000000..482ab785dc --- /dev/null +++ b/java/cuvs-lucene/README.md @@ -0,0 +1,26 @@ +# Lucene CuVS Integration + +This is an integration for [CuVS](https://github.com/rapidsai/cuvs), GPU accelerated vector search library from NVIDIA (formerly part of [Raft](https://github.com/rapidsai/raft)), into [Apache Lucene](https://github.com/apache/lucene). + +## Overview + +The CuVS library is plugged in as a new KnnVectorFormat via a custom codec. + +![Architecture](lucene-cuvs-architecture.png "Lucene CuVS Architecture") + +> :warning: This is not production ready yet. + +## Building + +Install NVIDIA drivers, CUDA 12.8, Maven 3.9.6+ and JDK 22. + + mvn clean compile package + +The artifacts would be built and available in target/ folder. + +## Contributors + +* Vivek Narang, SearchScale +* Ishan Chattopadhyaya, SearchScale & Committer, Apache Lucene & Solr +* Puneet Ahuja, SearchScale +* Corey Nolet, NVIDIA diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml new file mode 100644 index 0000000000..3c1b686ecc --- /dev/null +++ b/java/cuvs-lucene/pom.xml @@ -0,0 +1,118 @@ + + 4.0.0 + com.searchscale.lucene.vectorsearch + lucene-cuvs + 0.0.1-SNAPSHOT + lucene-cuvs + jar + + + 21 + 21 + + + + + cuvs-java + SearchScale Maven + https://maven.searchscale.com/snapshots + + + + + + org.apache.lucene + lucene-core + 10.2.0 + + + org.apache.lucene + lucene-codecs + 10.2.0 + test + + + org.apache.lucene + lucene-test-framework + 10.2.0 + test + + + com.opencsv + opencsv + 5.3 + + + commons-io + commons-io + 2.15.1 + + + + com.github.fommil + jniloader + 1.1 + + + + com.fasterxml.jackson.core + jackson-databind + 2.17.0 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-csv + 2.17.0 + + + com.nvidia.cuvs + cuvs-java + 25.08.0 + + + + + + diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java new file mode 100644 index 0000000000..c4d032fe6b --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import com.nvidia.cuvs.LibraryException; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene101.Lucene101Codec; +import org.apache.lucene.sandbox.vectorsearch.CuVSVectorsWriter.IndexType; + +/** CuVS based codec for GPU based vector search */ +public class CuVSCodec extends FilterCodec { + + public CuVSCodec() { + this("CuVSCodec", new Lucene101Codec()); + } + + public CuVSCodec(String name, Codec delegate) { + super(name, delegate); + KnnVectorsFormat format; + try { + format = new CuVSVectorsFormat(1, 128, 64, IndexType.CAGRA); + setKnnFormat(format); + } catch (LibraryException ex) { + Logger log = Logger.getLogger(CuVSCodec.class.getName()); + log.severe("Couldn't load native library, possible classloader issue. " + ex.getMessage()); + } + } + + KnnVectorsFormat knnFormat = null; + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return knnFormat; + } + + public void setKnnFormat(KnnVectorsFormat format) { + this.knnFormat = format; + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java new file mode 100644 index 0000000000..61b8f08792 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import java.io.IOException; +import java.util.List; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.util.RamUsageEstimator; + +/** CuVS based fields writer */ +/*package-private*/ class CuVSFieldWriter extends KnnFieldVectorsWriter { + + private static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(CuVSFieldWriter.class); + + private final FieldInfo fieldInfo; + private final FlatFieldVectorsWriter flatFieldVectorsWriter; + private int lastDocID = -1; + + public CuVSFieldWriter( + FieldInfo fieldInfo, FlatFieldVectorsWriter flatFieldVectorsWriter) { + this.fieldInfo = fieldInfo; + this.flatFieldVectorsWriter = flatFieldVectorsWriter; + } + + @Override + public void addValue(int docID, float[] vectorValue) throws IOException { + if (docID == lastDocID) { + throw new IllegalArgumentException( + "VectorValuesField \"" + + fieldInfo.name + + "\" appears more than once in this document (only one value is allowed per field)"); + } + flatFieldVectorsWriter.addValue(docID, vectorValue); + } + + List getVectors() { + return flatFieldVectorsWriter.getVectors(); + } + + FieldInfo fieldInfo() { + return fieldInfo; + } + + DocsWithFieldSet getDocsWithFieldSet() { + return flatFieldVectorsWriter.getDocsWithFieldSet(); + } + + @Override + public float[] copyValue(float[] vectorValue) { + throw new UnsupportedOperationException(); + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + flatFieldVectorsWriter.ramBytesUsed(); + } + + @Override + public String toString() { + return "CuVSFieldWriter[field name=" + fieldInfo.name + ", number=" + fieldInfo.number + "]"; + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java new file mode 100644 index 0000000000..d0cfe86d70 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java @@ -0,0 +1,119 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsReader.handleThrowable; + +import com.nvidia.cuvs.BruteForceIndex; +import com.nvidia.cuvs.CagraIndex; +import com.nvidia.cuvs.HnswIndex; +import java.io.Closeable; +import java.io.IOException; +import java.util.Objects; + +/** This class holds references to the actual CuVS Index (Cagra, Brute force, etc.) */ +public class CuVSIndex implements Closeable { + private final CagraIndex cagraIndex; + private final BruteForceIndex bruteforceIndex; + private final HnswIndex hnswIndex; + + private int maxDocs; + private String fieldName; + private String segmentName; + private volatile boolean closed; + + public CuVSIndex( + String segmentName, + String fieldName, + CagraIndex cagraIndex, + int maxDocs, + BruteForceIndex bruteforceIndex) { + this.cagraIndex = Objects.requireNonNull(cagraIndex); + this.bruteforceIndex = Objects.requireNonNull(bruteforceIndex); + this.fieldName = Objects.requireNonNull(fieldName); + this.segmentName = Objects.requireNonNull(segmentName); + if (maxDocs < 0) { + throw new IllegalArgumentException("negative maxDocs:" + maxDocs); + } + this.maxDocs = maxDocs; + this.hnswIndex = null; // TODO: + } + + public CuVSIndex(CagraIndex cagraIndex, BruteForceIndex bruteforceIndex, HnswIndex hnswIndex) { + this.cagraIndex = cagraIndex; + this.bruteforceIndex = bruteforceIndex; + this.hnswIndex = hnswIndex; + } + + public CagraIndex getCagraIndex() { + ensureOpen(); + return cagraIndex; + } + + public BruteForceIndex getBruteforceIndex() { + ensureOpen(); + return bruteforceIndex; + } + + public HnswIndex getHNSWIndex() { + ensureOpen(); + return hnswIndex; + } + + public String getFieldName() { + return fieldName; + } + + public String getSegmentName() { + return segmentName; + } + + public int getMaxDocs() { + return maxDocs; + } + + private void ensureOpen() { + if (closed) { + throw new IllegalStateException("index is closed"); + } + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + destroyIndices(); + } + + private void destroyIndices() throws IOException { + try { + if (cagraIndex != null) { + cagraIndex.destroyIndex(); + } + if (bruteforceIndex != null) { + bruteforceIndex.destroyIndex(); + } + if (hnswIndex != null) { + hnswIndex.destroyIndex(); + } + } catch (Throwable t) { + handleThrowable(t); + } + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java new file mode 100644 index 0000000000..2f6c636590 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import java.io.IOException; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.knn.KnnCollectorManager; +import org.apache.lucene.util.Bits; + +/** Query for CuVS */ +public class CuVSKnnFloatVectorQuery extends KnnFloatVectorQuery { + + private final int iTopK; + private final int searchWidth; + + public CuVSKnnFloatVectorQuery(String field, float[] target, int k, int iTopK, int searchWidth) { + super(field, target, k); + this.iTopK = iTopK; + this.searchWidth = searchWidth; + } + + @Override + protected TopDocs approximateSearch( + LeafReaderContext context, + Bits acceptDocs, + int visitedLimit, + KnnCollectorManager knnCollectorManager) + throws IOException { + + PerLeafCuVSKnnCollector results = new PerLeafCuVSKnnCollector(k, iTopK, searchWidth); + + LeafReader reader = context.reader(); + reader.searchNearestVectors(field, this.getTargetCopy(), results, null); + return results.topDocs(); + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java new file mode 100644 index 0000000000..9b12cdf610 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; +import java.util.zip.Deflater; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +/** Methods to deal with a CuVS composite file inside a segment */ +/*package-private*/ class CuVSSegmentFile implements AutoCloseable { + private final ZipOutputStream zos; + + private Set filesAdded = new HashSet(); + + public CuVSSegmentFile(OutputStream out) { + zos = new ZipOutputStream(out); + zos.setLevel(Deflater.NO_COMPRESSION); + } + + protected Logger log = Logger.getLogger(getClass().getName()); + + public void addFile(String name, byte[] bytes) throws IOException { + /*log.info( + "Writing the file: " + + name + + ", size=" + + bytes.length);*/ + ZipEntry indexFileZipEntry = new ZipEntry(name); + zos.putNextEntry(indexFileZipEntry); + zos.write(bytes, 0, bytes.length); + zos.closeEntry(); + filesAdded.add(name); + } + + public Set getFilesAdded() { + return Collections.unmodifiableSet(filesAdded); + } + + @Override + public void close() throws IOException { + zos.close(); + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsFormat.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsFormat.java new file mode 100644 index 0000000000..f5bd1bd909 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsFormat.java @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import com.nvidia.cuvs.CuVSResources; +import com.nvidia.cuvs.LibraryException; +import java.io.IOException; +import java.util.logging.Logger; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.sandbox.vectorsearch.CuVSVectorsWriter.IndexType; + +/** CuVS based KnnVectorsFormat for GPU acceleration */ +public class CuVSVectorsFormat extends KnnVectorsFormat { + + private static final Logger LOG = Logger.getLogger(CuVSVectorsFormat.class.getName()); + + // TODO: fix Lucene version in name, to the final targeted release, if any + static final String CUVS_META_CODEC_NAME = "Lucene102CuVSVectorsFormatMeta"; + static final String CUVS_META_CODEC_EXT = "vemc"; // ""cagmf"; + static final String CUVS_INDEX_CODEC_NAME = "Lucene102CuVSVectorsFormatIndex"; + static final String CUVS_INDEX_EXT = "vcag"; + + static final int VERSION_START = 0; + static final int VERSION_CURRENT = VERSION_START; + + public static final int DEFAULT_WRITER_THREADS = 32; + public static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; + public static final int DEFAULT_GRAPH_DEGREE = 64; + public static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; + + static CuVSResources resources = cuVSResourcesOrNull(); + + /** The format for storing, reading, and merging raw vectors on disk. */ + private static final FlatVectorsFormat flatVectorsFormat = + new Lucene99FlatVectorsFormat(DefaultFlatVectorScorer.INSTANCE); + + final int maxDimensions = 4096; + final int cuvsWriterThreads; + final int intGraphDegree; + final int graphDegree; + final CuVSVectorsWriter.IndexType indexType; // the index type to build, when writing + + /** + * Creates a CuVSVectorsFormat, with default values. + * + * @throws LibraryException if the native library fails to load + */ + public CuVSVectorsFormat() { + this( + DEFAULT_WRITER_THREADS, + DEFAULT_INTERMEDIATE_GRAPH_DEGREE, + DEFAULT_GRAPH_DEGREE, + DEFAULT_INDEX_TYPE); + } + + /** + * Creates a CuVSVectorsFormat, with the given threads, graph degree, etc. + * + * @throws LibraryException if the native library fails to load + */ + public CuVSVectorsFormat( + int cuvsWriterThreads, int intGraphDegree, int graphDegree, IndexType indexType) { + super("CuVSVectorsFormat"); + this.cuvsWriterThreads = cuvsWriterThreads; + this.intGraphDegree = intGraphDegree; + this.graphDegree = graphDegree; + this.indexType = indexType; + } + + private static CuVSResources cuVSResourcesOrNull() { + try { + resources = CuVSResources.create(); + return resources; + } catch (UnsupportedOperationException uoe) { + LOG.warning("cuvs is not supported on this platform or java version: " + uoe.getMessage()); + } catch (Throwable t) { + if (t instanceof ExceptionInInitializerError ex) { + t = ex.getCause(); + } + LOG.warning("Exception occurred during creation of cuvs resources. " + t); + } + return null; + } + + /** Tells whether the platform supports cuvs. */ + public static boolean supported() { + return resources != null; + } + + private static void checkSupported() { + if (!supported()) { + throw new UnsupportedOperationException(); + } + } + + @Override + public CuVSVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + checkSupported(); + var flatWriter = flatVectorsFormat.fieldsWriter(state); + return new CuVSVectorsWriter( + state, cuvsWriterThreads, intGraphDegree, graphDegree, indexType, resources, flatWriter); + } + + @Override + public CuVSVectorsReader fieldsReader(SegmentReadState state) throws IOException { + checkSupported(); + var flatReader = flatVectorsFormat.fieldsReader(state); + return new CuVSVectorsReader(state, resources, flatReader); + } + + @Override + public int getMaxDimensions(String fieldName) { + return maxDimensions; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder("CuVSVectorsFormat("); + sb.append("cuvsWriterThreads=").append(cuvsWriterThreads); + sb.append("intGraphDegree=").append(intGraphDegree); + sb.append("graphDegree=").append(graphDegree); + sb.append("resources=").append(resources); + sb.append(")"); + return sb.toString(); + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java new file mode 100644 index 0000000000..cfb59121e3 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java @@ -0,0 +1,487 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_INDEX_CODEC_NAME; +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_INDEX_EXT; +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_META_CODEC_EXT; +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_META_CODEC_NAME; +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.VERSION_CURRENT; +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.VERSION_START; + +import com.nvidia.cuvs.BruteForceIndex; +import com.nvidia.cuvs.BruteForceQuery; +import com.nvidia.cuvs.CagraIndex; +import com.nvidia.cuvs.CagraQuery; +import com.nvidia.cuvs.CagraSearchParams; +import com.nvidia.cuvs.CuVSResources; +import com.nvidia.cuvs.HnswIndex; +import com.nvidia.cuvs.HnswIndexParams; +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.logging.Logger; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.index.ByteVectorValues; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.internal.hppc.IntObjectHashMap; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.ReadAdvice; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.hnsw.IntToIntFunction; + +/** KnnVectorsReader instance associated with CuVS format */ +public class CuVSVectorsReader extends KnnVectorsReader { + + @SuppressWarnings("unused") + private static final Logger log = Logger.getLogger(CuVSVectorsReader.class.getName()); + + private final CuVSResources resources; + private final FlatVectorsReader flatVectorsReader; // for reading the raw vectors + private final FieldInfos fieldInfos; + private final IntObjectHashMap fields; + private final IntObjectHashMap cuvsIndices; + private final IndexInput cuvsIndexInput; + + public CuVSVectorsReader( + SegmentReadState state, CuVSResources resources, FlatVectorsReader flatReader) + throws IOException { + this.resources = resources; + this.flatVectorsReader = flatReader; + this.fieldInfos = state.fieldInfos; + this.fields = new IntObjectHashMap<>(); + + String metaFileName = + IndexFileNames.segmentFileName( + state.segmentInfo.name, state.segmentSuffix, CUVS_META_CODEC_EXT); + boolean success = false; + int versionMeta = -1; + try (ChecksumIndexInput meta = state.directory.openChecksumInput(metaFileName)) { + Throwable priorException = null; + try { + versionMeta = + CodecUtil.checkIndexHeader( + meta, + CUVS_META_CODEC_NAME, + VERSION_START, + VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix); + readFields(meta); + } catch (Throwable exception) { + priorException = exception; + } finally { + CodecUtil.checkFooter(meta, priorException); + } + var ioContext = state.context.withReadAdvice(ReadAdvice.SEQUENTIAL); + cuvsIndexInput = openCuVSInput(state, versionMeta, ioContext); + cuvsIndices = loadCuVSIndices(); + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + private static IndexInput openCuVSInput( + SegmentReadState state, int versionMeta, IOContext context) throws IOException { + String fileName = + IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, CUVS_INDEX_EXT); + IndexInput in = state.directory.openInput(fileName, context); + boolean success = false; + try { + int versionVectorData = + CodecUtil.checkIndexHeader( + in, + CUVS_INDEX_CODEC_NAME, + VERSION_START, + VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix); + checkVersion(versionMeta, versionVectorData, in); + CodecUtil.retrieveChecksum(in); + success = true; + return in; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(in); + } + } + } + + private void validateFieldEntry(FieldInfo info, FieldEntry fieldEntry) { + int dimension = info.getVectorDimension(); + if (dimension != fieldEntry.dims()) { + throw new IllegalStateException( + "Inconsistent vector dimension for field=\"" + + info.name + + "\"; " + + dimension + + " != " + + fieldEntry.dims()); + } + } + + private void readFields(ChecksumIndexInput meta) throws IOException { + for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) { + FieldInfo info = fieldInfos.fieldInfo(fieldNumber); + if (info == null) { + throw new CorruptIndexException("Invalid field number: " + fieldNumber, meta); + } + FieldEntry fieldEntry = readField(meta, info); + validateFieldEntry(info, fieldEntry); + fields.put(info.number, fieldEntry); + } + } + + // List of vector similarity functions. This list is defined here, in order + // to avoid an undesirable dependency on the declaration and order of values + // in VectorSimilarityFunction. The list values and order must be identical + // to that of {@link o.a.l.c.l.Lucene94FieldInfosFormat#SIMILARITY_FUNCTIONS}. + static final List SIMILARITY_FUNCTIONS = + List.of( + VectorSimilarityFunction.EUCLIDEAN, + VectorSimilarityFunction.DOT_PRODUCT, + VectorSimilarityFunction.COSINE, + VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT); + + static VectorSimilarityFunction readSimilarityFunction(DataInput input) throws IOException { + int i = input.readInt(); + if (i < 0 || i >= SIMILARITY_FUNCTIONS.size()) { + throw new IllegalArgumentException("invalid distance function: " + i); + } + return SIMILARITY_FUNCTIONS.get(i); + } + + static VectorEncoding readVectorEncoding(DataInput input) throws IOException { + int encodingId = input.readInt(); + if (encodingId < 0 || encodingId >= VectorEncoding.values().length) { + throw new CorruptIndexException("Invalid vector encoding id: " + encodingId, input); + } + return VectorEncoding.values()[encodingId]; + } + + private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException { + VectorEncoding vectorEncoding = readVectorEncoding(input); + VectorSimilarityFunction similarityFunction = readSimilarityFunction(input); + if (similarityFunction != info.getVectorSimilarityFunction()) { + throw new IllegalStateException( + "Inconsistent vector similarity function for field=\"" + + info.name + + "\"; " + + similarityFunction + + " != " + + info.getVectorSimilarityFunction()); + } + return FieldEntry.readEntry(input, vectorEncoding, info.getVectorSimilarityFunction()); + } + + private FieldEntry getFieldEntry(String field, VectorEncoding expectedEncoding) { + final FieldInfo info = fieldInfos.fieldInfo(field); + final FieldEntry fieldEntry; + if (info == null || (fieldEntry = fields.get(info.number)) == null) { + throw new IllegalArgumentException("field=\"" + field + "\" not found"); + } + if (fieldEntry.vectorEncoding != expectedEncoding) { + throw new IllegalArgumentException( + "field=\"" + + field + + "\" is encoded as: " + + fieldEntry.vectorEncoding + + " expected: " + + expectedEncoding); + } + return fieldEntry; + } + + private IntObjectHashMap loadCuVSIndices() throws IOException { + var indices = new IntObjectHashMap(); + for (var e : fields) { + var fieldEntry = e.value; + int fieldNumber = e.key; + var cuvsIndex = loadCuVSIndex(fieldEntry); + indices.put(fieldNumber, cuvsIndex); + } + return indices; + } + + private CuVSIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { + CagraIndex cagraIndex = null; + BruteForceIndex bruteForceIndex = null; + HnswIndex hnswIndex = null; + + try { + long len = fieldEntry.cagraIndexLength(); + if (len > 0) { + long off = fieldEntry.cagraIndexOffset(); + try (var slice = cuvsIndexInput.slice("cagra index", off, len); + var in = new IndexInputInputStream(slice)) { + cagraIndex = CagraIndex.newBuilder(resources).from(in).build(); + } + } + + len = fieldEntry.bruteForceIndexLength(); + if (len > 0) { + long off = fieldEntry.bruteForceIndexOffset(); + try (var slice = cuvsIndexInput.slice("bf index", off, len); + var in = new IndexInputInputStream(slice)) { + bruteForceIndex = BruteForceIndex.newBuilder(resources).from(in).build(); + } + } + + len = fieldEntry.hnswIndexLength(); + if (len > 0) { + long off = fieldEntry.hnswIndexOffset(); + try (var slice = cuvsIndexInput.slice("hnsw index", off, len); + var in = new IndexInputInputStream(slice)) { + var params = new HnswIndexParams.Builder().build(); + hnswIndex = HnswIndex.newBuilder(resources).withIndexParams(params).from(in).build(); + } + } + } catch (Throwable t) { + handleThrowable(t); + } + return new CuVSIndex(cagraIndex, bruteForceIndex, hnswIndex); + } + + @Override + public void close() throws IOException { + var closeableStream = + Stream.concat( + Stream.of(flatVectorsReader, cuvsIndexInput), + stream(cuvsIndices.values().iterator()).map(cursor -> cursor.value)); + IOUtils.close(closeableStream::iterator); + } + + static Stream stream(Iterator iterator) { + return StreamSupport.stream(((Iterable) () -> iterator).spliterator(), false); + } + + @Override + public void checkIntegrity() throws IOException { + // TODO: Pending implementation + } + + @Override + public FloatVectorValues getFloatVectorValues(String field) throws IOException { + return flatVectorsReader.getFloatVectorValues(field); + } + + @Override + public ByteVectorValues getByteVectorValues(String field) { + throw new UnsupportedOperationException("byte vectors not supported"); + } + + /** Native float to float function */ + public interface FloatToFloatFunction { + float apply(float v); + } + + static long[] bitsToLongArray(Bits bits) { + if (bits instanceof FixedBitSet fixedBitSet) { + return fixedBitSet.getBits(); + } else { + return FixedBitSet.copyOf(bits).getBits(); + } + } + + static FloatToFloatFunction getScoreNormalizationFunc(VectorSimilarityFunction sim) { + // TODO: check for different similarities + return score -> (1f / (1f + score)); + } + + // This is a hack - https://github.com/rapidsai/cuvs/issues/696 + static final int FILTER_OVER_SAMPLE = 10; + + @Override + public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) + throws IOException { + var fieldEntry = getFieldEntry(field, VectorEncoding.FLOAT32); + if (fieldEntry.count() == 0 || knnCollector.k() == 0) { + return; + } + + var fieldNumber = fieldInfos.fieldInfo(field).number; + // log.info("fieldNumber=" + fieldNumber + ", fieldEntry.count()=" + fieldEntry.count()); + + CuVSIndex cuvsIndex = cuvsIndices.get(fieldNumber); + if (cuvsIndex == null) { + throw new IllegalStateException("not index found for field:" + field); + } + + int collectorTopK = knnCollector.k(); + if (acceptDocs != null) { + collectorTopK = knnCollector.k() * FILTER_OVER_SAMPLE; + } + final int topK = Math.min(collectorTopK, fieldEntry.count()); + assert topK > 0 : "Expected topK > 0, got:" + topK; + + Map result; + if (knnCollector.k() <= 1024 && cuvsIndex.getCagraIndex() != null) { + // log.info("searching cagra index"); + CagraSearchParams searchParams = + new CagraSearchParams.Builder(resources) + .withItopkSize(topK) // TODO: params + .withSearchWidth(1) + .build(); + + var query = + new CagraQuery.Builder() + .withTopK(topK) + .withSearchParams(searchParams) + // we don't use ord to doc mapping, https://github.com/rapidsai/cuvs/issues/699 + .withMapping(null) + .withQueryVectors(new float[][] {target}) + .build(); + + CagraIndex cagraIndex = cuvsIndex.getCagraIndex(); + List> searchResult = null; + try { + searchResult = cagraIndex.search(query).getResults(); + } catch (Throwable t) { + handleThrowable(t); + } + // List expected to have only one entry because of single query "target". + assert searchResult.size() == 1; + result = searchResult.getFirst(); + } else { + BruteForceIndex bruteforceIndex = cuvsIndex.getBruteforceIndex(); + assert bruteforceIndex != null; + // log.info("searching brute index, with actual topK=" + topK); + var queryBuilder = + new BruteForceQuery.Builder().withQueryVectors(new float[][] {target}).withTopK(topK); + BruteForceQuery query = queryBuilder.build(); + + List> searchResult = null; + try { + searchResult = bruteforceIndex.search(query).getResults(); + } catch (Throwable t) { + handleThrowable(t); + } + assert searchResult.size() == 1; + result = searchResult.getFirst(); + } + assert result != null; + + final var rawValues = flatVectorsReader.getFloatVectorValues(field); + final Bits acceptedOrds = rawValues.getAcceptOrds(acceptDocs); + final var ordToDocFunction = (IntToIntFunction) rawValues::ordToDoc; + final var scoreCorrectionFunction = getScoreNormalizationFunc(fieldEntry.similarityFunction); + + for (var entry : result.entrySet()) { + int ord = entry.getKey(); + float score = entry.getValue(); + if (acceptedOrds == null || acceptedOrds.get(ord)) { + if (knnCollector.earlyTerminated()) { + break; + } + assert ord >= 0 : "unexpected ord: " + ord; + int doc = ordToDocFunction.apply(ord); + float correctedScore = scoreCorrectionFunction.apply(score); + knnCollector.incVisitedCount(1); + knnCollector.collect(doc, correctedScore); + } + } + } + + @Override + public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) + throws IOException { + throw new UnsupportedOperationException("byte vectors not supported"); + } + + record FieldEntry( + VectorEncoding vectorEncoding, + VectorSimilarityFunction similarityFunction, + int dims, + int count, + long cagraIndexOffset, + long cagraIndexLength, + long bruteForceIndexOffset, + long bruteForceIndexLength, + long hnswIndexOffset, + long hnswIndexLength) { + + static FieldEntry readEntry( + IndexInput input, + VectorEncoding vectorEncoding, + VectorSimilarityFunction similarityFunction) + throws IOException { + var dims = input.readInt(); + var count = input.readInt(); + var cagraIndexOffset = input.readVLong(); + var cagraIndexLength = input.readVLong(); + var bruteForceIndexOffset = input.readVLong(); + var bruteForceIndexLength = input.readVLong(); + var hnswIndexOffset = input.readVLong(); + var hnswIndexLength = input.readVLong(); + return new FieldEntry( + vectorEncoding, + similarityFunction, + dims, + count, + cagraIndexOffset, + cagraIndexLength, + bruteForceIndexOffset, + bruteForceIndexLength, + hnswIndexOffset, + hnswIndexLength); + } + } + + static void checkVersion(int versionMeta, int versionVectorData, IndexInput in) + throws CorruptIndexException { + if (versionMeta != versionVectorData) { + throw new CorruptIndexException( + "Format versions mismatch: meta=" + + versionMeta + + ", " + + CUVS_META_CODEC_NAME + + "=" + + versionVectorData, + in); + } + } + + static void handleThrowable(Throwable t) throws IOException { + switch (t) { + case IOException ioe -> throw ioe; + case Error error -> throw error; + case RuntimeException re -> throw re; + case null, default -> throw new RuntimeException("UNEXPECTED: exception type", t); + } + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java new file mode 100644 index 0000000000..29d32b9e6e --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java @@ -0,0 +1,495 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; +import static org.apache.lucene.index.VectorEncoding.FLOAT32; +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_INDEX_CODEC_NAME; +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_INDEX_EXT; +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_META_CODEC_EXT; +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_META_CODEC_NAME; +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.VERSION_CURRENT; +import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsReader.handleThrowable; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; + +import com.nvidia.cuvs.BruteForceIndex; +import com.nvidia.cuvs.BruteForceIndexParams; +import com.nvidia.cuvs.CagraIndex; +import com.nvidia.cuvs.CagraIndexParams; +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.CuVSResources; +import com.nvidia.cuvs.Dataset; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.logging.Logger; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Sorter; +import org.apache.lucene.index.Sorter.DocMap; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.InfoStream; + +/** KnnVectorsWriter for CuVS, responsible for merge and flush of vectors into GPU */ +public class CuVSVectorsWriter extends KnnVectorsWriter { + + private static final long SHALLOW_RAM_BYTES_USED = shallowSizeOfInstance(CuVSVectorsWriter.class); + + @SuppressWarnings("unused") + private static final Logger log = Logger.getLogger(CuVSVectorsWriter.class.getName()); + + /** The name of the CUVS component for the info-stream * */ + public static final String CUVS_COMPONENT = "CUVS"; + + // The minimum number of vectors in the dataset required before + // we attempt to build a Cagra index + static final int MIN_CAGRA_INDEX_SIZE = 2; + + private final int cuvsWriterThreads; + private final int intGraphDegree; + private final int graphDegree; + + private final CuVSResources resources; + private final IndexType indexType; + + private final FlatVectorsWriter flatVectorsWriter; // for writing the raw vectors + private final List fields = new ArrayList<>(); + private final IndexOutput meta, cuvsIndex; + private final InfoStream infoStream; + private boolean finished; + + /** The CuVS index Type. */ + public enum IndexType { + /** Builds a Cagra index. */ + CAGRA(true, false, false), + /** Builds a Brute Force index. */ + BRUTE_FORCE(false, true, false), + /** Builds an HSNW index - suitable for searching on CPU. */ + HNSW(false, false, true), + /** Builds a Cagra and a Brute Force index. */ + CAGRA_AND_BRUTE_FORCE(true, true, false); + private final boolean cagra, bruteForce, hnsw; + + IndexType(boolean cagra, boolean bruteForce, boolean hnsw) { + this.cagra = cagra; + this.bruteForce = bruteForce; + this.hnsw = hnsw; + } + + public boolean cagra() { + return cagra; + } + + public boolean bruteForce() { + return bruteForce; + } + + public boolean hnsw() { + return hnsw; + } + } + + public CuVSVectorsWriter( + SegmentWriteState state, + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + IndexType indexType, + CuVSResources resources, + FlatVectorsWriter flatVectorsWriter) + throws IOException { + super(); + this.indexType = indexType; + this.cuvsWriterThreads = cuvsWriterThreads; + this.intGraphDegree = intGraphDegree; + this.graphDegree = graphDegree; + this.resources = resources; + this.flatVectorsWriter = flatVectorsWriter; + this.infoStream = state.infoStream; + + String metaFileName = + IndexFileNames.segmentFileName( + state.segmentInfo.name, state.segmentSuffix, CUVS_META_CODEC_EXT); + String cagraFileName = + IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, CUVS_INDEX_EXT); + + boolean success = false; + try { + meta = state.directory.createOutput(metaFileName, state.context); + cuvsIndex = state.directory.createOutput(cagraFileName, state.context); + CodecUtil.writeIndexHeader( + meta, + CUVS_META_CODEC_NAME, + VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix); + CodecUtil.writeIndexHeader( + cuvsIndex, + CUVS_INDEX_CODEC_NAME, + VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix); + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + @Override + public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { + var encoding = fieldInfo.getVectorEncoding(); + if (encoding != FLOAT32) { + throw new IllegalArgumentException("expected float32, got:" + encoding); + } + var writer = Objects.requireNonNull(flatVectorsWriter.addField(fieldInfo)); + @SuppressWarnings("unchecked") + var flatWriter = (FlatFieldVectorsWriter) writer; + var cuvsFieldWriter = new CuVSFieldWriter(fieldInfo, flatWriter); + fields.add(cuvsFieldWriter); + return writer; + } + + static String indexMsg(int size, int... args) { + StringBuilder sb = new StringBuilder("cagra index params"); + sb.append(": size=").append(size); + sb.append(", intGraphDegree=").append(args[0]); + sb.append(", actualIntGraphDegree=").append(args[1]); + sb.append(", graphDegree=").append(args[2]); + sb.append(", actualGraphDegree=").append(args[3]); + return sb.toString(); + } + + private CagraIndexParams cagraIndexParams(int size) { + if (size < 2) { + // https://github.com/rapidsai/cuvs/issues/666 + throw new IllegalArgumentException("cagra index must be greater than 2"); + } + // var minIntGraphDegree = Math.min(intGraphDegree, size - 1); + // var minGraphDegree = Math.min(graphDegree, minIntGraphDegree); + // log.info(indexMsg(size, intGraphDegree, minIntGraphDegree, graphDegree, minGraphDegree)); + + return new CagraIndexParams.Builder() + .withNumWriterThreads(cuvsWriterThreads) + .withIntermediateGraphDegree(intGraphDegree) + .withGraphDegree(graphDegree) + .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) + .build(); + } + + static long nanosToMillis(long nanos) { + return Duration.ofNanos(nanos).toMillis(); + } + + private void info(String msg) { + if (infoStream.isEnabled(CUVS_COMPONENT)) { + infoStream.message(CUVS_COMPONENT, msg); + } + } + + private void writeCagraIndex(OutputStream os, Dataset dataset) throws Throwable { + if (dataset.size() < 2) { + throw new IllegalArgumentException(dataset.size() + " vectors, less than min [2] required"); + } + CagraIndexParams params = cagraIndexParams(dataset.size()); + long startTime = System.nanoTime(); + var index = + CagraIndex.newBuilder(resources).withDataset(dataset).withIndexParams(params).build(); + long elapsedMillis = nanosToMillis(System.nanoTime() - startTime); + info("Cagra index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); + Path tmpFile = Files.createTempFile(resources.tempDirectory(), "tmpindex", "cag"); + index.serialize(os, tmpFile); + index.destroyIndex(); + } + + private void writeBruteForceIndex(OutputStream os, Dataset dataset) throws Throwable { + BruteForceIndexParams params = + new BruteForceIndexParams.Builder() + .withNumWriterThreads(32) // TODO: Make this configurable later. + .build(); + long startTime = System.nanoTime(); + var index = + BruteForceIndex.newBuilder(resources).withIndexParams(params).withDataset(dataset).build(); + long elapsedMillis = nanosToMillis(System.nanoTime() - startTime); + info("bf index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); + index.serialize(os); + index.destroyIndex(); + } + + private void writeHNSWIndex(OutputStream os, Dataset dataset) throws Throwable { + if (dataset.size() < 2) { + throw new IllegalArgumentException(dataset.size() + " vectors, less than min [2] required"); + } + CagraIndexParams indexParams = cagraIndexParams(dataset.size()); + long startTime = System.nanoTime(); + var index = + CagraIndex.newBuilder(resources).withDataset(dataset).withIndexParams(indexParams).build(); + long elapsedMillis = nanosToMillis(System.nanoTime() - startTime); + info("HNSW index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); + Path tmpFile = Files.createTempFile("tmpindex", "hnsw"); + index.serializeToHNSW(os, tmpFile); + index.destroyIndex(); + } + + @Override + public void flush(int maxDoc, DocMap sortMap) throws IOException { + flatVectorsWriter.flush(maxDoc, sortMap); + for (var field : fields) { + if (sortMap == null) { + writeField(field); + } else { + writeSortingField(field, sortMap); + } + } + } + + private void writeField(CuVSFieldWriter fieldData) throws IOException { + // TODO: Argh! https://github.com/rapidsai/cuvs/issues/698 + List vectors = fieldData.getVectors(); + Dataset dataset = Dataset.create(vectors.size(), fieldData.fieldInfo().getVectorDimension()); + for (float[] vec : vectors) dataset.addVector(vec); + writeFieldInternal(fieldData.fieldInfo(), dataset); + } + + private void writeSortingField(CuVSFieldWriter fieldData, Sorter.DocMap sortMap) + throws IOException { + DocsWithFieldSet oldDocsWithFieldSet = fieldData.getDocsWithFieldSet(); + final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; // new ord to old ord + + mapOldOrdToNewOrd(oldDocsWithFieldSet, sortMap, null, new2OldOrd, null); + + float[][] oldVectors = fieldData.getVectors().toArray(float[][]::new); + Dataset dataset = + Dataset.create(fieldData.getVectors().size(), fieldData.fieldInfo().getVectorDimension()); + for (int i = 0; i < oldVectors.length; i++) { + float[] vec = oldVectors[new2OldOrd[i]]; + dataset.addVector(vec); + } + writeFieldInternal(fieldData.fieldInfo(), dataset); + } + + private void writeFieldInternal(FieldInfo fieldInfo, Dataset dataset) throws IOException { + if (dataset.size() == 0) { + writeEmpty(fieldInfo); + return; + } + long cagraIndexOffset, cagraIndexLength = 0L; + long bruteForceIndexOffset, bruteForceIndexLength = 0L; + long hnswIndexOffset, hnswIndexLength = 0L; + + // workaround for the minimum number of vectors for Cagra + IndexType indexType = + this.indexType.cagra() && dataset.size() < MIN_CAGRA_INDEX_SIZE + ? IndexType.BRUTE_FORCE + : this.indexType; + + try { + cagraIndexOffset = cuvsIndex.getFilePointer(); + if (indexType.cagra()) { + try { + var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); + writeCagraIndex(cagraIndexOutputStream, dataset); + } catch (Throwable t) { + handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); + // workaround for cuVS issue + indexType = IndexType.BRUTE_FORCE; + } + cagraIndexLength = cuvsIndex.getFilePointer() - cagraIndexOffset; + } + + bruteForceIndexOffset = cuvsIndex.getFilePointer(); + if (indexType.bruteForce()) { + var bruteForceIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); + writeBruteForceIndex(bruteForceIndexOutputStream, dataset); + bruteForceIndexLength = cuvsIndex.getFilePointer() - bruteForceIndexOffset; + } + + hnswIndexOffset = cuvsIndex.getFilePointer(); + if (indexType.hnsw()) { + var hnswIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); + if (dataset.size() > MIN_CAGRA_INDEX_SIZE) { + try { + writeHNSWIndex(hnswIndexOutputStream, dataset); + } catch (Throwable t) { + handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); + } + } + hnswIndexLength = cuvsIndex.getFilePointer() - hnswIndexOffset; + } + + // StringBuilder sb = new StringBuilder("writeField "); + // sb.append(": fieldInfo.name=").append(fieldInfo.name); + // sb.append(", fieldInfo.number=").append(fieldInfo.number); + // sb.append(", size=").append(vectors.length); + // sb.append(", cagraIndexLength=").append(cagraIndexLength); + // sb.append(", bruteForceIndexLength=").append(bruteForceIndexLength); + // sb.append(", hnswIndexLength=").append(hnswIndexLength); + // log.info(sb.toString()); + + writeMeta( + fieldInfo, + dataset.size(), + cagraIndexOffset, + cagraIndexLength, + bruteForceIndexOffset, + bruteForceIndexLength, + hnswIndexOffset, + hnswIndexLength); + } catch (Throwable t) { + handleThrowable(t); + } + } + + private void writeEmpty(FieldInfo fieldInfo) throws IOException { + writeMeta(fieldInfo, 0, 0L, 0L, 0L, 0L, 0L, 0L); + } + + private void writeMeta( + FieldInfo field, + int count, + long cagraIndexOffset, + long cagraIndexLength, + long bruteForceIndexOffset, + long bruteForceIndexLength, + long hnswIndexOffset, + long hnswIndexLength) + throws IOException { + meta.writeInt(field.number); + meta.writeInt(field.getVectorEncoding().ordinal()); + meta.writeInt(distFuncToOrd(field.getVectorSimilarityFunction())); + meta.writeInt(field.getVectorDimension()); + meta.writeInt(count); + meta.writeVLong(cagraIndexOffset); + meta.writeVLong(cagraIndexLength); + meta.writeVLong(bruteForceIndexOffset); + meta.writeVLong(bruteForceIndexLength); + meta.writeVLong(hnswIndexOffset); + meta.writeVLong(hnswIndexLength); + } + + static int distFuncToOrd(VectorSimilarityFunction func) { + for (int i = 0; i < SIMILARITY_FUNCTIONS.size(); i++) { + if (SIMILARITY_FUNCTIONS.get(i).equals(func)) { + return (byte) i; + } + } + throw new IllegalArgumentException("invalid distance function: " + func); + } + + // We currently ignore this, until cuVS supports tiered indices + private static final String CANNOT_GENERATE_CAGRA = + """ + Could not generate an intermediate CAGRA graph because the initial \ + kNN graph contains too many invalid or duplicated neighbor nodes. \ + This error can occur, for example, if too many overflows occur \ + during the norm computation between the dataset vectors\ + """; + + static void handleThrowableWithIgnore(Throwable t, String msg) throws IOException { + if (t.getMessage().contains(msg)) { + return; + } + handleThrowable(t); + } + + /** Copies the vector values into dst. Returns the actual number of vectors copied. */ + private static int getVectorData(FloatVectorValues floatVectorValues, Dataset dataset) + throws IOException { + DocsWithFieldSet docsWithField = new DocsWithFieldSet(); + int count = 0; + KnnVectorValues.DocIndexIterator iter = floatVectorValues.iterator(); + for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { + assert iter.index() == count; + // dst[iter.index()] = floatVectorValues.vectorValue(iter.index()); + dataset.addVector(floatVectorValues.vectorValue(iter.index())); // is this correct? + docsWithField.add(docV); + count++; + } + return docsWithField.cardinality(); + } + + @Override + public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + flatVectorsWriter.mergeOneField(fieldInfo, mergeState); + try { + final FloatVectorValues mergedVectorValues = + switch (fieldInfo.getVectorEncoding()) { + case BYTE -> throw new AssertionError("bytes not supported"); + case FLOAT32 -> + KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); + }; + + // Also will be replaced with the cuVS merge api + Dataset dataset = Dataset.create(mergedVectorValues.size(), mergedVectorValues.dimension()); + getVectorData(mergedVectorValues, dataset); + writeFieldInternal(fieldInfo, dataset); + } catch (Throwable t) { + handleThrowable(t); + } + } + + @Override + public void finish() throws IOException { + if (finished) { + throw new IllegalStateException("already finished"); + } + finished = true; + flatVectorsWriter.finish(); + + if (meta != null) { + // write end of fields marker + meta.writeInt(-1); + CodecUtil.writeFooter(meta); + } + if (cuvsIndex != null) { + CodecUtil.writeFooter(cuvsIndex); + } + } + + @Override + public void close() throws IOException { + IOUtils.close(meta, cuvsIndex, flatVectorsWriter); + } + + @Override + public long ramBytesUsed() { + long total = SHALLOW_RAM_BYTES_USED; + for (var field : fields) { + total += field.ramBytesUsed(); + } + return total; + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java new file mode 100644 index 0000000000..7b1d845320 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import com.nvidia.cuvs.BruteForceIndex; +import com.nvidia.cuvs.CagraIndex; +import com.nvidia.cuvs.CuVSResources; +import com.nvidia.cuvs.Dataset; +import com.nvidia.cuvs.HnswIndex; +import com.nvidia.cuvs.spi.CuVSProvider; +import java.nio.file.Path; + +/*package-private*/ class FilterCuVSProvider implements CuVSProvider { + + private final CuVSProvider delegate; + + FilterCuVSProvider(CuVSProvider delegate) { + this.delegate = delegate; + } + + @Override + public Path nativeLibraryPath() { + return CuVSProvider.TMPDIR; + } + + @Override + public CuVSResources newCuVSResources(Path tempPath) throws Throwable { + return delegate.newCuVSResources(tempPath); + } + + @Override + public BruteForceIndex.Builder newBruteForceIndexBuilder(CuVSResources cuVSResources) + throws UnsupportedOperationException { + return delegate.newBruteForceIndexBuilder(cuVSResources); + } + + @Override + public CagraIndex.Builder newCagraIndexBuilder(CuVSResources cuVSResources) + throws UnsupportedOperationException { + return delegate.newCagraIndexBuilder(cuVSResources); + } + + @Override + public HnswIndex.Builder newHnswIndexBuilder(CuVSResources cuVSResources) + throws UnsupportedOperationException { + return delegate.newHnswIndexBuilder(cuVSResources); + } + + @Override + public CagraIndex mergeCagraIndexes(CagraIndex[] arg0) throws Throwable { + return delegate.mergeCagraIndexes(arg0); + } + + @Override + public Dataset newDataset(int arg0, int arg1) throws UnsupportedOperationException { + return delegate.newDataset(arg0, arg1); + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java new file mode 100644 index 0000000000..eeb7b6895a --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import com.nvidia.cuvs.spi.CuVSProvider; +import com.nvidia.cuvs.spi.CuVSServiceProvider; + +/** A provider that creates instances of FilterCuVSProvider. */ +public class FilterCuVSServiceProvider extends CuVSServiceProvider { + @Override + public CuVSProvider get(CuVSProvider builtinProvider) { + return new FilterCuVSProvider(builtinProvider); + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java new file mode 100644 index 0000000000..4eb8ed558f --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import java.io.IOException; +import java.io.InputStream; +import org.apache.lucene.store.IndexInput; + +/** InputStream for reading from an IndexInput. */ +final class IndexInputInputStream extends InputStream { + + final IndexInput in; + long pos = 0; + final long limit; + + IndexInputInputStream(IndexInput in) { + this.in = in; + this.limit = in.length(); + } + + @Override + public int read() throws IOException { + if (pos >= limit) { + return -1; + } + pos++; + return in.readByte(); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + if (len <= 0) { + return 0; + } + if (pos >= limit) { + return -1; + } + long avail = limit - pos; + if (len > avail) { + len = (int) avail; + } + in.readBytes(b, off, len); + pos += len; + return len; + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java new file mode 100644 index 0000000000..ffb2b922e4 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import java.io.IOException; +import java.io.OutputStream; +import org.apache.lucene.store.IndexOutput; + +/** OutputStream for writing into an IndexOutput */ +final class IndexOutputOutputStream extends OutputStream { + + static final int DEFAULT_BUFFER_SIZE = 8192; + + final IndexOutput out; + final int bufferSize; + final byte[] buffer; + int idx; + + IndexOutputOutputStream(IndexOutput out) { + this(out, DEFAULT_BUFFER_SIZE); + } + + IndexOutputOutputStream(IndexOutput out, int bufferSize) { + this.out = out; + this.bufferSize = bufferSize; + this.buffer = new byte[bufferSize]; + } + + @Override + public void write(int b) throws IOException { + buffer[idx] = (byte) b; + idx++; + if (idx == bufferSize) { + flush(); + } + } + + @Override + public void write(byte[] b, int offset, int length) throws IOException { + if (idx != 0) { + flush(); + } + out.writeBytes(b, offset, length); + } + + @Override + public void flush() throws IOException { + out.writeBytes(buffer, 0, idx); + idx = 0; + } + + @Override + public void close() throws IOException { + this.flush(); + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java new file mode 100644 index 0000000000..90e0e1ff44 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import java.util.ArrayList; +import java.util.List; +import org.apache.lucene.search.KnnCollector; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.TotalHits; +import org.apache.lucene.search.knn.KnnSearchStrategy; + +/** KnnCollector for CuVS */ +/*package-private*/ class PerLeafCuVSKnnCollector implements KnnCollector { + + public List scoreDocs; + public int topK = 0; + public int iTopK = topK; // TODO getter, no setter + public int searchWidth = 1; // TODO getter, no setter + public int results = 0; + + public PerLeafCuVSKnnCollector(int topK, int iTopK, int searchWidth) { + super(); + this.topK = topK; + this.iTopK = iTopK; + this.searchWidth = searchWidth; + scoreDocs = new ArrayList(); + } + + @Override + public boolean earlyTerminated() { + // TODO: may need implementation + return false; + } + + @Override + public void incVisitedCount(int count) { + // TODO: may need implementation + } + + @Override + public long visitedCount() { + // TODO: may need implementation + return 0; + } + + @Override + public long visitLimit() { + // TODO: may need implementation + return 0; + } + + @Override + public int k() { + return topK; + } + + @Override + @SuppressWarnings("cast") + public boolean collect(int docId, float similarity) { + scoreDocs.add(new ScoreDoc(docId, similarity)); + return true; + } + + @Override + public float minCompetitiveSimilarity() { + // TODO: may need implementation + return 0; + } + + @Override + public TopDocs topDocs() { + return new TopDocs( + new TotalHits(scoreDocs.size(), TotalHits.Relation.EQUAL_TO), + scoreDocs.toArray(new ScoreDoc[scoreDocs.size()])); + } + + @Override + public KnnSearchStrategy getSearchStrategy() { + return KnnSearchStrategy.Patience.DEFAULT; + } +} diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java new file mode 100644 index 0000000000..86c56b909d --- /dev/null +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** CuVS based fast vector search */ +package org.apache.lucene.sandbox.vectorsearch; diff --git a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat new file mode 100644 index 0000000000..e14aa31324 --- /dev/null +++ b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat +org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat +com.searchscale.lucene.vectorsearch.CuVSVectorsFormat diff --git a/java/cuvs-lucene/src/main/resources/log4j2.xml b/java/cuvs-lucene/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..a810b6b71a --- /dev/null +++ b/java/cuvs-lucene/src/main/resources/log4j2.xml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java new file mode 100644 index 0000000000..a20a49be6f --- /dev/null +++ b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.analysis.MockTokenizer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.English; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "prints info from within cuvs") +public class TestCuVS extends LuceneTestCase { + + protected static Logger log = Logger.getLogger(TestCuVS.class.getName()); + + static final Codec codec = TestUtil.alwaysKnnVectorsFormat(new CuVSVectorsFormat()); + static IndexSearcher searcher; + static IndexReader reader; + static Directory directory; + + static int DATASET_SIZE_LIMIT = 1000; + static int DIMENSIONS_LIMIT = 2048; + static int NUM_QUERIES_LIMIT = 10; + static int TOP_K_LIMIT = 64; // TODO This fails beyond 64 + + public static float[][] dataset; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeTrue("cuvs not supported", CuVSVectorsFormat.supported()); + directory = newDirectory(); + + RandomIndexWriter writer = + new RandomIndexWriter( + random(), + directory, + newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy())); + + log.info("Merge Policy: " + writer.w.getConfig().getMergePolicy()); + + Random random = random(); + int datasetSize = random.nextInt(DATASET_SIZE_LIMIT) + 1; + int dimensions = random.nextInt(DIMENSIONS_LIMIT) + 1; + dataset = generateDataset(random, datasetSize, dimensions); + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(newTextField("field", English.intToEnglish(i), Field.Store.YES)); + boolean skipVector = + random.nextInt(10) < 0; // disable testing with holes for now, there's some bug. + if (!skipVector + || datasetSize < 100) { // about 10th of the documents shouldn't have a single vector + doc.add(new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + doc.add(new KnnFloatVectorField("vector2", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + } + + writer.addDocument(doc); + } + + reader = writer.getReader(); + searcher = newSearcher(reader); + writer.close(); + } + + @AfterClass + public static void afterClass() throws Exception { + if (reader != null) reader.close(); + if (directory != null) directory.close(); + searcher = null; + reader = null; + directory = null; + log.info("Test finished"); + } + + @Test + public void testVectorSearch() throws IOException { + Random random = random(); + int numQueries = random.nextInt(NUM_QUERIES_LIMIT) + 1; + int topK = Math.min(random.nextInt(TOP_K_LIMIT) + 1, dataset.length); + + if (dataset.length < topK) topK = dataset.length; + + float[][] queries = generateQueries(random, dataset[0].length, numQueries); + List> expected = generateExpectedResults(topK, dataset, queries); + + log.info("Dataset size: " + dataset.length + "x" + dataset[0].length); + log.info("Query size: " + numQueries + "x" + queries[0].length); + log.info("TopK: " + topK); + + // Query query = new CuVSKnnFloatVectorQuery("vector", queries[0], topK, topK, 1); + Query query = new KnnFloatVectorQuery("vector", queries[0], topK); + int correct[] = new int[topK]; + for (int i = 0; i < topK; i++) correct[i] = expected.get(0).get(i); + + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + log.info("RESULTS: " + Arrays.toString(hits)); + log.info("EXPECTD: " + expected.get(0)); + + for (ScoreDoc hit : hits) { + log.info("\t" + reader.storedFields().document(hit.doc).get("id") + ": " + hit.score); + } + + for (ScoreDoc hit : hits) { + int doc = Integer.parseInt(reader.storedFields().document(hit.doc).get("id")); + assertTrue("Result returned was not in topk*2: " + doc, expected.get(0).contains(doc)); + } + } + + private static float[][] generateQueries(Random random, int dimensions, int numQueries) { + // Generate random query vectors + float[][] queries = new float[numQueries][dimensions]; + for (int i = 0; i < numQueries; i++) { + for (int j = 0; j < dimensions; j++) { + queries[i][j] = random.nextFloat() * 100; + } + } + return queries; + } + + private static float[][] generateDataset(Random random, int datasetSize, int dimensions) { + // Generate a random dataset + float[][] dataset = new float[datasetSize][dimensions]; + for (int i = 0; i < datasetSize; i++) { + for (int j = 0; j < dimensions; j++) { + dataset[i][j] = random.nextFloat() * 100; + } + } + return dataset; + } + + private static List> generateExpectedResults( + int topK, float[][] dataset, float[][] queries) { + List> neighborsResult = new ArrayList<>(); + int dimensions = dataset[0].length; + + for (float[] query : queries) { + Map distances = new TreeMap<>(); + for (int j = 0; j < dataset.length; j++) { + double distance = 0; + for (int k = 0; k < dimensions; k++) { + distance += (query[k] - dataset[j][k]) * (query[k] - dataset[j][k]); + } + distances.put(j, (distance)); + } + + Map sorted = new TreeMap(distances); + log.info("EXPECTED: " + sorted); + + // Sort by distance and select the topK nearest neighbors + List neighbors = + distances.entrySet().stream() + .sorted(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .toList(); + neighborsResult.add( + neighbors.subList( + 0, + Math.min( + topK * 3, + dataset.length))); // generate double the topK results in the expected array + } + + log.info("Expected results generated successfully."); + return neighborsResult; + } +} diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java new file mode 100644 index 0000000000..c4fbf67367 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +import java.util.List; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.BeforeClass; + +public class TestCuVSVectorsFormat extends BaseKnnVectorsFormatTestCase { + + @BeforeClass + public static void beforeClass() { + assumeTrue("cuvs is not supported", CuVSVectorsFormat.supported()); + } + + @Override + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat(new CuVSVectorsFormat()); + // For convenience, to sanitize the test code, one can comment out + // the supported check and use another format, e.g. + // return TestUtil.alwaysKnnVectorsFormat(new Lucene99HnswVectorsFormat()); + } + + public void testMergeTwoSegsWithASingleDocPerSeg() throws Exception { + float[][] f = new float[][] {randomVector(384), randomVector(384)}; + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc1 = new Document(); + doc1.add(new StringField("id", "0", Field.Store.NO)); + doc1.add(new KnnFloatVectorField("f", f[0], EUCLIDEAN)); + w.addDocument(doc1); + w.commit(); + Document doc2 = new Document(); + doc2.add(new StringField("id", "1", Field.Store.NO)); + doc2.add(new KnnFloatVectorField("f", f[1], EUCLIDEAN)); + w.addDocument(doc2); + w.flush(); + w.commit(); + + // sanity - verify one doc per leaf + try (DirectoryReader reader = DirectoryReader.open(w)) { + List subReaders = reader.leaves(); + assertEquals(2, subReaders.size()); + assertEquals(1, subReaders.get(0).reader().getFloatVectorValues("f").size()); + assertEquals(1, subReaders.get(1).reader().getFloatVectorValues("f").size()); + } + + // now merge to a single segment + w.forceMerge(1); + + // verify merged content + try (DirectoryReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues values = r.getFloatVectorValues("f"); + assertNotNull(values); + assertEquals(2, values.size()); + assertArrayEquals(f[0], values.vectorValue(0), 0.0f); + assertArrayEquals(f[1], values.vectorValue(1), 0.0f); + } + } + } + + // Basic test for multiple vectors fields per document + public void testTwoVectorFieldsPerDoc() throws Exception { + float[][] f1 = new float[][] {randomVector(384), randomVector(384)}; + float[][] f2 = new float[][] {randomVector(384), randomVector(384)}; + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc1 = new Document(); + doc1.add(new StringField("id", "0", Field.Store.NO)); + doc1.add(new KnnFloatVectorField("f1", f1[0], EUCLIDEAN)); + doc1.add(new KnnFloatVectorField("f2", f2[0], EUCLIDEAN)); + w.addDocument(doc1); + Document doc2 = new Document(); + doc2.add(new StringField("id", "1", Field.Store.NO)); + doc2.add(new KnnFloatVectorField("f1", f1[1], EUCLIDEAN)); + doc2.add(new KnnFloatVectorField("f2", f2[1], EUCLIDEAN)); + w.addDocument(doc2); + w.forceMerge(1); + + try (DirectoryReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues values = r.getFloatVectorValues("f1"); + assertNotNull(values); + assertEquals(2, values.size()); + assertArrayEquals(f1[0], values.vectorValue(0), 0.0f); + assertArrayEquals(f1[1], values.vectorValue(1), 0.0f); + + values = r.getFloatVectorValues("f2"); + assertNotNull(values); + assertEquals(2, values.size()); + assertArrayEquals(f2[0], values.vectorValue(0), 0.0f); + assertArrayEquals(f2[1], values.vectorValue(1), 0.0f); + + // opportunistically check boundary condition - search with a 0 topK + var topDocs = r.searchNearestVectors("f1", randomVector(384), 0, null, 10); + assertEquals(0, topDocs.scoreDocs.length); + assertEquals(0, topDocs.totalHits.value()); + } + } + } +} diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java new file mode 100644 index 0000000000..e2e2b7600e --- /dev/null +++ b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.sandbox.vectorsearch; + +import static org.apache.lucene.util.ArrayUtil.copyOfSubArray; + +import java.io.IOException; +import java.util.Random; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.tests.util.LuceneTestCase; + +public class TestIndexOutputOutputStream extends LuceneTestCase { + + public void testBasic() throws IOException { + try (var dir = newDirectory()) { + try (var indexOut = dir.createOutput("test", IOContext.DEFAULT)) { + var out = new IndexOutputOutputStream(indexOut); + out.write(0x56); + out.write(new byte[] {0x10, 0x11, 0x12, 0x13, 0x14}); + out.close(); + } + + try (var indexIn = dir.openInput("test", IOContext.DEFAULT)) { + var in = new IndexInputInputStream(indexIn); + // assertEquals(0x56, in.read()); + byte[] ba = new byte[6]; + assertEquals(6, in.read(ba)); + assertArrayEquals(new byte[] {0x56, 0x10, 0x11, 0x12, 0x13, 0x14}, ba); + } + } + } + + public void testGetFilePointer() throws IOException { + try (var dir = newDirectory()) { + try (var indexOut = dir.createOutput("test", IOContext.DEFAULT)) { + var out = new IndexOutputOutputStream(indexOut); + out.write(0x56); + out.write(new byte[] {0x10, 0x11, 0x12}); + assertEquals(4, indexOut.getFilePointer()); + out.close(); + } + } + } + + public void testWithRandom() throws IOException { + byte[] data = new byte[Math.min(atLeast(10_000), 20_000)]; + Random random = random(); + random.nextBytes(data); + + try (var dir = newDirectory()) { + try (var indexOut = dir.createOutput("test", IOContext.DEFAULT)) { + var out = new IndexOutputOutputStream(indexOut); + int i = 0; + while (i < data.length) { + if (random.nextBoolean()) { + out.write(data[i]); + i++; + } else { + int numBytes = random.nextInt(Math.min(data.length - i, 100)); + out.write(data, i, numBytes); + i += numBytes; + } + } + out.close(); + } + + try (var indexIn = dir.openInput("test", IOContext.DEFAULT)) { + var in = new IndexInputInputStream(indexIn); + int i = 0; + while (i < data.length) { + if (random.nextBoolean()) { + int b = in.read(); + assertEquals(data[i], b); + i++; + } else { + int numBytes = random.nextInt(Math.min(data.length - i, 100)); + byte[] ba = new byte[numBytes]; + in.read(ba, 0, numBytes); + assertArrayEquals(copyOfSubArray(data, i, i + numBytes), ba); + i += numBytes; + } + } + assertEquals(-1, in.read()); + assertEquals(-1, in.read(new byte[2])); + } + } + } +} From 39de07f935859d932b68c321174610b9ec13be9f Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 2 Jul 2025 16:36:10 -0400 Subject: [PATCH 003/105] Create .pre-commit-config.yaml --- java/cuvs-lucene/.pre-commit-config.yaml | 79 ++++++++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 java/cuvs-lucene/.pre-commit-config.yaml diff --git a/java/cuvs-lucene/.pre-commit-config.yaml b/java/cuvs-lucene/.pre-commit-config.yaml new file mode 100644 index 0000000000..2664bbb271 --- /dev/null +++ b/java/cuvs-lucene/.pre-commit-config.yaml @@ -0,0 +1,79 @@ +# Copyright (c) 2022-2025, NVIDIA CORPORATION. + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-json + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-symlinks + - repo: local + hooks: + - id: no-deprecationwarning + name: no-deprecationwarning + description: 'Enforce that DeprecationWarning is not introduced (use FutureWarning instead)' + entry: '(category=|\s)DeprecationWarning[,)]' + language: pygrep + types_or: [python, cython] + - id: cmake-format + name: cmake-format + entry: ./cpp/scripts/run-cmake-format.sh cmake-format + language: python + types: [cmake] + exclude: .*/thirdparty/.*|.*FindAVX.cmake.* + # Note that pre-commit autoupdate does not update the versions + # of dependencies, so we'll have to update this manually. + additional_dependencies: + - cmakelang==0.6.13 + verbose: true + require_serial: true + - id: cmake-lint + name: cmake-lint + entry: ./cpp/scripts/run-cmake-format.sh cmake-lint + language: python + types: [cmake] + # Note that pre-commit autoupdate does not update the versions + # of dependencies, so we'll have to update this manually. + additional_dependencies: + - cmakelang==0.6.13 + verbose: true + require_serial: true + exclude: .*/thirdparty/.* + - id: include-check + name: include-check + entry: python ./cpp/scripts/include_checker.py cpp/bench cpp/include cpp/tests + pass_filenames: false + language: python + additional_dependencies: [gitpython] + - repo: https://github.com/codespell-project/codespell + rev: v2.2.2 + hooks: + - id: codespell + additional_dependencies: [tomli] + args: ["--toml", "pyproject.toml"] + exclude: | + (?x) + ^CHANGELOG[.]md$| + ^cpp/cmake/patches/cutlass/build-export[.]patch$ + - repo: https://github.com/rapidsai/pre-commit-hooks + rev: v0.6.0 + hooks: + - id: verify-copyright + files: | + (?x) + [.](cmake|cpp|cu|cuh|h|hpp|sh|pxd|py|pyx|rs)$| + CMakeLists[.]txt$| + CMakeLists_standalone[.]txt$| + meta[.]yaml$ + exclude: | + (?x) + docs/source/sphinxext/github_link\.py| + cpp/cmake/modules/FindAVX\.cmake + - id: verify-alpha-spec + - id: verify-codeowners + args: [--fix, --project-prefix=cuvs] + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 + hooks: + - id: shellcheck From 60204698974f6e1f2a7b382b015a5be5f127fff7 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 2 Jul 2025 16:48:17 -0400 Subject: [PATCH 004/105] Updates --- java/cuvs-lucene/.pre-commit-config.yaml | 52 +----------------------- java/cuvs-lucene/LICENSE | 1 - java/cuvs-lucene/VERSION | 1 + java/cuvs-lucene/pom.xml | 2 +- 4 files changed, 4 insertions(+), 52 deletions(-) create mode 100644 java/cuvs-lucene/VERSION diff --git a/java/cuvs-lucene/.pre-commit-config.yaml b/java/cuvs-lucene/.pre-commit-config.yaml index 2664bbb271..2e2dace2ff 100644 --- a/java/cuvs-lucene/.pre-commit-config.yaml +++ b/java/cuvs-lucene/.pre-commit-config.yaml @@ -8,61 +8,13 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - id: check-symlinks - - repo: local - hooks: - - id: no-deprecationwarning - name: no-deprecationwarning - description: 'Enforce that DeprecationWarning is not introduced (use FutureWarning instead)' - entry: '(category=|\s)DeprecationWarning[,)]' - language: pygrep - types_or: [python, cython] - - id: cmake-format - name: cmake-format - entry: ./cpp/scripts/run-cmake-format.sh cmake-format - language: python - types: [cmake] - exclude: .*/thirdparty/.*|.*FindAVX.cmake.* - # Note that pre-commit autoupdate does not update the versions - # of dependencies, so we'll have to update this manually. - additional_dependencies: - - cmakelang==0.6.13 - verbose: true - require_serial: true - - id: cmake-lint - name: cmake-lint - entry: ./cpp/scripts/run-cmake-format.sh cmake-lint - language: python - types: [cmake] - # Note that pre-commit autoupdate does not update the versions - # of dependencies, so we'll have to update this manually. - additional_dependencies: - - cmakelang==0.6.13 - verbose: true - require_serial: true - exclude: .*/thirdparty/.* - - id: include-check - name: include-check - entry: python ./cpp/scripts/include_checker.py cpp/bench cpp/include cpp/tests - pass_filenames: false - language: python - additional_dependencies: [gitpython] - - repo: https://github.com/codespell-project/codespell - rev: v2.2.2 - hooks: - - id: codespell - additional_dependencies: [tomli] - args: ["--toml", "pyproject.toml"] - exclude: | - (?x) - ^CHANGELOG[.]md$| - ^cpp/cmake/patches/cutlass/build-export[.]patch$ - repo: https://github.com/rapidsai/pre-commit-hooks rev: v0.6.0 hooks: - id: verify-copyright files: | (?x) - [.](cmake|cpp|cu|cuh|h|hpp|sh|pxd|py|pyx|rs)$| + [.](cmake|cpp|cu|cuh|h|hpp|sh|pxd|py|pyx|rs|java)$| CMakeLists[.]txt$| CMakeLists_standalone[.]txt$| meta[.]yaml$ @@ -72,7 +24,7 @@ repos: cpp/cmake/modules/FindAVX\.cmake - id: verify-alpha-spec - id: verify-codeowners - args: [--fix, --project-prefix=cuvs] + args: [--fix, --project-prefix=lucene-cuvs] - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.10.0.1 hooks: diff --git a/java/cuvs-lucene/LICENSE b/java/cuvs-lucene/LICENSE index 534e44ea18..24376a8922 100644 --- a/java/cuvs-lucene/LICENSE +++ b/java/cuvs-lucene/LICENSE @@ -223,4 +223,3 @@ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - diff --git a/java/cuvs-lucene/VERSION b/java/cuvs-lucene/VERSION new file mode 100644 index 0000000000..3af4bda020 --- /dev/null +++ b/java/cuvs-lucene/VERSION @@ -0,0 +1 @@ +25.08.00 diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 3c1b686ecc..ad5584f687 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -12,7 +12,7 @@ 21 21 - + cuvs-java From 98d5f0302ee1dd75021af40661ba8ea6bb75d8ca Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Wed, 2 Jul 2025 13:58:10 -0700 Subject: [PATCH 005/105] update copyright --- .../java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java | 2 ++ .../org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java | 2 ++ .../java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java | 2 ++ .../lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java | 2 ++ .../org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java | 2 ++ .../apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java | 2 ++ .../apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java | 2 ++ .../apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java | 2 ++ .../lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java | 2 ++ .../lucene/sandbox/vectorsearch/IndexInputInputStream.java | 2 ++ .../lucene/sandbox/vectorsearch/IndexOutputOutputStream.java | 2 ++ .../lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java | 2 ++ .../org/apache/lucene/sandbox/vectorsearch/package-info.java | 2 ++ .../java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java | 2 ++ .../lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java | 2 ++ .../sandbox/vectorsearch/TestIndexOutputOutputStream.java | 2 ++ 16 files changed, 32 insertions(+) diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java index c4d032fe6b..3e75dff043 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java index 61b8f08792..ce466b1837 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java index d0cfe86d70..5e0f402061 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java index 2f6c636590..8c2993f88c 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java index 9b12cdf610..f41e069bca 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java index cfb59121e3..b66564a7de 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java index 29d32b9e6e..0754b3448c 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java index 7b1d845320..57c30226a1 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java index eeb7b6895a..902790dcf6 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java index 4eb8ed558f..8630402ab5 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java index ffb2b922e4..04cb3db1bf 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java index 90e0e1ff44..5e60e0f3ef 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java index 86c56b909d..73ef06dbb9 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java index a20a49be6f..cf5bf9f723 100644 --- a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java +++ b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java index c4fbf67367..7279f708a4 100644 --- a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java index e2e2b7600e..c04368ff0b 100644 --- a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java @@ -1,4 +1,6 @@ /* + * Copyright (c) 2025, NVIDIA CORPORATION. + * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. From 42e5d02d476bd8c8ee695bede66fee0e947547b2 Mon Sep 17 00:00:00 2001 From: Ben Frederickson Date: Wed, 2 Jul 2025 14:01:14 -0700 Subject: [PATCH 006/105] standardize license text --- .../apache/lucene/sandbox/vectorsearch/CuVSCodec.java | 9 +++------ .../lucene/sandbox/vectorsearch/CuVSFieldWriter.java | 9 +++------ .../apache/lucene/sandbox/vectorsearch/CuVSIndex.java | 9 +++------ .../sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java | 9 +++------ .../lucene/sandbox/vectorsearch/CuVSSegmentFile.java | 9 +++------ .../lucene/sandbox/vectorsearch/CuVSVectorsReader.java | 9 +++------ .../lucene/sandbox/vectorsearch/CuVSVectorsWriter.java | 9 +++------ .../lucene/sandbox/vectorsearch/FilterCuVSProvider.java | 9 +++------ .../sandbox/vectorsearch/FilterCuVSServiceProvider.java | 9 +++------ .../sandbox/vectorsearch/IndexInputInputStream.java | 9 +++------ .../sandbox/vectorsearch/IndexOutputOutputStream.java | 9 +++------ .../sandbox/vectorsearch/PerLeafCuVSKnnCollector.java | 9 +++------ .../apache/lucene/sandbox/vectorsearch/package-info.java | 9 +++------ .../org/apache/lucene/sandbox/vectorsearch/TestCuVS.java | 9 +++------ .../sandbox/vectorsearch/TestCuVSVectorsFormat.java | 9 +++------ .../vectorsearch/TestIndexOutputOutputStream.java | 9 +++------ 16 files changed, 48 insertions(+), 96 deletions(-) diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java index 3e75dff043..98f7bf5d68 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java index ce466b1837..f10072599e 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java index 5e0f402061..a535e9648f 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java index 8c2993f88c..0547dfe07f 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java index f41e069bca..00f505e363 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java index b66564a7de..f7d52f864f 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java index 0754b3448c..67c2317524 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java index 57c30226a1..ea91b22271 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java index 902790dcf6..7f9b39c49f 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java index 8630402ab5..f32fbf9ccf 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java index 04cb3db1bf..68460abb43 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java index 5e60e0f3ef..7490778b97 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java index 73ef06dbb9..1fdaa013db 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java index cf5bf9f723..55238da599 100644 --- a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java +++ b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java index 7279f708a4..38eb96bb18 100644 --- a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java index c04368ff0b..9703c36fe8 100644 --- a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java @@ -1,12 +1,9 @@ /* * Copyright (c) 2025, NVIDIA CORPORATION. * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * From 2b2ae0889307e7b5685176eb4dad3d9afb0e5ed1 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 8 Jul 2025 10:20:00 -0400 Subject: [PATCH 007/105] Create CONTRIBUTING.md --- java/cuvs-lucene/CONTRIBUTING.md | 133 +++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 java/cuvs-lucene/CONTRIBUTING.md diff --git a/java/cuvs-lucene/CONTRIBUTING.md b/java/cuvs-lucene/CONTRIBUTING.md new file mode 100644 index 0000000000..cd5d29b2d8 --- /dev/null +++ b/java/cuvs-lucene/CONTRIBUTING.md @@ -0,0 +1,133 @@ +# Contributing + +If you are interested in contributing to Lucene-cuVS, your contributions will fall +into three categories: +1. You want to report a bug, feature request, or documentation issue + - File an [issue](https://github.com/rapidsai/cuvs/issues/new/choose) + describing what you encountered or what you want to see changed. + - The RAPIDS team will evaluate the issues and triage them, scheduling + them for a release. If you believe the issue needs priority attention + comment on the issue to notify the team. +2. You want to propose a new Feature and implement it + - Post about your intended feature, and we shall discuss the design and + implementation. + - Once we agree that the plan looks good, go ahead and implement it, using + the [code contributions](#code-contributions) guide below. +3. You want to implement a feature or bug-fix for an outstanding issue + - Follow the [code contributions](#code-contributions) guide below. + - If you need more context on a particular issue, please ask and we shall + provide. + + +## Code contributions + +### Your first issue + +1. Read the project's [README.md](https://github.com/rapidsai/cuvs) + to learn how to setup the development environment +2. Find an issue to work on. The best way is to look for the [good first issue](https://github.com/rapidsai/CUVS/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) + or [help wanted](https://github.com/rapidsai/cuvs/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) labels +3. Comment on the issue saying you are going to work on it +4. Code! Make sure to update unit tests! +5. When done, [create your pull request](https://github.com/rapidsai/cuvs/compare) +6. Verify that CI passes all [status checks](https://help.github.com/articles/about-status-checks/). Fix if needed +7. Wait for other developers to review your code and update code as needed +8. Once reviewed and approved, a RAPIDS developer will merge your pull request + +Remember, if you are unsure about anything, don't hesitate to comment on issues +and ask for clarifications! + + +### Python / Pre-commit hooks + +Lucene-cuVS uses [pre-commit](https://pre-commit.com/) to execute code linters and formatters. These tools ensure a consistent code format +throughout the project. Using pre-commit ensures that linter versions and options are aligned for +all developers. Additionally, there is a CI check in place to enforce that committed code follows +our standards. + +To use `pre-commit`, install via `conda` or `pip`: + +```bash +conda install -c conda-forge pre-commit +``` + +```bash +pip install pre-commit +``` + +Then run pre-commit hooks before committing code: + +```bash +pre-commit run +``` + +Optionally, you may set up the pre-commit hooks to run automatically when you make a git commit. This can be done by running: + +```bash +pre-commit install +``` + +Now code linters and formatters will be run each time you commit changes. + +You can skip these checks with `git commit --no-verify` or with the short version `git commit -n`. + + +### Seasoned developers + +Once you have gotten your feet wet and are more comfortable with the code, you +can look at the prioritized issues of our next release in our [project boards](https://github.com/rapidsai/CUVS/projects). + +> **Pro Tip:** Always look at the release board with the highest number for +issues to work on. This is where RAPIDS developers also focus their efforts. + +Look at the unassigned issues, and find an issue you are comfortable with +contributing to. Start with _Step 3_ from above, commenting on the issue to let +others know you are working on it. If you have any questions related to the +implementation of the issue, ask them in the issue instead of the PR. + + +### Signing Your Work + +* We require that all third party contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license. + + * Any contribution which contains commits that are not Signed-Off will not be accepted. + +* To sign off on a commit you simply use the `--signoff` (or `-s`) option when committing your changes: + ```bash + $ git commit -s -m "Add cool feature." + ``` + This will append the following to your commit message: + ``` + Signed-off-by: Your Name + ``` + +* Full text of the DCO: + + ``` + Developer Certificate of Origin + Version 1.1 + + Copyright (C) 2004, 2006 The Linux Foundation and its contributors. + 1 Letterman Drive + Suite D4700 + San Francisco, CA, 94129 + + Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + ``` + + ``` + Developer's Certificate of Origin 1.1 + + By making a contribution to this project, I certify that: + + (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or + + (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or + + (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. + + (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. + ``` + +## Attribution +Portions adopted from https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md From cdfeca0ca9698f4d2f7e9f6a970ace07d8824db7 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Wed, 16 Jul 2025 10:26:28 -0400 Subject: [PATCH 008/105] Update README.md --- java/cuvs-lucene/README.md | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/java/cuvs-lucene/README.md b/java/cuvs-lucene/README.md index 482ab785dc..7f65124436 100644 --- a/java/cuvs-lucene/README.md +++ b/java/cuvs-lucene/README.md @@ -1,13 +1,11 @@ # Lucene CuVS Integration -This is an integration for [CuVS](https://github.com/rapidsai/cuvs), GPU accelerated vector search library from NVIDIA (formerly part of [Raft](https://github.com/rapidsai/raft)), into [Apache Lucene](https://github.com/apache/lucene). +This is a codec for connecting [CuVS](https://github.com/rapidsai/cuvs), NVIDIA's GPU accelerated vector search library, into [Apache Lucene](https://github.com/apache/lucene). ## Overview The CuVS library is plugged in as a new KnnVectorFormat via a custom codec. -![Architecture](lucene-cuvs-architecture.png "Lucene CuVS Architecture") - > :warning: This is not production ready yet. ## Building @@ -17,10 +15,3 @@ Install NVIDIA drivers, CUDA 12.8, Maven 3.9.6+ and JDK 22. mvn clean compile package The artifacts would be built and available in target/ folder. - -## Contributors - -* Vivek Narang, SearchScale -* Ishan Chattopadhyaya, SearchScale & Committer, Apache Lucene & Solr -* Puneet Ahuja, SearchScale -* Corey Nolet, NVIDIA From c5cf67b6cdedf222739a1770f5b1f689f971fa50 Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Fri, 25 Jul 2025 16:09:52 -0400 Subject: [PATCH 009/105] Adding codeowners --- java/cuvs-lucene/.github/CODEOWNERS | 34 +++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 java/cuvs-lucene/.github/CODEOWNERS diff --git a/java/cuvs-lucene/.github/CODEOWNERS b/java/cuvs-lucene/.github/CODEOWNERS new file mode 100644 index 0000000000..22d9e8d886 --- /dev/null +++ b/java/cuvs-lucene/.github/CODEOWNERS @@ -0,0 +1,34 @@ +#cpp code owners +cpp/ @rapidsai/cuvs-cpp-codeowners +examples/cpp/ @rapidsai/cuvs-cpp-codeowners +examples/c/ @rapidsai/cuvs-cpp-codeowners + +#java code owners +java/ @rapidsai/cuvs-java-codeowners +examples/java/ @rapidsai/cuvs-java-codeowners + +#python code owners +python/ @rapidsai/cuvs-python-codeowners + +#rust code owners +rust/ @rapidsai/cuvs-rust-codeowners +examples/rust/ @rapidsai/cuvs-rust-codeowners + +# docs code owners +docs/ @rapidsai/cuvs-docs-codeowners +README.md @rapidsai/cuvs-docs-codeowners +CONTRIBUTING.md @rapidsai/cuvs-docs-codeowners +LICENSE @rapidsai/cuvs-docs-codeowners + +#cmake code owners +pom.xml @rapidsai/cuvs-build-codeowners + +#CI code owners +/.github/ @rapidsai/ci-codeowners +/ci/ @rapidsai/ci-codeowners + +# packaging code owners +/.pre-commit-config.yaml @rapidsai/packaging-codeowners + + +# src/ @rapidsai/cuvs-java-codeowners From d3ff041e3bc50282bab7d25d27363fe91cbe97c9 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Wed, 30 Jul 2025 09:01:23 -0400 Subject: [PATCH 010/105] feat: add ops-bot config --- java/cuvs-lucene/.github/ops-bot.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 java/cuvs-lucene/.github/ops-bot.yaml diff --git a/java/cuvs-lucene/.github/ops-bot.yaml b/java/cuvs-lucene/.github/ops-bot.yaml new file mode 100644 index 0000000000..2ed5231aef --- /dev/null +++ b/java/cuvs-lucene/.github/ops-bot.yaml @@ -0,0 +1,9 @@ +# This file controls which features from the `ops-bot` repository below are enabled. +# - https://github.com/rapidsai/ops-bot + +auto_merger: true +branch_checker: true +label_checker: true +release_drafter: true +recently_updated: true +forward_merger: true From ad8813faa646ca3830e47294655e38db531e257a Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Wed, 30 Jul 2025 09:02:18 -0400 Subject: [PATCH 011/105] Add the Spotless Maven plugin (and other initial changes) (NVIDIA/cuvs-lucene#2) Fixes NVIDIA/cuvs-lucene#1 Things included in this PR: - Add the spotless maven plugin configuration. - Add the gitignore file. - Add a source license-header file that is used by the spotless plugin (to add/update license headers automatically if missing). - Update the readme file. Authors: - Vivek Narang (https://github.com/narangvivek10) - Gil Forsyth (https://github.com/gforsyth) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cuvs-lucene/pull/2 --- java/cuvs-lucene/.gitignore | 1 + java/cuvs-lucene/README.md | 24 ++- java/cuvs-lucene/license-header.txt | 15 ++ java/cuvs-lucene/pom.xml | 155 ++++++++---------- .../vectorsearch/PerLeafCuVSKnnCollector.java | 2 +- .../vectorsearch/TestCuVSVectorsFormat.java | 1 - 6 files changed, 105 insertions(+), 93 deletions(-) create mode 100644 java/cuvs-lucene/.gitignore create mode 100644 java/cuvs-lucene/license-header.txt diff --git a/java/cuvs-lucene/.gitignore b/java/cuvs-lucene/.gitignore new file mode 100644 index 0000000000..1de565933b --- /dev/null +++ b/java/cuvs-lucene/.gitignore @@ -0,0 +1 @@ +target \ No newline at end of file diff --git a/java/cuvs-lucene/README.md b/java/cuvs-lucene/README.md index 7f65124436..e956c69d79 100644 --- a/java/cuvs-lucene/README.md +++ b/java/cuvs-lucene/README.md @@ -1,17 +1,25 @@ -# Lucene CuVS Integration +# Lucene cuVS -This is a codec for connecting [CuVS](https://github.com/rapidsai/cuvs), NVIDIA's GPU accelerated vector search library, into [Apache Lucene](https://github.com/apache/lucene). +This is a codec for connecting [cuVS](https://github.com/rapidsai/cuvs), NVIDIA's GPU accelerated vector search library, into [Apache Lucene](https://github.com/apache/lucene). ## Overview -The CuVS library is plugged in as a new KnnVectorFormat via a custom codec. +The cuVS library is plugged in as a new `KnnVectorFormat` via a custom codec. -> :warning: This is not production ready yet. +> [!CAUTION] +> This is not production ready yet. -## Building +### Prerequisites +- [CUDA 12.0+](https://developer.nvidia.com/cuda-toolkit-archive), +- [Maven 3.9.6+](https://maven.apache.org/download.cgi), +- [JDK 22](https://jdk.java.net/archive/) -Install NVIDIA drivers, CUDA 12.8, Maven 3.9.6+ and JDK 22. +### Building +```sh +mvn clean compile package +``` - mvn clean compile package +The artifacts would be built and available in the target/ folder. -The artifacts would be built and available in target/ folder. +> [!NOTE] +> The code style format is automatically enforced (including the missing license header, if any) using the [Spotless maven plugin](https://github.com/diffplug/spotless/tree/main/plugin-maven). This currently happens in the maven's `validate` stage. \ No newline at end of file diff --git a/java/cuvs-lucene/license-header.txt b/java/cuvs-lucene/license-header.txt new file mode 100644 index 0000000000..2001ad3d3a --- /dev/null +++ b/java/cuvs-lucene/license-header.txt @@ -0,0 +1,15 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index ad5584f687..c6adc0c140 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -22,97 +22,86 @@ - - org.apache.lucene - lucene-core - 10.2.0 - - - org.apache.lucene - lucene-codecs - 10.2.0 - test - - - org.apache.lucene - lucene-test-framework - 10.2.0 - test - - - com.opencsv - opencsv - 5.3 - - - commons-io - commons-io - 2.15.1 - - - - com.github.fommil - jniloader - 1.1 - - - - com.fasterxml.jackson.core - jackson-databind - 2.17.0 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-csv - 2.17.0 - - - com.nvidia.cuvs - cuvs-java - 25.08.0 - + + org.apache.lucene + lucene-core + 10.2.0 + + + org.apache.lucene + lucene-codecs + 10.2.0 + test + + + org.apache.lucene + lucene-test-framework + 10.2.0 + test + + + com.opencsv + opencsv + 5.3 + + + commons-io + commons-io + 2.15.1 + + + com.github.fommil + jniloader + 1.1 + + + com.fasterxml.jackson.core + jackson-databind + 2.17.0 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-csv + 2.17.0 + + + com.nvidia.cuvs + cuvs-java + 25.08.0 + - - + diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java index 7490778b97..c969fe9241 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java +++ b/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java @@ -90,6 +90,6 @@ public TopDocs topDocs() { @Override public KnnSearchStrategy getSearchStrategy() { - return KnnSearchStrategy.Patience.DEFAULT; + return KnnSearchStrategy.Patience.DEFAULT; } } diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java index 38eb96bb18..fea3b58c97 100644 --- a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java @@ -18,7 +18,6 @@ import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; import java.util.List; - import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; From 9ed4a68a8af30cd48af0fe3fd92535af31f7e685 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Thu, 31 Jul 2025 12:23:27 -0400 Subject: [PATCH 012/105] Update package names (NVIDIA/cuvs-lucene#9) Fixes NVIDIA/cuvs-lucene#8 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/9 --- java/cuvs-lucene/pom.xml | 6 +++--- .../nvidia/cuvs/lucene}/CuVSCodec.java | 4 ++-- .../nvidia/cuvs/lucene}/CuVSFieldWriter.java | 2 +- .../nvidia/cuvs/lucene}/CuVSIndex.java | 4 ++-- .../cuvs/lucene}/CuVSKnnFloatVectorQuery.java | 2 +- .../nvidia/cuvs/lucene}/CuVSSegmentFile.java | 2 +- .../nvidia/cuvs/lucene}/CuVSVectorsFormat.java | 4 ++-- .../nvidia/cuvs/lucene}/CuVSVectorsReader.java | 16 ++++++++-------- .../nvidia/cuvs/lucene}/CuVSVectorsWriter.java | 16 ++++++++-------- .../nvidia/cuvs/lucene}/FilterCuVSProvider.java | 2 +- .../cuvs/lucene}/FilterCuVSServiceProvider.java | 2 +- .../cuvs/lucene}/IndexInputInputStream.java | 2 +- .../cuvs/lucene}/IndexOutputOutputStream.java | 2 +- .../cuvs/lucene}/PerLeafCuVSKnnCollector.java | 2 +- .../nvidia/cuvs/lucene}/package-info.java | 2 +- .../nvidia/cuvs/lucene}/TestCuVS.java | 2 +- .../cuvs/lucene}/TestCuVSVectorsFormat.java | 2 +- .../lucene}/TestIndexOutputOutputStream.java | 2 +- 18 files changed, 37 insertions(+), 37 deletions(-) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/CuVSCodec.java (93%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/CuVSFieldWriter.java (98%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/CuVSIndex.java (95%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/CuVSKnnFloatVectorQuery.java (97%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/CuVSSegmentFile.java (97%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/CuVSVectorsFormat.java (97%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/CuVSVectorsReader.java (96%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/CuVSVectorsWriter.java (96%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/FilterCuVSProvider.java (97%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/FilterCuVSServiceProvider.java (95%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/IndexInputInputStream.java (96%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/IndexOutputOutputStream.java (97%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/PerLeafCuVSKnnCollector.java (98%) rename java/cuvs-lucene/src/main/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/package-info.java (93%) rename java/cuvs-lucene/src/test/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/TestCuVS.java (99%) rename java/cuvs-lucene/src/test/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/TestCuVSVectorsFormat.java (99%) rename java/cuvs-lucene/src/test/java/{org/apache/lucene/sandbox/vectorsearch => com/nvidia/cuvs/lucene}/TestIndexOutputOutputStream.java (98%) diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index c6adc0c140..59b858843b 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -2,10 +2,10 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.searchscale.lucene.vectorsearch - lucene-cuvs + com.nvidia.cuvs.lucene + cuvs-lucene 0.0.1-SNAPSHOT - lucene-cuvs + cuvs-lucene jar diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodec.java similarity index 93% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodec.java index 98f7bf5d68..ef4dde5f42 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodec.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import com.nvidia.cuvs.LibraryException; +import com.nvidia.cuvs.lucene.CuVSVectorsWriter.IndexType; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.lucene101.Lucene101Codec; -import org.apache.lucene.sandbox.vectorsearch.CuVSVectorsWriter.IndexType; /** CuVS based codec for GPU based vector search */ public class CuVSCodec extends FilterCodec { diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSFieldWriter.java similarity index 98% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSFieldWriter.java index f10072599e..acd151817a 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSFieldWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSFieldWriter.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import java.io.IOException; import java.util.List; diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSIndex.java similarity index 95% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSIndex.java index a535e9648f..78c5dc1409 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSIndex.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSIndex.java @@ -13,9 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsReader.handleThrowable; +import static com.nvidia.cuvs.lucene.CuVSVectorsReader.handleThrowable; import com.nvidia.cuvs.BruteForceIndex; import com.nvidia.cuvs.CagraIndex; diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSKnnFloatVectorQuery.java similarity index 97% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSKnnFloatVectorQuery.java index 0547dfe07f..fa1e71d5e4 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSKnnFloatVectorQuery.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSKnnFloatVectorQuery.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import java.io.IOException; import org.apache.lucene.index.LeafReader; diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSSegmentFile.java similarity index 97% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSSegmentFile.java index 00f505e363..b133fd57e1 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSSegmentFile.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSSegmentFile.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import java.io.IOException; import java.io.OutputStream; diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsFormat.java similarity index 97% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsFormat.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsFormat.java index f5bd1bd909..f82198b1c3 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsFormat.java @@ -13,10 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import com.nvidia.cuvs.CuVSResources; import com.nvidia.cuvs.LibraryException; +import com.nvidia.cuvs.lucene.CuVSVectorsWriter.IndexType; import java.io.IOException; import java.util.logging.Logger; import org.apache.lucene.codecs.KnnVectorsFormat; @@ -25,7 +26,6 @@ import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.sandbox.vectorsearch.CuVSVectorsWriter.IndexType; /** CuVS based KnnVectorsFormat for GPU acceleration */ public class CuVSVectorsFormat extends KnnVectorsFormat { diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java similarity index 96% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java index f7d52f864f..873f4de670 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; - -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_INDEX_CODEC_NAME; -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_INDEX_EXT; -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_META_CODEC_EXT; -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_META_CODEC_NAME; -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.VERSION_CURRENT; -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.VERSION_START; +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_INDEX_CODEC_NAME; +import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_INDEX_EXT; +import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_META_CODEC_EXT; +import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_META_CODEC_NAME; +import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.VERSION_CURRENT; +import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.VERSION_START; import com.nvidia.cuvs.BruteForceIndex; import com.nvidia.cuvs.BruteForceQuery; diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java similarity index 96% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java index 67c2317524..59cadf92ce 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/CuVSVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; - +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_INDEX_CODEC_NAME; +import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_INDEX_EXT; +import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_META_CODEC_EXT; +import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_META_CODEC_NAME; +import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.VERSION_CURRENT; +import static com.nvidia.cuvs.lucene.CuVSVectorsReader.handleThrowable; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; import static org.apache.lucene.index.VectorEncoding.FLOAT32; -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_INDEX_CODEC_NAME; -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_INDEX_EXT; -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_META_CODEC_EXT; -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.CUVS_META_CODEC_NAME; -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsFormat.VERSION_CURRENT; -import static org.apache.lucene.sandbox.vectorsearch.CuVSVectorsReader.handleThrowable; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java similarity index 97% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index ea91b22271..e65e541f05 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import com.nvidia.cuvs.BruteForceIndex; import com.nvidia.cuvs.CagraIndex; diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java similarity index 95% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java index 7f9b39c49f..9416f082b0 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/FilterCuVSServiceProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import com.nvidia.cuvs.spi.CuVSProvider; import com.nvidia.cuvs.spi.CuVSServiceProvider; diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java similarity index 96% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java index f32fbf9ccf..1beead66b7 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexInputInputStream.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import java.io.IOException; import java.io.InputStream; diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java similarity index 97% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java index 68460abb43..b7866363fc 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/IndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import java.io.IOException; import java.io.OutputStream; diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/PerLeafCuVSKnnCollector.java similarity index 98% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/PerLeafCuVSKnnCollector.java index c969fe9241..8e005570fa 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/PerLeafCuVSKnnCollector.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/PerLeafCuVSKnnCollector.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import java.util.ArrayList; import java.util.List; diff --git a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java similarity index 93% rename from java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java index 1fdaa013db..043bfe0818 100644 --- a/java/cuvs-lucene/src/main/java/org/apache/lucene/sandbox/vectorsearch/package-info.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java @@ -15,4 +15,4 @@ */ /** CuVS based fast vector search */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVS.java similarity index 99% rename from java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java rename to java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVS.java index 55238da599..7f66943030 100644 --- a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVS.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVS.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import java.io.IOException; import java.util.ArrayList; diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java similarity index 99% rename from java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java rename to java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java index fea3b58c97..4937db311d 100644 --- a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestCuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; diff --git a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java similarity index 98% rename from java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java rename to java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java index 9703c36fe8..c04cb586ae 100644 --- a/java/cuvs-lucene/src/test/java/org/apache/lucene/sandbox/vectorsearch/TestIndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.lucene.sandbox.vectorsearch; +package com.nvidia.cuvs.lucene; import static org.apache.lucene.util.ArrayUtil.copyOfSubArray; From 054fafb6765f8bf88129012940cf08c0ac846a9d Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Sat, 2 Aug 2025 17:32:46 -0400 Subject: [PATCH 013/105] Updates to adapt with cuVS-Java API changes (NVIDIA/cuvs-lucene#10) Fixes NVIDIA/cuvs-lucene#5 Main changes in this PR: - Adapt to the cuVS-Java API updates: Mainly replace the use of `Dataset` with the new `CuVSMatrix` - Disable the tests that use byte vector encoding, as we do not support it. Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/10 --- java/cuvs-lucene/README.md | 6 +- java/cuvs-lucene/pom.xml | 3 +- .../com/nvidia/cuvs/lucene/CuVSCodec.java | 1 + .../nvidia/cuvs/lucene/CuVSSegmentFile.java | 5 -- .../nvidia/cuvs/lucene/CuVSVectorsReader.java | 13 ++-- .../nvidia/cuvs/lucene/CuVSVectorsWriter.java | 73 ++++++++++--------- .../cuvs/lucene/FilterCuVSProvider.java | 35 ++++++++- .../org.apache.lucene.codecs.KnnVectorsFormat | 2 +- .../java/com/nvidia/cuvs/lucene/TestCuVS.java | 1 - .../cuvs/lucene/TestCuVSVectorsFormat.java | 41 ++++++++++- .../lucene/TestIndexOutputOutputStream.java | 2 +- 11 files changed, 122 insertions(+), 60 deletions(-) diff --git a/java/cuvs-lucene/README.md b/java/cuvs-lucene/README.md index e956c69d79..0cbf52ac4f 100644 --- a/java/cuvs-lucene/README.md +++ b/java/cuvs-lucene/README.md @@ -18,8 +18,12 @@ The cuVS library is plugged in as a new `KnnVectorFormat` via a custom codec. ```sh mvn clean compile package ``` +The artifacts would be built and available in the target / folder. -The artifacts would be built and available in the target/ folder. +### Run Tests +```sh +export LD_LIBRARY_PATH={ PATH TO YOUR LOCAL libcuvs_c.so }:$LD_LIBRARY_PATH && mvn clean test +``` > [!NOTE] > The code style format is automatically enforced (including the missing license header, if any) using the [Spotless maven plugin](https://github.com/diffplug/spotless/tree/main/plugin-maven). This currently happens in the maven's `validate` stage. \ No newline at end of file diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 59b858843b..c79f26eec3 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -67,7 +67,8 @@ com.nvidia.cuvs cuvs-java - 25.08.0 + + 25.8.0-4f53f-SNAPSHOT diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodec.java index ef4dde5f42..ec861765db 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodec.java @@ -34,6 +34,7 @@ public CuVSCodec(String name, Codec delegate) { super(name, delegate); KnnVectorsFormat format; try { + // TODO: Remove this hard coded values. format = new CuVSVectorsFormat(1, 128, 64, IndexType.CAGRA); setKnnFormat(format); } catch (LibraryException ex) { diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSSegmentFile.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSSegmentFile.java index b133fd57e1..8a601b7eee 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSSegmentFile.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSSegmentFile.java @@ -39,11 +39,6 @@ public CuVSSegmentFile(OutputStream out) { protected Logger log = Logger.getLogger(getClass().getName()); public void addFile(String name, byte[] bytes) throws IOException { - /*log.info( - "Writing the file: " - + name - + ", size=" - + bytes.length);*/ ZipEntry indexFileZipEntry = new ZipEntry(name); zos.putNextEntry(indexFileZipEntry); zos.write(bytes, 0, bytes.length); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java index 873f4de670..c770015de2 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java @@ -334,7 +334,6 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits } var fieldNumber = fieldInfos.fieldInfo(field).number; - // log.info("fieldNumber=" + fieldNumber + ", fieldEntry.count()=" + fieldEntry.count()); CuVSIndex cuvsIndex = cuvsIndices.get(fieldNumber); if (cuvsIndex == null) { @@ -350,19 +349,16 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits Map result; if (knnCollector.k() <= 1024 && cuvsIndex.getCagraIndex() != null) { - // log.info("searching cagra index"); CagraSearchParams searchParams = - new CagraSearchParams.Builder(resources) + new CagraSearchParams.Builder() .withItopkSize(topK) // TODO: params .withSearchWidth(1) .build(); var query = - new CagraQuery.Builder() + new CagraQuery.Builder(resources) .withTopK(topK) .withSearchParams(searchParams) - // we don't use ord to doc mapping, https://github.com/rapidsai/cuvs/issues/699 - .withMapping(null) .withQueryVectors(new float[][] {target}) .build(); @@ -379,9 +375,10 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits } else { BruteForceIndex bruteforceIndex = cuvsIndex.getBruteforceIndex(); assert bruteforceIndex != null; - // log.info("searching brute index, with actual topK=" + topK); var queryBuilder = - new BruteForceQuery.Builder().withQueryVectors(new float[][] {target}).withTopK(topK); + new BruteForceQuery.Builder(resources) + .withQueryVectors(new float[][] {target}) + .withTopK(topK); BruteForceQuery query = queryBuilder.build(); List> searchResult = null; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java index 59cadf92ce..3e22b4dab1 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java @@ -31,8 +31,8 @@ import com.nvidia.cuvs.CagraIndex; import com.nvidia.cuvs.CagraIndexParams; import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.CuVSMatrix; import com.nvidia.cuvs.CuVSResources; -import com.nvidia.cuvs.Dataset; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; @@ -61,7 +61,10 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.InfoStream; -/** KnnVectorsWriter for CuVS, responsible for merge and flush of vectors into GPU */ +/** + * KnnVectorsWriter for CuVS, responsible for merge and flush of vectors into + * GPU + */ public class CuVSVectorsWriter extends KnnVectorsWriter { private static final long SHALLOW_RAM_BYTES_USED = shallowSizeOfInstance(CuVSVectorsWriter.class); @@ -197,9 +200,6 @@ private CagraIndexParams cagraIndexParams(int size) { // https://github.com/rapidsai/cuvs/issues/666 throw new IllegalArgumentException("cagra index must be greater than 2"); } - // var minIntGraphDegree = Math.min(intGraphDegree, size - 1); - // var minGraphDegree = Math.min(graphDegree, minIntGraphDegree); - // log.info(indexMsg(size, intGraphDegree, minIntGraphDegree, graphDegree, minGraphDegree)); return new CagraIndexParams.Builder() .withNumWriterThreads(cuvsWriterThreads) @@ -219,11 +219,11 @@ private void info(String msg) { } } - private void writeCagraIndex(OutputStream os, Dataset dataset) throws Throwable { + private void writeCagraIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { if (dataset.size() < 2) { throw new IllegalArgumentException(dataset.size() + " vectors, less than min [2] required"); } - CagraIndexParams params = cagraIndexParams(dataset.size()); + CagraIndexParams params = cagraIndexParams((int) dataset.size()); long startTime = System.nanoTime(); var index = CagraIndex.newBuilder(resources).withDataset(dataset).withIndexParams(params).build(); @@ -234,10 +234,11 @@ private void writeCagraIndex(OutputStream os, Dataset dataset) throws Throwable index.destroyIndex(); } - private void writeBruteForceIndex(OutputStream os, Dataset dataset) throws Throwable { + private void writeBruteForceIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { BruteForceIndexParams params = new BruteForceIndexParams.Builder() - .withNumWriterThreads(32) // TODO: Make this configurable later. + .withNumWriterThreads(32) // TODO: Make this + // configurable later. .build(); long startTime = System.nanoTime(); var index = @@ -248,11 +249,11 @@ private void writeBruteForceIndex(OutputStream os, Dataset dataset) throws Throw index.destroyIndex(); } - private void writeHNSWIndex(OutputStream os, Dataset dataset) throws Throwable { + private void writeHNSWIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { if (dataset.size() < 2) { throw new IllegalArgumentException(dataset.size() + " vectors, less than min [2] required"); } - CagraIndexParams indexParams = cagraIndexParams(dataset.size()); + CagraIndexParams indexParams = cagraIndexParams((int) dataset.size()); long startTime = System.nanoTime(); var index = CagraIndex.newBuilder(resources).withDataset(dataset).withIndexParams(indexParams).build(); @@ -278,9 +279,11 @@ public void flush(int maxDoc, DocMap sortMap) throws IOException { private void writeField(CuVSFieldWriter fieldData) throws IOException { // TODO: Argh! https://github.com/rapidsai/cuvs/issues/698 List vectors = fieldData.getVectors(); - Dataset dataset = Dataset.create(vectors.size(), fieldData.fieldInfo().getVectorDimension()); - for (float[] vec : vectors) dataset.addVector(vec); - writeFieldInternal(fieldData.fieldInfo(), dataset); + CuVSMatrix.Builder builder = + CuVSMatrix.builder( + vectors.size(), fieldData.fieldInfo().getVectorDimension(), CuVSMatrix.DataType.FLOAT); + for (float[] vec : vectors) builder.addVector(vec); + writeFieldInternal(fieldData.fieldInfo(), builder.build()); } private void writeSortingField(CuVSFieldWriter fieldData, Sorter.DocMap sortMap) @@ -291,16 +294,19 @@ private void writeSortingField(CuVSFieldWriter fieldData, Sorter.DocMap sortMap) mapOldOrdToNewOrd(oldDocsWithFieldSet, sortMap, null, new2OldOrd, null); float[][] oldVectors = fieldData.getVectors().toArray(float[][]::new); - Dataset dataset = - Dataset.create(fieldData.getVectors().size(), fieldData.fieldInfo().getVectorDimension()); + CuVSMatrix.Builder builder = + CuVSMatrix.builder( + fieldData.getVectors().size(), + fieldData.fieldInfo().getVectorDimension(), + CuVSMatrix.DataType.FLOAT); for (int i = 0; i < oldVectors.length; i++) { float[] vec = oldVectors[new2OldOrd[i]]; - dataset.addVector(vec); + builder.addVector(vec); } - writeFieldInternal(fieldData.fieldInfo(), dataset); + writeFieldInternal(fieldData.fieldInfo(), builder.build()); } - private void writeFieldInternal(FieldInfo fieldInfo, Dataset dataset) throws IOException { + private void writeFieldInternal(FieldInfo fieldInfo, CuVSMatrix dataset) throws IOException { if (dataset.size() == 0) { writeEmpty(fieldInfo); return; @@ -349,18 +355,9 @@ private void writeFieldInternal(FieldInfo fieldInfo, Dataset dataset) throws IOE hnswIndexLength = cuvsIndex.getFilePointer() - hnswIndexOffset; } - // StringBuilder sb = new StringBuilder("writeField "); - // sb.append(": fieldInfo.name=").append(fieldInfo.name); - // sb.append(", fieldInfo.number=").append(fieldInfo.number); - // sb.append(", size=").append(vectors.length); - // sb.append(", cagraIndexLength=").append(cagraIndexLength); - // sb.append(", bruteForceIndexLength=").append(bruteForceIndexLength); - // sb.append(", hnswIndexLength=").append(hnswIndexLength); - // log.info(sb.toString()); - writeMeta( fieldInfo, - dataset.size(), + (int) dataset.size(), cagraIndexOffset, cagraIndexLength, bruteForceIndexOffset, @@ -424,16 +421,18 @@ static void handleThrowableWithIgnore(Throwable t, String msg) throws IOExceptio handleThrowable(t); } - /** Copies the vector values into dst. Returns the actual number of vectors copied. */ - private static int getVectorData(FloatVectorValues floatVectorValues, Dataset dataset) + /** + * Copies the vector values into dst. Returns the actual number of vectors + * copied. + */ + private static int getVectorData(FloatVectorValues floatVectorValues, CuVSMatrix.Builder builder) throws IOException { DocsWithFieldSet docsWithField = new DocsWithFieldSet(); int count = 0; KnnVectorValues.DocIndexIterator iter = floatVectorValues.iterator(); for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { assert iter.index() == count; - // dst[iter.index()] = floatVectorValues.vectorValue(iter.index()); - dataset.addVector(floatVectorValues.vectorValue(iter.index())); // is this correct? + builder.addVector(floatVectorValues.vectorValue(iter.index())); // is this correct? docsWithField.add(docV); count++; } @@ -452,9 +451,11 @@ public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOE }; // Also will be replaced with the cuVS merge api - Dataset dataset = Dataset.create(mergedVectorValues.size(), mergedVectorValues.dimension()); - getVectorData(mergedVectorValues, dataset); - writeFieldInternal(fieldInfo, dataset); + CuVSMatrix.Builder builder = + CuVSMatrix.builder( + mergedVectorValues.size(), mergedVectorValues.dimension(), CuVSMatrix.DataType.FLOAT); + getVectorData(mergedVectorValues, builder); + writeFieldInternal(fieldInfo, builder.build()); } catch (Throwable t) { handleThrowable(t); } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index e65e541f05..5c3f7d146c 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -17,10 +17,13 @@ import com.nvidia.cuvs.BruteForceIndex; import com.nvidia.cuvs.CagraIndex; +import com.nvidia.cuvs.CuVSMatrix; +import com.nvidia.cuvs.CuVSMatrix.Builder; +import com.nvidia.cuvs.CuVSMatrix.DataType; import com.nvidia.cuvs.CuVSResources; -import com.nvidia.cuvs.Dataset; import com.nvidia.cuvs.HnswIndex; import com.nvidia.cuvs.spi.CuVSProvider; +import java.lang.invoke.MethodHandle; import java.nio.file.Path; /*package-private*/ class FilterCuVSProvider implements CuVSProvider { @@ -65,7 +68,33 @@ public CagraIndex mergeCagraIndexes(CagraIndex[] arg0) throws Throwable { } @Override - public Dataset newDataset(int arg0, int arg1) throws UnsupportedOperationException { - return delegate.newDataset(arg0, arg1); + public Builder newMatrixBuilder(int size, int dimensions, DataType dataType) { + return delegate.newMatrixBuilder(size, dimensions, dataType); + } + + @Override + public MethodHandle newNativeMatrixBuilder() { + return delegate.newNativeMatrixBuilder(); + } + + @Override + public CuVSMatrix newMatrixFromArray(float[][] vectors) { + return delegate.newMatrixFromArray(vectors); + } + + @Override + public CuVSMatrix newMatrixFromArray(int[][] vectors) { + return delegate.newMatrixFromArray(vectors); + } + + @Override + public CuVSMatrix newMatrixFromArray(byte[][] vectors) { + return delegate.newMatrixFromArray(vectors); + } + + @Override + public com.nvidia.cuvs.TieredIndex.Builder newTieredIndexBuilder(CuVSResources cuVSResources) + throws UnsupportedOperationException { + return delegate.newTieredIndexBuilder(cuVSResources); } } diff --git a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index e14aa31324..6e486ee194 100644 --- a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -15,4 +15,4 @@ org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat -com.searchscale.lucene.vectorsearch.CuVSVectorsFormat +com.nvidia.cuvs.lucene.CuVSVectorsFormat diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVS.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVS.java index 7f66943030..124c2f9354 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVS.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVS.java @@ -128,7 +128,6 @@ public void testVectorSearch() throws IOException { log.info("Query size: " + numQueries + "x" + queries[0].length); log.info("TopK: " + topK); - // Query query = new CuVSKnnFloatVectorQuery("vector", queries[0], topK, topK, 1); Query query = new KnnFloatVectorQuery("vector", queries[0], topK); int correct[] = new int[topK]; for (int i = 0; i < topK; i++) correct[i] = expected.get(0).get(i); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java index 4937db311d..d19180eb43 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java @@ -28,10 +28,12 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; import org.apache.lucene.tests.util.TestUtil; import org.junit.BeforeClass; +import org.junit.Ignore; public class TestCuVSVectorsFormat extends BaseKnnVectorsFormatTestCase { @@ -43,9 +45,6 @@ public static void beforeClass() { @Override protected Codec getCodec() { return TestUtil.alwaysKnnVectorsFormat(new CuVSVectorsFormat()); - // For convenience, to sanitize the test code, one can comment out - // the supported check and use another format, e.g. - // return TestUtil.alwaysKnnVectorsFormat(new Lucene99HnswVectorsFormat()); } public void testMergeTwoSegsWithASingleDocPerSeg() throws Exception { @@ -126,4 +125,40 @@ public void testTwoVectorFieldsPerDoc() throws Exception { } } } + + @Override + // Overriding this method from superclass for the tests to only use float vector encoding + protected VectorEncoding randomVectorEncoding() { + return VectorEncoding.FLOAT32; + } + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testByteVectorScorerIteration() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testEmptyByteVectorData() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testMergingWithDifferentByteKnnFields() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testMismatchedFields() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testRandomBytes() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testSortedIndexBytes() {} } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java index c04cb586ae..d5f46d7370 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java @@ -35,7 +35,6 @@ public void testBasic() throws IOException { try (var indexIn = dir.openInput("test", IOContext.DEFAULT)) { var in = new IndexInputInputStream(indexIn); - // assertEquals(0x56, in.read()); byte[] ba = new byte[6]; assertEquals(6, in.read(ba)); assertArrayEquals(new byte[] {0x56, 0x10, 0x11, 0x12, 0x13, 0x14}, ba); @@ -78,6 +77,7 @@ public void testWithRandom() throws IOException { } try (var indexIn = dir.openInput("test", IOContext.DEFAULT)) { + // TODO: close this stream properly in a subsequent PR. var in = new IndexInputInputStream(indexIn); int i = 0; while (i < data.length) { From 2bbd0e3e6895247ebe3e11b688898eda591944f7 Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Mon, 11 Aug 2025 10:24:14 -0500 Subject: [PATCH 014/105] Merge pull request NVIDIA/cuvs-lucene#12 from dantegd/fea-add-ci Add CI files and workflows --- java/cuvs-lucene/.github/copy-pr-bot.yaml | 5 ++ java/cuvs-lucene/.github/workflows/build.yaml | 46 ++++++++++ java/cuvs-lucene/.github/workflows/pr.yaml | 85 +++++++++++++++++++ java/cuvs-lucene/.github/workflows/test.yaml | 37 ++++++++ java/cuvs-lucene/.gitignore | 3 +- java/cuvs-lucene/VERSION | 2 +- java/cuvs-lucene/build.sh | 26 ++++++ java/cuvs-lucene/ci/build_java.sh | 43 ++++++++++ java/cuvs-lucene/ci/release/update-version.sh | 45 ++++++++++ java/cuvs-lucene/ci/test_java.sh | 20 +++++ java/cuvs-lucene/dependencies.yaml | 75 ++++++++++++++++ 11 files changed, 385 insertions(+), 2 deletions(-) create mode 100644 java/cuvs-lucene/.github/copy-pr-bot.yaml create mode 100644 java/cuvs-lucene/.github/workflows/build.yaml create mode 100644 java/cuvs-lucene/.github/workflows/pr.yaml create mode 100644 java/cuvs-lucene/.github/workflows/test.yaml create mode 100644 java/cuvs-lucene/build.sh create mode 100644 java/cuvs-lucene/ci/build_java.sh create mode 100644 java/cuvs-lucene/ci/release/update-version.sh create mode 100644 java/cuvs-lucene/ci/test_java.sh create mode 100644 java/cuvs-lucene/dependencies.yaml diff --git a/java/cuvs-lucene/.github/copy-pr-bot.yaml b/java/cuvs-lucene/.github/copy-pr-bot.yaml new file mode 100644 index 0000000000..5d1c07e76a --- /dev/null +++ b/java/cuvs-lucene/.github/copy-pr-bot.yaml @@ -0,0 +1,5 @@ +# Configuration file for `copy-pr-bot` GitHub App +# https://docs.gha-runners.nvidia.com/platform/apps/copy-pr-bot/ + +enabled: true +auto_sync_draft: false \ No newline at end of file diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml new file mode 100644 index 0000000000..0270fcd5fb --- /dev/null +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -0,0 +1,46 @@ +name: build + +on: + push: + branches: + - "branch-*" + tags: + - v[0-9][0-9].[0-9][0-9].[0-9][0-9] + workflow_dispatch: + inputs: + branch: + description: | + branch: git branch the workflow run targets. + Required even when 'sha' is provided because it is also used for organizing artifacts. + required: true + type: string + date: + description: "date: Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" + required: true + type: string + sha: + description: "sha: full git commit SHA to check out" + required: true + type: string + build_type: + description: "build_type: one of [branch, nightly, pull-request]" + type: string + default: nightly + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + java-build: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + arch: "amd64" + date: ${{ inputs.date }} + container_image: "rapidsai/ci-conda:25.10-latest" + script: "ci/build_java.sh" + file_to_upload: "java/cuvs-java/target/" + sha: ${{ inputs.sha }} \ No newline at end of file diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml new file mode 100644 index 0000000000..b9a0284ce2 --- /dev/null +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -0,0 +1,85 @@ +name: pr +on: + push: + branches: + - "pull-request/[0-9]+" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + pr-builder: + needs: + - check-nightly-ci + - changed-files + - checks + - conda-java-tests + - telemetry-setup + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.10 + if: always() + with: + needs: ${{ toJSON(needs) }} + telemetry-setup: + continue-on-error: true + runs-on: ubuntu-latest + env: + OTEL_SERVICE_NAME: 'pr-cuvs' + steps: + - name: Telemetry setup + if: ${{ vars.TELEMETRY_ENABLED == 'true' }} + uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main + check-nightly-ci: + needs: telemetry-setup + runs-on: ubuntu-latest + env: + RAPIDS_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Check if nightly CI is passing + uses: rapidsai/shared-actions/check_nightly_success/dispatch@main + with: + repo: cuvs + max_days_without_success: 30 + changed-files: + needs: telemetry-setup + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-25.10 + with: + files_yaml: | + test_java: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!thirdparty/LICENSES/**' + checks: + needs: telemetry-setup + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.10 + with: + enable_check_generated_files: false + ignored_pr_jobs: "telemetry-summarize" + conda-java-tests: + needs: [conda-cpp-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java + with: + build_type: pull-request + node_type: "gpu-l4-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:25.10-latest" + script: "ci/test_java.sh" + file_to_upload: "java/cuvs-java/target/" + telemetry-summarize: + # This job must use a self-hosted runner to record telemetry traces. + runs-on: linux-amd64-cpu4 + needs: pr-builder + if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} + continue-on-error: true + steps: + - name: Telemetry summarize + uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main + env: + GH_TOKEN: ${{ github.token }} diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml new file mode 100644 index 0000000000..ab7da27463 --- /dev/null +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -0,0 +1,37 @@ +name: test + +on: + workflow_dispatch: + inputs: + branch: + description: | + branch: git branch the workflow run targets. + Required even when 'sha' is provided because it is also used for organizing artifacts. + required: true + type: string + date: + description: "date: Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" + required: true + type: string + sha: + description: "sha: full git commit SHA to check out" + required: true + type: string + build_type: + description: "build_type: one of [branch, nightly, pull-request]" + type: string + default: nightly + +jobs: + conda-java-tests: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 + with: + build_type: ${{ inputs.build_type }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + node_type: "gpu-l4-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:25.10-latest" + script: "ci/test_java.sh" \ No newline at end of file diff --git a/java/cuvs-lucene/.gitignore b/java/cuvs-lucene/.gitignore index 1de565933b..63d9e5e0c6 100644 --- a/java/cuvs-lucene/.gitignore +++ b/java/cuvs-lucene/.gitignore @@ -1 +1,2 @@ -target \ No newline at end of file +target +**/.DS_Store \ No newline at end of file diff --git a/java/cuvs-lucene/VERSION b/java/cuvs-lucene/VERSION index 3af4bda020..296e35288d 100644 --- a/java/cuvs-lucene/VERSION +++ b/java/cuvs-lucene/VERSION @@ -1 +1 @@ -25.08.00 +25.10.00 diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh new file mode 100644 index 0000000000..ca83606b29 --- /dev/null +++ b/java/cuvs-lucene/build.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Copyright (c) 2025, NVIDIA CORPORATION. + +set -e -u -o pipefail + +ARGS="$*" +NUMARGS=$# + +CURDIR=$(cd "$(dirname "$0")"; pwd) +VERSION="25.10.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked +GROUP_ID="com.nvidia.cuvs" + +function hasArg { + (( NUMARGS != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") +} + +MAVEN_VERIFY_ARGS=() +if ! hasArg --run-java-tests; then + MAVEN_VERIFY_ARGS=("-DskipTests") +fi + +mvn verify "${MAVEN_VERIFY_ARGS[@]}" \ + && mvn install:install-file -Dfile=./target/cuvs-lucene-$VERSION.jar -DgroupId=$GROUP_ID -DartifactId=cuvs-lucene -Dversion=$VERSION -Dpackaging=jar \ + && cp pom.xml ./target/ +- \ No newline at end of file diff --git a/java/cuvs-lucene/ci/build_java.sh b/java/cuvs-lucene/ci/build_java.sh new file mode 100644 index 0000000000..33444a799a --- /dev/null +++ b/java/cuvs-lucene/ci/build_java.sh @@ -0,0 +1,43 @@ +/bin/bash +# Copyright (c) 2025, NVIDIA CORPORATION. + +set -euo pipefail + +# TODO: Remove this argument-handling when build and test workflows are separated, +# and test_java.sh no longer calls build_java.sh +# ref: https://github.com/rapidsai/cuvs/issues/868 +EXTRA_BUILD_ARGS=() +if [[ "${1:-}" == "--run-java-tests" ]]; then + EXTRA_BUILD_ARGS+=("--run-java-tests") +fi + +. /opt/conda/etc/profile.d/conda.sh + +rapids-logger "Generate Java testing dependencies" + +ENV_YAML_DIR="$(mktemp -d)" + +rapids-dependency-file-generator \ + --output conda \ + --file-key java \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n java + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate java +set -u + +rapids-print-env + +EXITCODE=0 +trap "EXITCODE=1" ERR +set +e + +rapids-logger "Run Java build" + +bash ./build.sh ${EXTRA_BUILD_ARGS[@]}" + +rapids-logger "Build script exiting with value: $EXITCODE" +exit ${EXITCODE} \ No newline at end of file diff --git a/java/cuvs-lucene/ci/release/update-version.sh b/java/cuvs-lucene/ci/release/update-version.sh new file mode 100644 index 0000000000..19232f2883 --- /dev/null +++ b/java/cuvs-lucene/ci/release/update-version.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Copyright (c) 2020-2025, NVIDIA CORPORATION. +######################## +# CUVS Version Updater # +######################## + +## Usage +# bash update-version.sh + + +# Format is YY.MM.PP - no leading 'v' or trailing 'a' +NEXT_FULL_TAG=$1 + +# Get current version +CURRENT_TAG=$(git tag --merged HEAD | grep -xE '^v.*' | sort --version-sort | tail -n 1 | tr -d 'v') +CURRENT_MAJOR=$(echo "$CURRENT_TAG" | awk '{split($0, a, "."); print a[1]}') +CURRENT_MINOR=$(echo "$CURRENT_TAG" | awk '{split($0, a, "."); print a[2]}') +CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR} + +# Get . for next version +NEXT_MAJOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[1]}') +NEXT_MINOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[2]}') +NEXT_PATCH=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[3]}') +NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} + +# Need to distutils-normalize the original version +NEXT_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_SHORT_TAG}'))") +NEXT_UCXX_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_UCXX_SHORT_TAG}'))") +PATCH_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_PATCH}'))") + +echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" + +# Inplace sed replace; workaround for Linux and Mac +function sed_runner() { + sed -i.bak ''"$1"'' "$2" && rm -f "${2}".bak +} + +# Update Java version +NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}" +sed_runner "s/VERSION=\".*\"/VERSION=\"${NEXT_FULL_JAVA_TAG}\"/g" build.sh +sed_runner "/.*/s//${NEXT_FULL_JAVA_TAG}<\/version>/g" pom.xml + +sed_runner "s| CuVS [[:digit:]]\{2\}\.[[:digit:]]\{2\} | CuVS ${NEXT_SHORT_TAG} |g" README.md +sed_runner "s|-[[:digit:]]\{2\}\.[[:digit:]]\{2\}\.[[:digit:]]\{1,2\}\.jar|-${NEXT_FULL_JAVA_TAG}\.jar|g" java/examples/README.md +sed_runner "s|/[[:digit:]]\{2\}\.[[:digit:]]\{2\}\.[[:digit:]]\{1,2\}/|/${NEXT_FULL_JAVA_TAG}/|g" java/examples/README.md diff --git a/java/cuvs-lucene/ci/test_java.sh b/java/cuvs-lucene/ci/test_java.sh new file mode 100644 index 0000000000..5dd12c8314 --- /dev/null +++ b/java/cuvs-lucene/ci/test_java.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Copyright (c) 2025, NVIDIA CORPORATION. + +set -euo pipefail + +EXITCODE=0 +trap "EXITCODE=1" ERR +set +e + +rapids-logger "Check GPU usage" +nvidia-smi + +rapids-logger "Run Java build and tests" + +# TODO: switch to installing pre-built artifacts instead of rebuilding in test jobs +# ref: https://github.com/rapidsai/cuvs/issues/868 +ci/build_java.sh --run-java-tests + +rapids-logger "Test script exiting with value: $EXITCODE" +exit ${EXITCODE} \ No newline at end of file diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml new file mode 100644 index 0000000000..7dfb4a4913 --- /dev/null +++ b/java/cuvs-lucene/dependencies.yaml @@ -0,0 +1,75 @@ +# Dependency list for https://github.com/rapidsai/dependency-file-generator +files: + all: + output: conda + matrix: + cuda: ["12.9"] + arch: [x86_64, aarch64] + includes: + - cuda + - cuda_version + - java + checks: + output: none + includes: + - checks + java: + output: none + includes: + - cuda + - cuda_version + - java +channels: + - rapidsai + - rapidsai-nightly + - conda-forge +dependencies: + checks: + common: + - output_types: [conda, requirements] + packages: + - pre-commit + cuda_version: + specific: + - output_types: conda + matrices: + - matrix: + cuda: "12.0" + packages: + - cuda-version=12.0 + - matrix: + cuda: "12.2" + packages: + - cuda-version=12.2 + - matrix: + cuda: "12.5" + packages: + - cuda-version=12.5 + - matrix: + cuda: "12.8" + packages: + - cuda-version=12.8 + - matrix: + cuda: "12.9" + packages: + - cuda-version=12.9 + cuda: + specific: + - output_types: conda + matrices: + - matrix: + cuda: "12.*" + packages: + - cuda-nvtx-dev + - cuda-cudart-dev + - cuda-profiler-api + - libcublas-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev + java: + common: + - output_types: conda + packages: + - maven + - openjdk=22.* From 906cca6568ca43c86802856257695ae67d87f58a Mon Sep 17 00:00:00 2001 From: Dante Gama Dessavre Date: Mon, 25 Aug 2025 20:59:37 -0500 Subject: [PATCH 015/105] Enable CI (NVIDIA/cuvs-lucene#15) This PR adds a few additional needed changes to enable CI. - change permission to make `.sh` files executable - removes unnecessary configuration from `.pre-commit-config.yaml`, `CODEOWNERS`, `dependencies.yaml` - updates all `pre-commit` hooks - adds `rapids-dependency-file-generator` hook (to validated `dependencies.yaml`) - fixes all issues found by `pre-commit` hooks (broken shell scripts, trailing whitespace, etc.) - fixes versioning in `pom.xml` Authors: - Dante Gama Dessavre (https://github.com/dantegd) - James Lamb (https://github.com/jameslamb) Approvers: - James Lamb (https://github.com/jameslamb) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/15 --- java/cuvs-lucene/.github/CODEOWNERS | 36 ++++++------------- java/cuvs-lucene/.github/copy-pr-bot.yaml | 2 +- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 6 +--- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/.gitignore | 3 +- java/cuvs-lucene/.pre-commit-config.yaml | 18 +++++----- java/cuvs-lucene/CONTRIBUTING.md | 14 ++++---- java/cuvs-lucene/README.md | 2 +- java/cuvs-lucene/build.sh | 2 -- java/cuvs-lucene/ci/build_java.sh | 7 ++-- java/cuvs-lucene/ci/check_style.sh | 19 ++++++++++ java/cuvs-lucene/ci/release/update-version.sh | 13 ++----- java/cuvs-lucene/ci/test_java.sh | 2 +- .../all_cuda-129_arch-aarch64.yaml | 16 +++++++++ .../all_cuda-129_arch-x86_64.yaml | 16 +++++++++ java/cuvs-lucene/dependencies.yaml | 2 -- java/cuvs-lucene/pom.xml | 3 +- 18 files changed, 94 insertions(+), 71 deletions(-) mode change 100644 => 100755 java/cuvs-lucene/build.sh mode change 100644 => 100755 java/cuvs-lucene/ci/build_java.sh create mode 100755 java/cuvs-lucene/ci/check_style.sh mode change 100644 => 100755 java/cuvs-lucene/ci/release/update-version.sh mode change 100644 => 100755 java/cuvs-lucene/ci/test_java.sh create mode 100644 java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml create mode 100644 java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml diff --git a/java/cuvs-lucene/.github/CODEOWNERS b/java/cuvs-lucene/.github/CODEOWNERS index 22d9e8d886..4b8b023cb7 100644 --- a/java/cuvs-lucene/.github/CODEOWNERS +++ b/java/cuvs-lucene/.github/CODEOWNERS @@ -1,34 +1,20 @@ -#cpp code owners -cpp/ @rapidsai/cuvs-cpp-codeowners -examples/cpp/ @rapidsai/cuvs-cpp-codeowners -examples/c/ @rapidsai/cuvs-cpp-codeowners +# default group +# (will be overridden by more specific matches below) +* @rapidsai/lucene-cuvs-write -#java code owners -java/ @rapidsai/cuvs-java-codeowners -examples/java/ @rapidsai/cuvs-java-codeowners - -#python code owners -python/ @rapidsai/cuvs-python-codeowners - -#rust code owners -rust/ @rapidsai/cuvs-rust-codeowners -examples/rust/ @rapidsai/cuvs-rust-codeowners +# java code owners +src/ @rapidsai/cuvs-java-codeowners # docs code owners -docs/ @rapidsai/cuvs-docs-codeowners -README.md @rapidsai/cuvs-docs-codeowners -CONTRIBUTING.md @rapidsai/cuvs-docs-codeowners -LICENSE @rapidsai/cuvs-docs-codeowners - -#cmake code owners -pom.xml @rapidsai/cuvs-build-codeowners +LICENSE @rapidsai/cuvs-docs-codeowners +/README.md @rapidsai/cuvs-docs-codeowners -#CI code owners +# CI code owners /.github/ @rapidsai/ci-codeowners /ci/ @rapidsai/ci-codeowners # packaging code owners +build.sh @rapidsai/packaging-codeowners +dependencies.yaml @rapidsai/packaging-codeowners +pom.xml @rapidsai/cuvs-build-codeowners /.pre-commit-config.yaml @rapidsai/packaging-codeowners - - -# src/ @rapidsai/cuvs-java-codeowners diff --git a/java/cuvs-lucene/.github/copy-pr-bot.yaml b/java/cuvs-lucene/.github/copy-pr-bot.yaml index 5d1c07e76a..4e99200c8c 100644 --- a/java/cuvs-lucene/.github/copy-pr-bot.yaml +++ b/java/cuvs-lucene/.github/copy-pr-bot.yaml @@ -2,4 +2,4 @@ # https://docs.gha-runners.nvidia.com/platform/apps/copy-pr-bot/ enabled: true -auto_sync_draft: false \ No newline at end of file +auto_sync_draft: false diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 0270fcd5fb..11429f7245 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -43,4 +43,4 @@ jobs: container_image: "rapidsai/ci-conda:25.10-latest" script: "ci/build_java.sh" file_to_upload: "java/cuvs-java/target/" - sha: ${{ inputs.sha }} \ No newline at end of file + sha: ${{ inputs.sha }} diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index b9a0284ce2..864c98865f 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -47,12 +47,8 @@ jobs: files_yaml: | test_java: - '**' - - '!.devcontainer/**' - '!.pre-commit-config.yaml' - '!README.md' - - '!docs/**' - - '!img/**' - - '!thirdparty/LICENSES/**' checks: needs: telemetry-setup secrets: inherit @@ -61,7 +57,7 @@ jobs: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" conda-java-tests: - needs: [conda-cpp-build, changed-files] + needs: [changed-files, checks] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index ab7da27463..52d6a142ed 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -34,4 +34,4 @@ jobs: node_type: "gpu-l4-latest-1" arch: "amd64" container_image: "rapidsai/ci-conda:25.10-latest" - script: "ci/test_java.sh" \ No newline at end of file + script: "ci/test_java.sh" diff --git a/java/cuvs-lucene/.gitignore b/java/cuvs-lucene/.gitignore index 63d9e5e0c6..7738e864cb 100644 --- a/java/cuvs-lucene/.gitignore +++ b/java/cuvs-lucene/.gitignore @@ -1,2 +1,3 @@ +*.jar target -**/.DS_Store \ No newline at end of file +**/.DS_Store diff --git a/java/cuvs-lucene/.pre-commit-config.yaml b/java/cuvs-lucene/.pre-commit-config.yaml index 2e2dace2ff..9970e498da 100644 --- a/java/cuvs-lucene/.pre-commit-config.yaml +++ b/java/cuvs-lucene/.pre-commit-config.yaml @@ -2,14 +2,14 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-json - id: trailing-whitespace - id: end-of-file-fixer - id: check-symlinks - repo: https://github.com/rapidsai/pre-commit-hooks - rev: v0.6.0 + rev: v0.7.0 hooks: - id: verify-copyright files: | @@ -18,14 +18,12 @@ repos: CMakeLists[.]txt$| CMakeLists_standalone[.]txt$| meta[.]yaml$ - exclude: | - (?x) - docs/source/sphinxext/github_link\.py| - cpp/cmake/modules/FindAVX\.cmake - - id: verify-alpha-spec - - id: verify-codeowners - args: [--fix, --project-prefix=lucene-cuvs] + - repo: https://github.com/rapidsai/dependency-file-generator + rev: v1.19.2 + hooks: + - id: rapids-dependency-file-generator + args: ["--clean"] - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.10.0.1 + rev: v0.11.0.1 hooks: - id: shellcheck diff --git a/java/cuvs-lucene/CONTRIBUTING.md b/java/cuvs-lucene/CONTRIBUTING.md index cd5d29b2d8..ff9a3b91f7 100644 --- a/java/cuvs-lucene/CONTRIBUTING.md +++ b/java/cuvs-lucene/CONTRIBUTING.md @@ -106,26 +106,26 @@ implementation of the issue, ask them in the issue instead of the PR. ``` Developer Certificate of Origin Version 1.1 - + Copyright (C) 2004, 2006 The Linux Foundation and its contributors. 1 Letterman Drive Suite D4700 San Francisco, CA, 94129 - + Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. ``` ``` Developer's Certificate of Origin 1.1 - + By making a contribution to this project, I certify that: - + (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or - + (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or - + (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. - + (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. ``` diff --git a/java/cuvs-lucene/README.md b/java/cuvs-lucene/README.md index 0cbf52ac4f..cc4da5a1d1 100644 --- a/java/cuvs-lucene/README.md +++ b/java/cuvs-lucene/README.md @@ -26,4 +26,4 @@ export LD_LIBRARY_PATH={ PATH TO YOUR LOCAL libcuvs_c.so }:$LD_LIBRARY_PATH && m ``` > [!NOTE] -> The code style format is automatically enforced (including the missing license header, if any) using the [Spotless maven plugin](https://github.com/diffplug/spotless/tree/main/plugin-maven). This currently happens in the maven's `validate` stage. \ No newline at end of file +> The code style format is automatically enforced (including the missing license header, if any) using the [Spotless maven plugin](https://github.com/diffplug/spotless/tree/main/plugin-maven). This currently happens in the maven's `validate` stage. diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh old mode 100644 new mode 100755 index ca83606b29..a68169555d --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -7,7 +7,6 @@ set -e -u -o pipefail ARGS="$*" NUMARGS=$# -CURDIR=$(cd "$(dirname "$0")"; pwd) VERSION="25.10.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked GROUP_ID="com.nvidia.cuvs" @@ -23,4 +22,3 @@ fi mvn verify "${MAVEN_VERIFY_ARGS[@]}" \ && mvn install:install-file -Dfile=./target/cuvs-lucene-$VERSION.jar -DgroupId=$GROUP_ID -DartifactId=cuvs-lucene -Dversion=$VERSION -Dpackaging=jar \ && cp pom.xml ./target/ -- \ No newline at end of file diff --git a/java/cuvs-lucene/ci/build_java.sh b/java/cuvs-lucene/ci/build_java.sh old mode 100644 new mode 100755 index 33444a799a..ebc9b16073 --- a/java/cuvs-lucene/ci/build_java.sh +++ b/java/cuvs-lucene/ci/build_java.sh @@ -1,4 +1,4 @@ -/bin/bash +#!/bin/bash # Copyright (c) 2025, NVIDIA CORPORATION. set -euo pipefail @@ -11,6 +11,7 @@ if [[ "${1:-}" == "--run-java-tests" ]]; then EXTRA_BUILD_ARGS+=("--run-java-tests") fi +# shellcheck disable=SC1091 . /opt/conda/etc/profile.d/conda.sh rapids-logger "Generate Java testing dependencies" @@ -37,7 +38,7 @@ set +e rapids-logger "Run Java build" -bash ./build.sh ${EXTRA_BUILD_ARGS[@]}" +bash ./build.sh "${EXTRA_BUILD_ARGS[@]}" rapids-logger "Build script exiting with value: $EXITCODE" -exit ${EXITCODE} \ No newline at end of file +exit ${EXITCODE} diff --git a/java/cuvs-lucene/ci/check_style.sh b/java/cuvs-lucene/ci/check_style.sh new file mode 100755 index 0000000000..07126990f9 --- /dev/null +++ b/java/cuvs-lucene/ci/check_style.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright (c) 2020-2025, NVIDIA CORPORATION. + +set -euo pipefail + +rapids-logger "Create checks conda environment" +# shellcheck disable=SC1091 +. /opt/conda/etc/profile.d/conda.sh + +rapids-dependency-file-generator \ + --output conda \ + --file-key checks \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml + +rapids-mamba-retry env create --yes -f env.yaml -n checks +conda activate checks + +# Run pre-commit checks +pre-commit run --all-files --show-diff-on-failure diff --git a/java/cuvs-lucene/ci/release/update-version.sh b/java/cuvs-lucene/ci/release/update-version.sh old mode 100644 new mode 100755 index 19232f2883..c7ea7a1bce --- a/java/cuvs-lucene/ci/release/update-version.sh +++ b/java/cuvs-lucene/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2020-2025, NVIDIA CORPORATION. +# Copyright (c) 2025, NVIDIA CORPORATION. ######################## # CUVS Version Updater # ######################## @@ -13,9 +13,6 @@ NEXT_FULL_TAG=$1 # Get current version CURRENT_TAG=$(git tag --merged HEAD | grep -xE '^v.*' | sort --version-sort | tail -n 1 | tr -d 'v') -CURRENT_MAJOR=$(echo "$CURRENT_TAG" | awk '{split($0, a, "."); print a[1]}') -CURRENT_MINOR=$(echo "$CURRENT_TAG" | awk '{split($0, a, "."); print a[2]}') -CURRENT_SHORT_TAG=${CURRENT_MAJOR}.${CURRENT_MINOR} # Get . for next version NEXT_MAJOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[1]}') @@ -23,9 +20,7 @@ NEXT_MINOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[2]}') NEXT_PATCH=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[3]}') NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} -# Need to distutils-normalize the original version -NEXT_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_SHORT_TAG}'))") -NEXT_UCXX_SHORT_TAG_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_UCXX_SHORT_TAG}'))") +# Strip leading 0s in versions, so e.g. '25.10.00' becomes '25.10.0' PATCH_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_PATCH}'))") echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" @@ -38,8 +33,6 @@ function sed_runner() { # Update Java version NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}" sed_runner "s/VERSION=\".*\"/VERSION=\"${NEXT_FULL_JAVA_TAG}\"/g" build.sh -sed_runner "/.*/s//${NEXT_FULL_JAVA_TAG}<\/version>/g" pom.xml +sed_runner "/.*/s//${NEXT_FULL_JAVA_TAG}<\/version>/g" pom.xml sed_runner "s| CuVS [[:digit:]]\{2\}\.[[:digit:]]\{2\} | CuVS ${NEXT_SHORT_TAG} |g" README.md -sed_runner "s|-[[:digit:]]\{2\}\.[[:digit:]]\{2\}\.[[:digit:]]\{1,2\}\.jar|-${NEXT_FULL_JAVA_TAG}\.jar|g" java/examples/README.md -sed_runner "s|/[[:digit:]]\{2\}\.[[:digit:]]\{2\}\.[[:digit:]]\{1,2\}/|/${NEXT_FULL_JAVA_TAG}/|g" java/examples/README.md diff --git a/java/cuvs-lucene/ci/test_java.sh b/java/cuvs-lucene/ci/test_java.sh old mode 100644 new mode 100755 index 5dd12c8314..f643bda8ff --- a/java/cuvs-lucene/ci/test_java.sh +++ b/java/cuvs-lucene/ci/test_java.sh @@ -17,4 +17,4 @@ rapids-logger "Run Java build and tests" ci/build_java.sh --run-java-tests rapids-logger "Test script exiting with value: $EXITCODE" -exit ${EXITCODE} \ No newline at end of file +exit ${EXITCODE} diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml new file mode 100644 index 0000000000..dd3018c5b1 --- /dev/null +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -0,0 +1,16 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- conda-forge +dependencies: +- cuda-cudart-dev +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=12.9 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- maven +- openjdk=22.* +name: all_cuda-129_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml new file mode 100644 index 0000000000..8d067104f8 --- /dev/null +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -0,0 +1,16 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- conda-forge +dependencies: +- cuda-cudart-dev +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=12.9 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- maven +- openjdk=22.* +name: all_cuda-129_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index 7dfb4a4913..fe9f3d5b7f 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -20,8 +20,6 @@ files: - cuda_version - java channels: - - rapidsai - - rapidsai-nightly - conda-forge dependencies: checks: diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index c79f26eec3..073781929b 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -4,7 +4,8 @@ 4.0.0 com.nvidia.cuvs.lucene cuvs-lucene - 0.0.1-SNAPSHOT + + 25.10.0 cuvs-lucene jar From 9e23cba8e1a67a805af24706aa46e919f3bfa292 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 27 Aug 2025 13:54:49 -0500 Subject: [PATCH 016/105] simplify build scripts (NVIDIA/cuvs-lucene#19) * updates `rapids-dependency-file-generator`, to pull in changes from https://github.com/rapidsai/dependency-file-generator/pull/163 * removes unnecessary error-trapping in shell scripts Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/cuvs-lucene/pull/19 --- java/cuvs-lucene/.pre-commit-config.yaml | 4 ++-- java/cuvs-lucene/ci/build_java.sh | 7 ------- java/cuvs-lucene/ci/test_java.sh | 7 ------- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/java/cuvs-lucene/.pre-commit-config.yaml b/java/cuvs-lucene/.pre-commit-config.yaml index 9970e498da..8d8052c38c 100644 --- a/java/cuvs-lucene/.pre-commit-config.yaml +++ b/java/cuvs-lucene/.pre-commit-config.yaml @@ -19,10 +19,10 @@ repos: CMakeLists_standalone[.]txt$| meta[.]yaml$ - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.19.2 + rev: v1.20.0 hooks: - id: rapids-dependency-file-generator - args: ["--clean"] + args: ["--clean", "--warn-all", "--strict"] - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.11.0.1 hooks: diff --git a/java/cuvs-lucene/ci/build_java.sh b/java/cuvs-lucene/ci/build_java.sh index ebc9b16073..1338da897c 100755 --- a/java/cuvs-lucene/ci/build_java.sh +++ b/java/cuvs-lucene/ci/build_java.sh @@ -32,13 +32,6 @@ set -u rapids-print-env -EXITCODE=0 -trap "EXITCODE=1" ERR -set +e - rapids-logger "Run Java build" bash ./build.sh "${EXTRA_BUILD_ARGS[@]}" - -rapids-logger "Build script exiting with value: $EXITCODE" -exit ${EXITCODE} diff --git a/java/cuvs-lucene/ci/test_java.sh b/java/cuvs-lucene/ci/test_java.sh index f643bda8ff..783b73efda 100755 --- a/java/cuvs-lucene/ci/test_java.sh +++ b/java/cuvs-lucene/ci/test_java.sh @@ -3,10 +3,6 @@ set -euo pipefail -EXITCODE=0 -trap "EXITCODE=1" ERR -set +e - rapids-logger "Check GPU usage" nvidia-smi @@ -15,6 +11,3 @@ rapids-logger "Run Java build and tests" # TODO: switch to installing pre-built artifacts instead of rebuilding in test jobs # ref: https://github.com/rapidsai/cuvs/issues/868 ci/build_java.sh --run-java-tests - -rapids-logger "Test script exiting with value: $EXITCODE" -exit ${EXITCODE} From 15096526c0b334745ce2fdb2a2ffc2a6333beb67 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 2 Sep 2025 15:03:02 -0500 Subject: [PATCH 017/105] Build and test with CUDA 13.0 (NVIDIA/cuvs-lucene#20) Contributes to https://github.com/rapidsai/build-planning/issues/208 * uses CUDA 13.0.0 to build and test (using the same patterns from the `cuvs-java` tests, in https://github.com/rapidsai/cuvs/pull/1273) ## Notes for Reviewers This switches GitHub Actions workflows to the `cuda13.0` branch from here: https://github.com/rapidsai/shared-workflows/pull/413 A future round of PRs will revert that back to `branch-25.10`, once all of RAPIDS supports CUDA 13. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Jake Awe (https://github.com/AyodeAwe) - Ben Frederickson (https://github.com/benfred) - rhdong (https://github.com/rhdong) URL: https://github.com/rapidsai/cuvs-lucene/pull/20 --- java/cuvs-lucene/.github/workflows/build.yaml | 13 +++++++-- java/cuvs-lucene/.github/workflows/pr.yaml | 19 +++++++++---- java/cuvs-lucene/.github/workflows/test.yaml | 10 +++++-- .../all_cuda-130_arch-aarch64.yaml | 16 +++++++++++ .../all_cuda-130_arch-x86_64.yaml | 16 +++++++++++ java/cuvs-lucene/dependencies.yaml | 27 ++++++++++--------- 6 files changed, 79 insertions(+), 22 deletions(-) create mode 100644 java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml create mode 100644 java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 11429f7245..d426f9f685 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -34,13 +34,22 @@ concurrency: jobs: java-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda13.0 + # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. + # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. + strategy: + fail-fast: false + matrix: + cuda_version: + - '12.9.1' + - '13.0.0' with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} arch: "amd64" date: ${{ inputs.date }} - container_image: "rapidsai/ci-conda:25.10-latest" + container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/build_java.sh" file_to_upload: "java/cuvs-java/target/" + artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" sha: ${{ inputs.sha }} diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 864c98865f..06a66252fd 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.10 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda13.0 if: always() with: needs: ${{ toJSON(needs) }} @@ -42,7 +42,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-25.10 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@cuda13.0 with: files_yaml: | test_java: @@ -52,22 +52,31 @@ jobs: checks: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.10 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda13.0 with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" conda-java-tests: needs: [changed-files, checks] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda13.0 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java + # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. + # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. + strategy: + fail-fast: false + matrix: + cuda_version: + - '12.9.1' + - '13.0.0' with: build_type: pull-request node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:25.10-latest" + container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" file_to_upload: "java/cuvs-java/target/" + artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" telemetry-summarize: # This job must use a self-hosted runner to record telemetry traces. runs-on: linux-amd64-cpu4 diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 52d6a142ed..2f7c4f6c8a 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -25,7 +25,13 @@ on: jobs: conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda13.0 + strategy: + fail-fast: false + matrix: + cuda_version: + - '12.9.1' + - '13.0.0' with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} @@ -33,5 +39,5 @@ jobs: sha: ${{ inputs.sha }} node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:25.10-latest" + container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml new file mode 100644 index 0000000000..77eaa69ae4 --- /dev/null +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -0,0 +1,16 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- conda-forge +dependencies: +- cuda-cudart-dev +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=13.0 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- maven +- openjdk=22.* +name: all_cuda-130_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml new file mode 100644 index 0000000000..ee8a720584 --- /dev/null +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -0,0 +1,16 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- conda-forge +dependencies: +- cuda-cudart-dev +- cuda-nvtx-dev +- cuda-profiler-api +- cuda-version=13.0 +- libcublas-dev +- libcurand-dev +- libcusolver-dev +- libcusparse-dev +- maven +- openjdk=22.* +name: all_cuda-130_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index fe9f3d5b7f..4c6cec0c6a 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -3,7 +3,7 @@ files: all: output: conda matrix: - cuda: ["12.9"] + cuda: ["12.9", "13.0"] arch: [x86_64, aarch64] includes: - cuda @@ -51,20 +51,21 @@ dependencies: cuda: "12.9" packages: - cuda-version=12.9 - cuda: - specific: - - output_types: conda - matrices: - matrix: - cuda: "12.*" + cuda: "13.0" packages: - - cuda-nvtx-dev - - cuda-cudart-dev - - cuda-profiler-api - - libcublas-dev - - libcurand-dev - - libcusolver-dev - - libcusparse-dev + - cuda-version=13.0 + cuda: + common: + - output_types: [conda] + packages: + - cuda-nvtx-dev + - cuda-cudart-dev + - cuda-profiler-api + - libcublas-dev + - libcurand-dev + - libcusolver-dev + - libcusparse-dev java: common: - output_types: conda From d8af7fe7783b500fc7a46f48fd103fe57cf821cc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 5 Sep 2025 14:57:10 -0500 Subject: [PATCH 018/105] Use branch-25.10 again (NVIDIA/cuvs-lucene#21) Contributes to https://github.com/rapidsai/build-planning/issues/208 Now that https://github.com/rapidsai/shared-workflows/pull/413 is merged, this converts all GitHub Actions references from `@cuda13.0` back to `branch-25.10`. ## Notes for Reviewers This is safe to admin-merge because the change is a no-op... configs on those 2 branches are identical. --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 8 ++++---- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index d426f9f685..f1085f3388 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -34,7 +34,7 @@ concurrency: jobs: java-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda13.0 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 06a66252fd..e7fffb5949 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda13.0 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.10 if: always() with: needs: ${{ toJSON(needs) }} @@ -42,7 +42,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@cuda13.0 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-25.10 with: files_yaml: | test_java: @@ -52,14 +52,14 @@ jobs: checks: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda13.0 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.10 with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" conda-java-tests: needs: [changed-files, checks] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda13.0 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 2f7c4f6c8a..4a8f5ae497 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -25,7 +25,7 @@ on: jobs: conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda13.0 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 strategy: fail-fast: false matrix: From 68f04dc8d0c24707055ba654eb31c62e5c4a231d Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Mon, 22 Sep 2025 09:13:04 -0500 Subject: [PATCH 019/105] Configure repo for automatic release notes generation (NVIDIA/cuvs-lucene#22) This PR configures this repo (adds a `.github/release.yml` file) for automatic release notes generation. See https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes. This will fix such issues as hotfix releases including wrong content (e.g [cugraph v25.04.01](https://github.com/rapidsai/cugraph/releases/tag/v25.04.01) contains 50+ PRs from the full v25.04.00 release when it should only contain the single hotfix PR (#5017).) Example auto-generated release notes: https://github.com/rapidsai/literate-octo-potato/releases/tag/v25.10.00a --- java/cuvs-lucene/.github/release.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 java/cuvs-lucene/.github/release.yml diff --git a/java/cuvs-lucene/.github/release.yml b/java/cuvs-lucene/.github/release.yml new file mode 100644 index 0000000000..2c9a85805c --- /dev/null +++ b/java/cuvs-lucene/.github/release.yml @@ -0,0 +1,27 @@ +# GitHub Auto-Generated Release Notes Configuration for RAPIDS +# This file configures how GitHub automatically generates release notes + +changelog: + exclude: + labels: + - ignore-for-release + - dependencies + authors: + - rapids-bot[bot] + - dependabot[bot] + categories: + - title: 🚨 Breaking Changes + labels: + - breaking + - title: 🐛 Bug Fixes + labels: + - bug + - title: 📖 Documentation + labels: + - doc + - title: 🚀 New Features + labels: + - feature request + - title: 🛠️ Improvements + labels: + - improvement From 22c71b2c1a23160f69d110de1f8041b3021d489d Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Thu, 25 Sep 2025 02:40:38 +0530 Subject: [PATCH 020/105] Fix Tests/CI, refactor merge to call CAGRA's merge(), implement CAGRA prefiltering (NVIDIA/cuvs-lucene#14) Refactoring, CI fixes (pulling libcuvs from pypi if not found), prefiltering support. Added tests: * TestCuVSGaps (for missing vectors in documents) * TestCuVSDeletedDocuments (for deleted vectors, that will leverage prefiltering) * TestMerge (dedicated test for testing merges) Authors: - Ishan Chattopadhyaya (https://github.com/chatman) - Vivek Narang (https://github.com/narangvivek10) Approvers: - Mike Sarahan (https://github.com/msarahan) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/14 --- java/cuvs-lucene/ci/build_java.sh | 21 + .../all_cuda-129_arch-aarch64.yaml | 2 + .../all_cuda-129_arch-x86_64.yaml | 2 + .../all_cuda-130_arch-aarch64.yaml | 2 + .../all_cuda-130_arch-x86_64.yaml | 2 + java/cuvs-lucene/dependencies.yaml | 2 + java/cuvs-lucene/pom.xml | 2 +- .../com/nvidia/cuvs/lucene/CuVSIndex.java | 10 +- .../cuvs/lucene/CuVSKnnFloatVectorQuery.java | 8 +- .../nvidia/cuvs/lucene/CuVSVectorsFormat.java | 6 + .../nvidia/cuvs/lucene/CuVSVectorsReader.java | 23 +- .../nvidia/cuvs/lucene/CuVSVectorsWriter.java | 261 +++- .../cuvs/lucene/FilterCuVSProvider.java | 21 +- .../java/com/nvidia/cuvs/lucene/Utils.java | 48 + .../cuvs/lucene/TestCuVSDeletedDocuments.java | 342 +++++ .../com/nvidia/cuvs/lucene/TestCuVSGaps.java | 197 +++ ...va => TestCuVSRandomizedVectorSearch.java} | 86 +- .../cuvs/lucene/TestCuVSVectorsFormat.java | 4 +- .../lucene/TestIndexOutputOutputStream.java | 11 +- .../com/nvidia/cuvs/lucene/TestMerge.java | 1169 +++++++++++++++++ .../com/nvidia/cuvs/lucene/TestUtils.java | 50 + 21 files changed, 2147 insertions(+), 122 deletions(-) create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java rename java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/{TestCuVS.java => TestCuVSRandomizedVectorSearch.java} (73%) create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestUtils.java diff --git a/java/cuvs-lucene/ci/build_java.sh b/java/cuvs-lucene/ci/build_java.sh index 1338da897c..43536e23ae 100755 --- a/java/cuvs-lucene/ci/build_java.sh +++ b/java/cuvs-lucene/ci/build_java.sh @@ -32,6 +32,27 @@ set -u rapids-print-env +# Locates the libcuvs.so file path and appends it to LD_LIBRARY_PATH +rapids-logger "Find libcuvs so file and prepend paths to LD_LIBRARY_PATH" + +CONDA_PKG_CACHE_DIR="/opt/conda/pkgs" # comes from `conda info`. Dont know if this ever changes. +if [ -d "$CONDA_PKG_CACHE_DIR" ]; then + echo "==> Directory '$CONDA_PKG_CACHE_DIR' exists." + LIBCUVS_SO_FILE="libcuvs.so" + LIBCUVS_PATH=$(find $CONDA_PKG_CACHE_DIR -name $LIBCUVS_SO_FILE) + if [ -z "$LIBCUVS_PATH" ]; then + echo "==> Could not find the so file. Not updating LD_LIBRARY_PATH" + exit 1 + else + LIBCUVS_DIR=$(dirname "$LIBCUVS_PATH") + export LD_LIBRARY_PATH="$LIBCUVS_DIR:$LD_LIBRARY_PATH" + echo "LD_LIBRARY_PATH is: $LD_LIBRARY_PATH" + fi +else + echo "==> Directory '$CONDA_PKG_CACHE_DIR' does not exist. Not updating LD_LIBRARY_PATH" + exit 1 +fi + rapids-logger "Run Java build" bash ./build.sh "${EXTRA_BUILD_ARGS[@]}" diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index dd3018c5b1..0b310207ce 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -2,6 +2,7 @@ # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: - conda-forge +- rapidsai-nightly dependencies: - cuda-cudart-dev - cuda-nvtx-dev @@ -11,6 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev +- libcuvs - maven - openjdk=22.* name: all_cuda-129_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index 8d067104f8..a8f0444b68 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -2,6 +2,7 @@ # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: - conda-forge +- rapidsai-nightly dependencies: - cuda-cudart-dev - cuda-nvtx-dev @@ -11,6 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev +- libcuvs - maven - openjdk=22.* name: all_cuda-129_arch-x86_64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml index 77eaa69ae4..50e93bd652 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -2,6 +2,7 @@ # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: - conda-forge +- rapidsai-nightly dependencies: - cuda-cudart-dev - cuda-nvtx-dev @@ -11,6 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev +- libcuvs - maven - openjdk=22.* name: all_cuda-130_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml index ee8a720584..c1e7ab3a9e 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -2,6 +2,7 @@ # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: - conda-forge +- rapidsai-nightly dependencies: - cuda-cudart-dev - cuda-nvtx-dev @@ -11,6 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev +- libcuvs - maven - openjdk=22.* name: all_cuda-130_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index 4c6cec0c6a..ee0613541d 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -21,6 +21,7 @@ files: - java channels: - conda-forge + - rapidsai-nightly dependencies: checks: common: @@ -66,6 +67,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev + - libcuvs java: common: - output_types: conda diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 073781929b..6400cc49e3 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -69,7 +69,7 @@ com.nvidia.cuvs cuvs-java - 25.8.0-4f53f-SNAPSHOT + 25.10.0-2c0e1-SNAPSHOT diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSIndex.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSIndex.java index 78c5dc1409..11519807bc 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSIndex.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSIndex.java @@ -15,8 +15,6 @@ */ package com.nvidia.cuvs.lucene; -import static com.nvidia.cuvs.lucene.CuVSVectorsReader.handleThrowable; - import com.nvidia.cuvs.BruteForceIndex; import com.nvidia.cuvs.CagraIndex; import com.nvidia.cuvs.HnswIndex; @@ -103,16 +101,16 @@ public void close() throws IOException { private void destroyIndices() throws IOException { try { if (cagraIndex != null) { - cagraIndex.destroyIndex(); + cagraIndex.close(); } if (bruteforceIndex != null) { - bruteforceIndex.destroyIndex(); + bruteforceIndex.close(); } if (hnswIndex != null) { - hnswIndex.destroyIndex(); + hnswIndex.close(); } } catch (Throwable t) { - handleThrowable(t); + Utils.handleThrowable(t); } } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSKnnFloatVectorQuery.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSKnnFloatVectorQuery.java index fa1e71d5e4..8caf30aea1 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSKnnFloatVectorQuery.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSKnnFloatVectorQuery.java @@ -19,6 +19,7 @@ import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.knn.KnnCollectorManager; import org.apache.lucene.util.Bits; @@ -29,8 +30,9 @@ public class CuVSKnnFloatVectorQuery extends KnnFloatVectorQuery { private final int iTopK; private final int searchWidth; - public CuVSKnnFloatVectorQuery(String field, float[] target, int k, int iTopK, int searchWidth) { - super(field, target, k); + public CuVSKnnFloatVectorQuery( + String field, float[] target, int k, Query filter, int iTopK, int searchWidth) { + super(field, target, k, filter); this.iTopK = iTopK; this.searchWidth = searchWidth; } @@ -46,7 +48,7 @@ protected TopDocs approximateSearch( PerLeafCuVSKnnCollector results = new PerLeafCuVSKnnCollector(k, iTopK, searchWidth); LeafReader reader = context.reader(); - reader.searchNearestVectors(field, this.getTargetCopy(), results, null); + reader.searchNearestVectors(field, this.getTargetCopy(), results, acceptDocs); return results.topDocs(); } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsFormat.java index f82198b1c3..1f7e4395d9 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsFormat.java @@ -86,6 +86,12 @@ public CuVSVectorsFormat( } private static CuVSResources cuVSResourcesOrNull() { + try { + System.loadLibrary( + "cudart"); // nocommit: this is here so as to pass CI, should goto cuvs-java + } catch (UnsatisfiedLinkError e) { + LOG.warning("Could not load CUDA runtime library: " + e.getMessage()); + } try { resources = CuVSResources.create(); return resources; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java index c770015de2..4118a0aac7 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java @@ -271,7 +271,7 @@ private CuVSIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { } } } catch (Throwable t) { - handleThrowable(t); + Utils.handleThrowable(t); } return new CuVSIndex(cagraIndex, bruteForceIndex, hnswIndex); } @@ -367,7 +367,7 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits try { searchResult = cagraIndex.search(query).getResults(); } catch (Throwable t) { - handleThrowable(t); + Utils.handleThrowable(t); } // List expected to have only one entry because of single query "target". assert searchResult.size() == 1; @@ -385,7 +385,7 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits try { searchResult = bruteforceIndex.search(query).getResults(); } catch (Throwable t) { - handleThrowable(t); + Utils.handleThrowable(t); } assert searchResult.size() == 1; result = searchResult.getFirst(); @@ -472,12 +472,15 @@ static void checkVersion(int versionMeta, int versionVectorData, IndexInput in) } } - static void handleThrowable(Throwable t) throws IOException { - switch (t) { - case IOException ioe -> throw ioe; - case Error error -> throw error; - case RuntimeException re -> throw re; - case null, default -> throw new RuntimeException("UNEXPECTED: exception type", t); - } + public FieldInfos getFieldInfos() { + return fieldInfos; + } + + public IntObjectHashMap getCuvsIndexes() { + return cuvsIndices; + } + + public IntObjectHashMap getFieldEntries() { + return fields; } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java index 3e22b4dab1..7861ad12b4 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java @@ -20,7 +20,6 @@ import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_META_CODEC_EXT; import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_META_CODEC_NAME; import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.VERSION_CURRENT; -import static com.nvidia.cuvs.lucene.CuVSVectorsReader.handleThrowable; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; import static org.apache.lucene.index.VectorEncoding.FLOAT32; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; @@ -41,14 +40,18 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.function.Supplier; import java.util.logging.Logger; +import java.util.stream.IntStream; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; import org.apache.lucene.index.DocsWithFieldSet; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FloatVectorValues; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.KnnVectorValues; @@ -57,6 +60,7 @@ import org.apache.lucene.index.Sorter; import org.apache.lucene.index.Sorter.DocMap; import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.internal.hppc.IntObjectHashMap; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.InfoStream; @@ -231,7 +235,7 @@ private void writeCagraIndex(OutputStream os, CuVSMatrix dataset) throws Throwab info("Cagra index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); Path tmpFile = Files.createTempFile(resources.tempDirectory(), "tmpindex", "cag"); index.serialize(os, tmpFile); - index.destroyIndex(); + index.close(); } private void writeBruteForceIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { @@ -246,7 +250,7 @@ private void writeBruteForceIndex(OutputStream os, CuVSMatrix dataset) throws Th long elapsedMillis = nanosToMillis(System.nanoTime() - startTime); info("bf index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); index.serialize(os); - index.destroyIndex(); + index.close(); } private void writeHNSWIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { @@ -261,7 +265,7 @@ private void writeHNSWIndex(OutputStream os, CuVSMatrix dataset) throws Throwabl info("HNSW index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); Path tmpFile = Files.createTempFile("tmpindex", "hnsw"); index.serializeToHNSW(os, tmpFile); - index.destroyIndex(); + index.close(); } @Override @@ -277,37 +281,36 @@ public void flush(int maxDoc, DocMap sortMap) throws IOException { } private void writeField(CuVSFieldWriter fieldData) throws IOException { - // TODO: Argh! https://github.com/rapidsai/cuvs/issues/698 + // TODO: Loading all vectors into memory is inefficient. Is there a way to stream the vectors + // from the flat writer to the CuVSMatrix? List vectors = fieldData.getVectors(); - CuVSMatrix.Builder builder = - CuVSMatrix.builder( - vectors.size(), fieldData.fieldInfo().getVectorDimension(), CuVSMatrix.DataType.FLOAT); - for (float[] vec : vectors) builder.addVector(vec); - writeFieldInternal(fieldData.fieldInfo(), builder.build()); + writeFieldInternal( + fieldData.fieldInfo(), + () -> Utils.createFloatMatrix(vectors, fieldData.fieldInfo().getVectorDimension()), + vectors.size()); } private void writeSortingField(CuVSFieldWriter fieldData, Sorter.DocMap sortMap) throws IOException { DocsWithFieldSet oldDocsWithFieldSet = fieldData.getDocsWithFieldSet(); final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; // new ord to old ord - mapOldOrdToNewOrd(oldDocsWithFieldSet, sortMap, null, new2OldOrd, null); - - float[][] oldVectors = fieldData.getVectors().toArray(float[][]::new); - CuVSMatrix.Builder builder = - CuVSMatrix.builder( - fieldData.getVectors().size(), - fieldData.fieldInfo().getVectorDimension(), - CuVSMatrix.DataType.FLOAT); - for (int i = 0; i < oldVectors.length; i++) { - float[] vec = oldVectors[new2OldOrd[i]]; - builder.addVector(vec); + // TODO: Loading all vectors into memory is inefficient. Is there a way to stream the vectors + // from the flat writer to the CuVSMatrix? + List sortedVectors = new ArrayList(); + for (int i = 0; i < fieldData.getVectors().size(); i++) { + sortedVectors.add(fieldData.getVectors().get(new2OldOrd[i])); } - writeFieldInternal(fieldData.fieldInfo(), builder.build()); + writeFieldInternal( + fieldData.fieldInfo(), + () -> Utils.createFloatMatrix(sortedVectors, fieldData.fieldInfo().getVectorDimension()), + sortedVectors.size()); } - private void writeFieldInternal(FieldInfo fieldInfo, CuVSMatrix dataset) throws IOException { - if (dataset.size() == 0) { + private void writeFieldInternal( + FieldInfo fieldInfo, Supplier datasetSupplier, int datasetSize) + throws IOException { + if (datasetSize == 0) { writeEmpty(fieldInfo); return; } @@ -317,7 +320,7 @@ private void writeFieldInternal(FieldInfo fieldInfo, CuVSMatrix dataset) throws // workaround for the minimum number of vectors for Cagra IndexType indexType = - this.indexType.cagra() && dataset.size() < MIN_CAGRA_INDEX_SIZE + this.indexType.cagra() && datasetSize < MIN_CAGRA_INDEX_SIZE ? IndexType.BRUTE_FORCE : this.indexType; @@ -326,7 +329,7 @@ private void writeFieldInternal(FieldInfo fieldInfo, CuVSMatrix dataset) throws if (indexType.cagra()) { try { var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - writeCagraIndex(cagraIndexOutputStream, dataset); + writeCagraIndex(cagraIndexOutputStream, datasetSupplier.get()); } catch (Throwable t) { handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); // workaround for cuVS issue @@ -338,16 +341,16 @@ private void writeFieldInternal(FieldInfo fieldInfo, CuVSMatrix dataset) throws bruteForceIndexOffset = cuvsIndex.getFilePointer(); if (indexType.bruteForce()) { var bruteForceIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - writeBruteForceIndex(bruteForceIndexOutputStream, dataset); + writeBruteForceIndex(bruteForceIndexOutputStream, datasetSupplier.get()); bruteForceIndexLength = cuvsIndex.getFilePointer() - bruteForceIndexOffset; } hnswIndexOffset = cuvsIndex.getFilePointer(); if (indexType.hnsw()) { var hnswIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - if (dataset.size() > MIN_CAGRA_INDEX_SIZE) { + if (datasetSize > MIN_CAGRA_INDEX_SIZE) { try { - writeHNSWIndex(hnswIndexOutputStream, dataset); + writeHNSWIndex(hnswIndexOutputStream, datasetSupplier.get()); } catch (Throwable t) { handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); } @@ -357,7 +360,7 @@ private void writeFieldInternal(FieldInfo fieldInfo, CuVSMatrix dataset) throws writeMeta( fieldInfo, - (int) dataset.size(), + (int) datasetSize, cagraIndexOffset, cagraIndexLength, bruteForceIndexOffset, @@ -365,7 +368,7 @@ private void writeFieldInternal(FieldInfo fieldInfo, CuVSMatrix dataset) throws hnswIndexOffset, hnswIndexLength); } catch (Throwable t) { - handleThrowable(t); + Utils.handleThrowable(t); } } @@ -418,46 +421,184 @@ static void handleThrowableWithIgnore(Throwable t, String msg) throws IOExceptio if (t.getMessage().contains(msg)) { return; } - handleThrowable(t); + Utils.handleThrowable(t); + } + + private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + try { + + List cagraIndexes = new ArrayList<>(); + // We need this count so that the merged segment's meta information has the vector count. + int totalVectorCount = 0; + + for (int i = 0; i < mergeState.knnVectorsReaders.length; i++) { + KnnVectorsReader knnReader = mergeState.knnVectorsReaders[i]; + // Access the CAGRA index for this field from the reader + + if (knnReader != null) { + if (knnReader instanceof CuVSVectorsReader cvr) { + if (cvr != null) { + totalVectorCount += cvr.getFieldEntries().get(fieldInfo.number).count(); + CagraIndex cagraIndex = getCagraIndexFromReader(cvr, fieldInfo.name); + if (cagraIndex != null) { + cagraIndexes.add(cagraIndex); + } + } + } else { + // This should never happen + throw new RuntimeException( + "Reader is not of CuVSVectorsReader type. Instead it is: " + knnReader.getClass()); + } + } + } + assert cagraIndexes.size() > 1; + + CagraIndex mergedIndex = + CagraIndex.merge(cagraIndexes.toArray(new CagraIndex[cagraIndexes.size()])); + writeMergedCagraIndex(fieldInfo, mergedIndex, totalVectorCount); + info("Successfully merged " + cagraIndexes.size() + " CAGRA indexes using native merge API"); + + } catch (Throwable t) { + Utils.handleThrowable(t); + } } /** - * Copies the vector values into dst. Returns the actual number of vectors - * copied. + * Fallback method that rebuilds indexes from merged vectors. + * Used when native CAGRA merge() is not possible. Also used + * when non-CAGRA index types are used (for e.g. Brute Force index). */ - private static int getVectorData(FloatVectorValues floatVectorValues, CuVSMatrix.Builder builder) - throws IOException { - DocsWithFieldSet docsWithField = new DocsWithFieldSet(); - int count = 0; - KnnVectorValues.DocIndexIterator iter = floatVectorValues.iterator(); + private void vectorBasedMerge(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + if (fieldInfo.getVectorEncoding() != FLOAT32) { + throw new AssertionError("Only Float32 supported"); + } + try { + // We need to compute the size of the number of merged documents up-front so that we can + // compute the CuVSMatrix capacity. TODO: Find a way to do this without merging twice. + final int numMergedDocs = getMergedDocsCount(fieldInfo, mergeState); + + if (numMergedDocs != 0) { + writeFieldInternal( + fieldInfo, + () -> { + try { + return createMatrixFromMergedVectors( + KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues( + fieldInfo, mergeState), + numMergedDocs); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, + numMergedDocs); + } else { + writeEmpty(fieldInfo); + } + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + + private int getMergedDocsCount(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + KnnVectorValues.DocIndexIterator iter = + KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState) + .iterator(); + int numMergedDocs = 0; + for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { + numMergedDocs++; + } + return numMergedDocs; + } + + /** + * Creates CuVSMatrix from merged vectors + * */ + private CuVSMatrix createMatrixFromMergedVectors( + FloatVectorValues mergedVectorValues, int numMergedDocs) throws IOException { + List vectors = new ArrayList<>(numMergedDocs); + KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { - assert iter.index() == count; - builder.addVector(floatVectorValues.vectorValue(iter.index())); // is this correct? - docsWithField.add(docV); - count++; + int ordinal = iter.index(); + float[] vector = mergedVectorValues.vectorValue(ordinal); + vectors.add(vector.clone()); + } + return Utils.createFloatMatrix(vectors, mergedVectorValues.dimension()); + } + + /** + * Extracts the CAGRA index for a specific field from a CuVSVectorsReader. + */ + private CagraIndex getCagraIndexFromReader(CuVSVectorsReader reader, String fieldName) { + try { + IntObjectHashMap cuvsIndices = reader.getCuvsIndexes(); + FieldInfos fieldInfos = reader.getFieldInfos(); + + FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldName); + + if (fieldInfo != null) { + CuVSIndex cuvsIndex = cuvsIndices.get(fieldInfo.number); + if (cuvsIndex != null) { + return cuvsIndex.getCagraIndex(); + } + } + } catch (Exception e) { + e.printStackTrace(); + info("Failed to extract CAGRA index for field " + fieldName + ": " + e.getMessage()); + } + return null; + } + + /** + * Writes a pre-built merged CAGRA index to the output. + */ + private void writeMergedCagraIndex(FieldInfo fieldInfo, CagraIndex mergedIndex, int vectorCount) + throws IOException { + try { + long cagraIndexOffset = cuvsIndex.getFilePointer(); + var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); + + // Serialize the merged index + Path tmpFile = Files.createTempFile(resources.tempDirectory(), "mergedindex", "cag"); + mergedIndex.serialize(cagraIndexOutputStream, tmpFile); + long cagraIndexLength = cuvsIndex.getFilePointer() - cagraIndexOffset; + + // Write metadata (assuming no brute force or HNSW indexes for merged result) + writeMeta(fieldInfo, vectorCount, cagraIndexOffset, cagraIndexLength, 0L, 0L, 0L, 0L); + + // Clean up the merged index + mergedIndex.close(); + } catch (Throwable t) { + Utils.handleThrowable(t); } - return docsWithField.cardinality(); } @Override public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { flatVectorsWriter.mergeOneField(fieldInfo, mergeState); - try { - final FloatVectorValues mergedVectorValues = - switch (fieldInfo.getVectorEncoding()) { - case BYTE -> throw new AssertionError("bytes not supported"); - case FLOAT32 -> - KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); - }; - - // Also will be replaced with the cuVS merge api - CuVSMatrix.Builder builder = - CuVSMatrix.builder( - mergedVectorValues.size(), mergedVectorValues.dimension(), CuVSMatrix.DataType.FLOAT); - getVectorData(mergedVectorValues, builder); - writeFieldInternal(fieldInfo, builder.build()); - } catch (Throwable t) { - handleThrowable(t); + + if (indexType.cagra() && !indexType.bruteForce()) { + // Since CAGRA merge does not support merging of indexes with purging of deletes, + // we fallback to vector-based re-indexing. Issue: + // https://github.com/rapidsai/cuvs/issues/1253 + boolean hasDeletions = + IntStream.range(0, mergeState.liveDocs.length) + .anyMatch( + i -> + mergeState.liveDocs[i] == null + || IntStream.range(0, mergeState.maxDocs[i]) + .anyMatch(j -> !mergeState.liveDocs[i].get(j))); + + if (mergeState.knnVectorsReaders.length > 1 && !hasDeletions) { + mergeCagraIndexes(fieldInfo, mergeState); + } else { + // CAGRA's merge API does not handle the trivial case of merging 1 index. + vectorBasedMerge(fieldInfo, mergeState); + } + + } else { + // If there is a Brute Force index then re-index using the vectors even if there is a CAGRA + // index. + vectorBasedMerge(fieldInfo, mergeState); } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index 5c3f7d146c..05acbeef53 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -68,8 +68,25 @@ public CagraIndex mergeCagraIndexes(CagraIndex[] arg0) throws Throwable { } @Override - public Builder newMatrixBuilder(int size, int dimensions, DataType dataType) { - return delegate.newMatrixBuilder(size, dimensions, dataType); + public com.nvidia.cuvs.GPUInfoProvider gpuInfoProvider() { + return delegate.gpuInfoProvider(); + } + + @Override + public Builder newHostMatrixBuilder(long rows, long cols, DataType dataType) { + return delegate.newHostMatrixBuilder(rows, cols, dataType); + } + + @Override + public Builder newDeviceMatrixBuilder( + CuVSResources resources, long rows, long cols, DataType dataType) { + return delegate.newDeviceMatrixBuilder(resources, rows, cols, dataType); + } + + @Override + public Builder newDeviceMatrixBuilder( + CuVSResources resources, long rows, long cols, int maxRows, int maxCols, DataType dataType) { + return delegate.newDeviceMatrixBuilder(resources, rows, cols, maxRows, maxCols, dataType); } @Override diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java new file mode 100644 index 0000000000..8d1d4bd015 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.cuvs.lucene; + +import com.nvidia.cuvs.CuVSMatrix; +import java.io.IOException; +import java.util.List; + +public class Utils { + + static void handleThrowable(Throwable t) throws IOException { + switch (t) { + case IOException ioe -> throw ioe; + case Error error -> throw error; + case RuntimeException re -> throw re; + case null, default -> throw new RuntimeException("UNEXPECTED: exception type", t); + } + } + + /** + * A method to build a {@link CuVSMatrix} from a list of float vectors. + * + * Note: This could be a memory-intensive operation and should therefore be avoided. + * Consider using this {@link CuVSMatrix.Builder} instead for copying the vectors without loading them in heap. + * + * @param data The float vectors + * @param dimensions The number float elements in each vector + * @return an instance of {@link CuVSMatrix} + */ + static CuVSMatrix createFloatMatrix(List data, int dimensions) { + // Convert List to float[][] for the ofArray method + float[][] vectors = data.toArray(new float[0][]); + return CuVSMatrix.ofArray(vectors); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java new file mode 100644 index 0000000000..ef17136e8e --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java @@ -0,0 +1,342 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.TestUtils.generateRandomVector; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.analysis.MockTokenizer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "prints info from within cuvs") +public class TestCuVSDeletedDocuments extends LuceneTestCase { + + protected static Logger log = Logger.getLogger(TestCuVSDeletedDocuments.class.getName()); + + static final Codec codec = TestUtil.alwaysKnnVectorsFormat(new CuVSVectorsFormat()); + private static Random random; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeTrue("cuvs not supported", CuVSVectorsFormat.supported()); + random = random(); + } + + @Test + public void testVectorSearchWithDeletedDocuments() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200, 1000); // 200-1200 documents + int dimensions = random.nextInt(64, 256); // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float deletionProbability = random.nextFloat() * 0.4f + 0.1f; // 10-50% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + Set deletedDocs = new HashSet<>(); + + // Create index with all documents having vectors + try (RandomIndexWriter writer = createWriter(directory)) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + } + + // Delete documents randomly based on probability + for (int i = 0; i < datasetSize; i++) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + deletedDocs.add(i); + } + } + writer.commit(); + } + + // Search and verify deleted documents are not returned + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + // Use a random vector for query + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify we got results + assertTrue("Should have search results", hits.length > 0); + + // Verify no deleted documents in results + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertFalse( + "Deleted document " + id + " should not appear in results", deletedDocs.contains(id)); + log.info("Found non-deleted document: " + id + ", Score: " + hit.score); + } + + // Verify deleted documents are truly deleted + for (int deletedId : deletedDocs) { + TopDocs result = + searcher.search(new TermQuery(new Term("id", String.valueOf(deletedId))), 1); + assertEquals( + "Deleted document " + deletedId + " should not be found", + 0, + result.totalHits.value()); + } + } + } + } + + @Test + public void testVectorSearchWithMixedDeletedAndMissingVectors() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200) + 50; // 50-250 documents + int dimensions = random.nextInt(256) + 64; // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float vectorProbability = random.nextFloat() * 0.5f + 0.3f; // 30-80% have vectors + float deletionProbability = random.nextFloat() * 0.3f + 0.1f; // 10-40% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + Set docsWithoutVectors = new HashSet<>(); + Set deletedDocs = new HashSet<>(); + + // Create index with mixed documents + try (RandomIndexWriter writer = createWriter(directory)) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + // Randomly assign categories + String category = random.nextBoolean() ? "A" : "B"; + doc.add(new StringField("category", category, Field.Store.YES)); + + // Randomly decide whether to add vectors + if (random.nextFloat() < vectorProbability) { + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + } else { + docsWithoutVectors.add(i); + } + writer.addDocument(doc); + } + + // Delete documents randomly + for (int i = 0; i < datasetSize; i++) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + deletedDocs.add(i); + } + } + writer.commit(); + } + + // Test vector search behavior + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify results + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertFalse("Deleted document should not appear", deletedDocs.contains(id)); + assertFalse("Document without vector should not appear", docsWithoutVectors.contains(id)); + log.info("Found document with vector: " + id + ", Score: " + hit.score); + } + + // Test filtered search with deletions + Query filter = new TermQuery(new Term("category", "A")); + Query filteredQuery = + new CuVSKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); + ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; + + for (ScoreDoc hit : filteredHits) { + Document doc = reader.storedFields().document(hit.doc); + String category = doc.get("category"); + assertEquals("Should only match category A", "A", category); + int id = Integer.parseInt(doc.get("id")); + assertFalse( + "Deleted document should not appear in filtered results", deletedDocs.contains(id)); + } + } + } + } + + @Test + public void testVectorSearchAfterAllDocumentsDeleted() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(20) + 5; // 5-25 documents for this test + int dimensions = random.nextInt(128) + 32; // 32-160 dimensions + int topK = Math.min(random.nextInt(10) + 5, datasetSize); // 5-15 results + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + + // Create and delete all documents + try (IndexWriter writer = new IndexWriter(directory, createWriterConfig())) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + } + writer.commit(); + + // Delete all documents + for (int i = 0; i < datasetSize; i++) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + } + writer.commit(); + writer.forceMerge(1); // Force merge to apply deletions + } + + // Verify search returns no results + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + TopDocs results = searcher.search(query, topK); + + assertEquals( + "Should return no results when all documents are deleted", + 0, + results.totalHits.value()); + } + } + } + + @Test + public void testVectorSearchWithPartialDeletionAndReindexing() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200) + 50; // 50-250 documents + int dimensions = random.nextInt(256) + 64; // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float deletionProbability = random.nextFloat() * 0.3f + 0.1f; // 10-40% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + List activeDocIds = new ArrayList<>(); + + // Initial indexing + try (IndexWriter writer = new IndexWriter(directory, createWriterConfig())) { + int initialDocs = datasetSize / 2 + random.nextInt(datasetSize / 4); // 50-75% of dataset + for (int i = 0; i < initialDocs; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + activeDocIds.add(i); + } + + // Delete some documents randomly + List candidatesForDeletion = new ArrayList<>(activeDocIds); + for (int docId : candidatesForDeletion) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(docId))); + activeDocIds.remove(Integer.valueOf(docId)); + } + } + + // Add new documents with higher IDs + for (int i = initialDocs; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + activeDocIds.add(i); + } + writer.commit(); + } + + // Verify search behavior after deletions and additions + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + Set resultIds = new HashSet<>(); + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + resultIds.add(id); + assertTrue("Result should be from active documents", activeDocIds.contains(id)); + } + + log.info( + "Search returned " + + hits.length + + " results from " + + activeDocIds.size() + + " active documents"); + } + } + } + + private RandomIndexWriter createWriter(Directory directory) throws IOException { + return new RandomIndexWriter( + random(), + directory, + newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy())); + } + + private IndexWriterConfig createWriterConfig() { + return newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy()); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java new file mode 100644 index 0000000000..e27568b5e3 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java @@ -0,0 +1,197 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.analysis.MockTokenizer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.English; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "prints info from within cuvs") +public class TestCuVSGaps extends LuceneTestCase { + + protected static Logger log = Logger.getLogger(TestCuVSGaps.class.getName()); + + static final Codec codec = TestUtil.alwaysKnnVectorsFormat(new CuVSVectorsFormat()); + static IndexSearcher searcher; + static IndexReader reader; + static Directory directory; + static Random random; + + static int DATASET_SIZE_LIMIT = 1000; + static int DIMENSIONS_LIMIT = 2048; + static int NUM_QUERIES_LIMIT = 10; + static int TOP_K_LIMIT = 64; + + static int datasetSize; + static int dimension; + static float[][] dataset; + + @BeforeClass + public static void beforeClass() throws Exception { + assertTrue("cuvs not supported", CuVSVectorsFormat.supported()); + directory = newDirectory(); + random = random(); + + RandomIndexWriter writer = + new RandomIndexWriter( + random(), + directory, + newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy())); + + log.info("Merge Policy: " + writer.w.getConfig().getMergePolicy()); + + datasetSize = random.nextInt(100, DATASET_SIZE_LIMIT); + dimension = random.nextInt(8, DIMENSIONS_LIMIT); + dataset = generateDataset(random, datasetSize, dimension); + + // Create documents where only even-numbered documents have vectors + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(newTextField("field", English.intToEnglish(i), Field.Store.YES)); + + // Only add vectors to even-numbered documents + if (i % 2 == 0) { + doc.add(new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + } + + writer.addDocument(doc); + } + + reader = writer.getReader(); + searcher = newSearcher(reader); + writer.close(); + } + + @AfterClass + public static void afterClass() throws Exception { + if (reader != null) reader.close(); + if (directory != null) directory.close(); + searcher = null; + reader = null; + directory = null; + log.info("Test finished"); + } + + @Test + public void testVectorSearchWithAlternatingDocuments() throws IOException { + assertTrue("cuvs not supported", CuVSVectorsFormat.supported()); + + // Use the first vector (from document 0) as query + float[] queryVector = dataset[0]; + int topK = random.nextInt(5, TOP_K_LIMIT); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify we get exactly TOP_K results + assertEquals("Should return exactly " + topK + " results", topK, hits.length); + + // Verify all returned documents have vectors (even-numbered IDs) + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertEquals("All results should be even-numbered (have vectors)", 0, id % 2); + log.info("Document ID: " + id + ", Score: " + hit.score); + } + + // Verify the results match expected top-k based on Euclidean distance + List expectedIds = calculateExpectedTopK(queryVector, topK, dataset); + for (int i = 0; i < hits.length; i++) { + String docId = reader.storedFields().document(hits[i].doc).get("id"); + int id = Integer.parseInt(docId); + assertTrue("Result " + id + " should be in expected top-k results", expectedIds.contains(id)); + } + + log.info("Alternating document test passed with " + hits.length + " results"); + } + + @Test + public void testVectorSearchWithFilterAndAlternatingDocuments() throws IOException { + assumeTrue("cuvs not supported", CuVSVectorsFormat.supported()); + + // Use the first vector (from document 0) as query + float[] queryVector = dataset[0]; + int topK = random.nextInt(5, TOP_K_LIMIT); + + // Create a filter that only matches documents with ID less than 10 + // This should further restrict our results to even numbers 0, 2, 4, 6, 8 + Query filter = new TermQuery(new Term("id", "8")); // Only match document 8 + + Query filteredQuery = new CuVSKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); + ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; + + // Should only get document 8 (the only one that matches the filter and has a vector) + assertEquals("Should return exactly 1 result", 1, filteredHits.length); + + String docId = reader.storedFields().document(filteredHits[0].doc).get("id"); + assertEquals("Should only return document 8", "8", docId); + + log.info("Filtered alternating document test passed with " + filteredHits.length + " results"); + } + + public static List calculateExpectedTopK(float[] query, int topK, float[][] dataset) { + Map distances = new TreeMap<>(); + + // Calculate distances only for documents that have vectors (even-numbered) + for (int i = 0; i < dataset.length; i += 2) { + double distance = 0; + for (int j = 0; j < dataset[0].length; j++) { + distance += (query[j] - dataset[i][j]) * (query[j] - dataset[i][j]); + } + distances.put(i, distance); + } + + // Sort by distance and return top-k + return distances.entrySet().stream() + .sorted(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .limit(topK) + .toList(); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVS.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java similarity index 73% rename from java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVS.java rename to java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java index 124c2f9354..8802373466 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVS.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java @@ -15,6 +15,9 @@ */ package com.nvidia.cuvs.lucene; +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.TestUtils.generateQueries; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -29,11 +32,13 @@ import org.apache.lucene.document.KnnFloatVectorField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.analysis.MockTokenizer; @@ -47,9 +52,9 @@ import org.junit.Test; @SuppressSysoutChecks(bugUrl = "prints info from within cuvs") -public class TestCuVS extends LuceneTestCase { +public class TestCuVSRandomizedVectorSearch extends LuceneTestCase { - protected static Logger log = Logger.getLogger(TestCuVS.class.getName()); + protected static Logger log = Logger.getLogger(TestCuVSRandomizedVectorSearch.class.getName()); static final Codec codec = TestUtil.alwaysKnnVectorsFormat(new CuVSVectorsFormat()); static IndexSearcher searcher; @@ -60,12 +65,11 @@ public class TestCuVS extends LuceneTestCase { static int DIMENSIONS_LIMIT = 2048; static int NUM_QUERIES_LIMIT = 10; static int TOP_K_LIMIT = 64; // TODO This fails beyond 64 - - public static float[][] dataset; + static float[][] dataset; @BeforeClass public static void beforeClass() throws Exception { - assumeTrue("cuvs not supported", CuVSVectorsFormat.supported()); + assertTrue("cuvs not supported", CuVSVectorsFormat.supported()); directory = newDirectory(); RandomIndexWriter writer = @@ -88,7 +92,8 @@ public static void beforeClass() throws Exception { doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); doc.add(newTextField("field", English.intToEnglish(i), Field.Store.YES)); boolean skipVector = - random.nextInt(10) < 0; // disable testing with holes for now, there's some bug. + random.nextInt(10) + < 4; // some documents won't have vectors to test deleted/missing vectors if (!skipVector || datasetSize < 100) { // about 10th of the documents shouldn't have a single vector doc.add(new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); @@ -146,28 +151,6 @@ public void testVectorSearch() throws IOException { } } - private static float[][] generateQueries(Random random, int dimensions, int numQueries) { - // Generate random query vectors - float[][] queries = new float[numQueries][dimensions]; - for (int i = 0; i < numQueries; i++) { - for (int j = 0; j < dimensions; j++) { - queries[i][j] = random.nextFloat() * 100; - } - } - return queries; - } - - private static float[][] generateDataset(Random random, int datasetSize, int dimensions) { - // Generate a random dataset - float[][] dataset = new float[datasetSize][dimensions]; - for (int i = 0; i < datasetSize; i++) { - for (int j = 0; j < dimensions; j++) { - dataset[i][j] = random.nextFloat() * 100; - } - } - return dataset; - } - private static List> generateExpectedResults( int topK, float[][] dataset, float[][] queries) { List> neighborsResult = new ArrayList<>(); @@ -192,15 +175,50 @@ private static List> generateExpectedResults( .sorted(Map.Entry.comparingByValue()) .map(Map.Entry::getKey) .toList(); - neighborsResult.add( - neighbors.subList( - 0, - Math.min( - topK * 3, - dataset.length))); // generate double the topK results in the expected array + neighborsResult.add(neighbors.subList(0, Math.min(topK * 3, dataset.length))); } log.info("Expected results generated successfully."); return neighborsResult; } + + @Test + public void testVectorSearchWithFilter() throws IOException { + assertTrue("cuvs not supported", CuVSVectorsFormat.supported()); + + Random random = random(); + int topK = Math.min(random.nextInt(TOP_K_LIMIT) + 1, dataset.length); + + if (dataset.length < topK) topK = dataset.length; + + // Find a document that has a vector by doing a search first + Query unfiltered = new KnnFloatVectorQuery("vector", dataset[0], 1); + ScoreDoc[] unfilteredHits = searcher.search(unfiltered, 1).scoreDocs; + + // Skip test if no vectors found at all + assumeTrue( + "Need at least one document with vector for filtering test", unfilteredHits.length > 0); + + String targetDocId = reader.storedFields().document(unfilteredHits[0].doc).get("id"); + float[] queryVector = dataset[0]; + + // Create a filter that matches only the document we know has a vector + Query filter = new TermQuery(new Term("id", targetDocId)); + + // Test the new constructor with filter + Query filteredQuery = new CuVSKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); + + ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; + + // Ensure we got some results + assertTrue("Should have at least one result", filteredHits.length > 0); + + // Verify that all results match the filter + for (ScoreDoc hit : filteredHits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + assertEquals("All results should match the filter", targetDocId, docId); + } + + log.info("Prefiltering test passed with " + filteredHits.length + " results"); + } } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java index d19180eb43..69aeb29082 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java @@ -31,15 +31,17 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.store.Directory; import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; import org.apache.lucene.tests.util.TestUtil; import org.junit.BeforeClass; import org.junit.Ignore; +@SuppressSysoutChecks(bugUrl = "") public class TestCuVSVectorsFormat extends BaseKnnVectorsFormatTestCase { @BeforeClass public static void beforeClass() { - assumeTrue("cuvs is not supported", CuVSVectorsFormat.supported()); + assertTrue("cuvs is not supported", CuVSVectorsFormat.supported()); } @Override diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java index d5f46d7370..5f6ff3aa57 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java @@ -21,7 +21,9 @@ import java.util.Random; import org.apache.lucene.store.IOContext; import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +@SuppressSysoutChecks(bugUrl = "") public class TestIndexOutputOutputStream extends LuceneTestCase { public void testBasic() throws IOException { @@ -33,8 +35,8 @@ public void testBasic() throws IOException { out.close(); } - try (var indexIn = dir.openInput("test", IOContext.DEFAULT)) { - var in = new IndexInputInputStream(indexIn); + try (var indexIn = dir.openInput("test", IOContext.DEFAULT); + var in = new IndexInputInputStream(indexIn)) { byte[] ba = new byte[6]; assertEquals(6, in.read(ba)); assertArrayEquals(new byte[] {0x56, 0x10, 0x11, 0x12, 0x13, 0x14}, ba); @@ -76,9 +78,8 @@ public void testWithRandom() throws IOException { out.close(); } - try (var indexIn = dir.openInput("test", IOContext.DEFAULT)) { - // TODO: close this stream properly in a subsequent PR. - var in = new IndexInputInputStream(indexIn); + try (var indexIn = dir.openInput("test", IOContext.DEFAULT); + var in = new IndexInputInputStream(indexIn); ) { int i = 0; while (i < data.length) { if (random.nextBoolean()) { diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java new file mode 100644 index 0000000000..a424492800 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java @@ -0,0 +1,1169 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.cuvs.lucene; + +import static org.apache.lucene.tests.util.TestUtil.alwaysKnnVectorsFormat; + +import com.nvidia.cuvs.lucene.CuVSVectorsWriter.IndexType; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.logging.Logger; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.TieredMergePolicy; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.util.BytesRef; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Comprehensive tests for merge functionality with CuVS indexes. + * Tests merge operations across different index types including brute force, + * CAGRA, and combined index configurations to ensure proper vector handling + * and search functionality after segment merging. + */ +@SuppressSysoutChecks(bugUrl = "CuVS native library produces verbose logging output") +public class TestMerge extends LuceneTestCase { + + private static final Logger log = Logger.getLogger(TestMerge.class.getName()); + + private static final int MIN_VECTOR_DIMENSION = 64; + private static final int MAX_VECTOR_DIMENSION = 256; + private static final int TOP_K_LIMIT = 64; + + @BeforeClass + public static void beforeClass() { + assertTrue("cuVS is not supported", CuVSVectorsFormat.supported()); + } + + private Directory directory; + private int vectorDimension; + + @Before + public void setUp() throws Exception { + super.setUp(); + directory = newDirectory(); + + // Randomize vector dimension for each test + vectorDimension = + MIN_VECTOR_DIMENSION + random().nextInt(MAX_VECTOR_DIMENSION - MIN_VECTOR_DIMENSION + 1); + // Ensure dimension is multiple of 4 for better performance + vectorDimension = (vectorDimension / 4) * 4; + + log.info("Using randomized vector dimension: " + vectorDimension); + } + + @After + public void tearDown() throws Exception { + if (directory != null) { + directory.close(); + } + super.tearDown(); + } + + /** + * Test merging many documents across multiple segments + **/ + @Test + public void testMergeManyDocumentsMultipleSegments() throws IOException { + log.info("Starting testMergeManyDocumentsMultipleSegments"); + + // Randomize configuration parameters + int maxBufferedDocs = 5 + random().nextInt(16); // 5-20 docs per buffer + int totalBatches = 8 + random().nextInt(8); // 8-15 batches + int docsPerBatch = 15 + random().nextInt(11); // 15-25 docs per batch + int totalDocuments = totalBatches * docsPerBatch; + + // Randomize vector presence probability (60-85%) + double vectorProbability = 0.6 + (random().nextDouble() * 0.25); + + log.info( + "Randomized parameters: maxBufferedDocs=" + + maxBufferedDocs + + ", totalBatches=" + + totalBatches + + ", docsPerBatch=" + + docsPerBatch + + ", totalDocuments=" + + totalDocuments + + ", vectorProbability=" + + vectorProbability); + + IndexWriterConfig config = + new IndexWriterConfig() + .setCodec(alwaysKnnVectorsFormat(new CuVSVectorsFormat())) + .setMaxBufferedDocs(maxBufferedDocs) // Randomized buffer size + .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + + List expectedVectors = new ArrayList<>(); + List expectedDocIds = new ArrayList<>(); + int documentsWithVectors = 0; + + try (IndexWriter writer = new IndexWriter(directory, config)) { + // Add documents in multiple batches to create many segments + for (int batch = 0; batch < totalBatches; batch++) { + for (int i = 0; i < docsPerBatch; i++) { + int docId = batch * docsPerBatch + i; + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(docId), Field.Store.YES)); + doc.add(new NumericDocValuesField("batch", batch)); + + // Randomly decide if document has vector + if (random().nextDouble() < vectorProbability) { + float[] vector = generateRandomVector(vectorDimension, random()); + doc.add(new KnnFloatVectorField("vector", vector, VectorSimilarityFunction.COSINE)); + expectedVectors.add(vector); + expectedDocIds.add(docId); + documentsWithVectors++; + } + + writer.addDocument(doc); + } + writer.commit(); // Create a new segment + } + + int documentsWithoutVectors = totalDocuments - documentsWithVectors; + log.info("Created " + totalDocuments + " documents in " + totalBatches + " segments"); + log.info("Documents with vectors: " + documentsWithVectors); + log.info("Documents without vectors: " + documentsWithoutVectors); + + // Force merge to trigger merge logic + writer.forceMerge(1); + log.info("Forced merge to single segment completed"); + } + + // Verify the merged index + try (DirectoryReader reader = DirectoryReader.open(directory)) { + assertEquals("Should have exactly one segment after merge", 1, reader.leaves().size()); + + LeafReader leafReader = reader.leaves().get(0).reader(); + assertEquals("Total documents should match", totalDocuments, leafReader.maxDoc()); + + // Verify vector search works correctly after merge + if (documentsWithVectors > 0) { + IndexSearcher searcher = new IndexSearcher(reader); + float[] queryVector = generateRandomVector(vectorDimension, random()); + + // Randomize search parameters + int searchK = + Math.min(5 + random().nextInt(10), Math.min(documentsWithVectors, TOP_K_LIMIT)); + + KnnFloatVectorQuery query = new KnnFloatVectorQuery("vector", queryVector, searchK); + TopDocs results = searcher.search(query, searchK); + + assertTrue("Should find some results after merge", results.scoreDocs.length > 0); + assertTrue( + "Should find reasonable number of results", + results.scoreDocs.length <= documentsWithVectors); + + log.info( + "Vector search returned " + + results.scoreDocs.length + + " results out of " + + documentsWithVectors + + " documents with vectors"); + + // Verify all returned documents have valid IDs + for (ScoreDoc scoreDoc : results.scoreDocs) { + int docId = Integer.parseInt(searcher.storedFields().document(scoreDoc.doc).get("id")); + assertTrue("Document ID should be valid", docId >= 0 && docId < totalDocuments); + } + } else { + log.info("No documents with vectors - skipping vector search verification"); + } + + log.info("Merge verification completed successfully"); + } + } + + /** + * Test merging with index sorting enabled using text-based sorting and SortingMergePolicy + **/ + @Test + public void testMergeWithIndexSorting() throws IOException { + log.info("Starting testMergeWithIndexSorting with text-based sorting"); + + // Randomize sort field type + SortField.Type sortType = random().nextBoolean() ? SortField.Type.STRING : SortField.Type.LONG; + String sortFieldName = sortType == SortField.Type.STRING ? "text_sort_key" : "numeric_sort_key"; + + // Configure index sorting by a randomized field + Sort indexSort = new Sort(new SortField(sortFieldName, sortType)); + + // Randomize merge policy parameters + TieredMergePolicy mergePolicy = new TieredMergePolicy(); + mergePolicy.setMaxMergedSegmentMB(128 + random().nextInt(257)); // 128-384 MB + mergePolicy.setSegmentsPerTier(3 + random().nextInt(4)); // 3-6 segments per tier + + // Randomize writer configuration parameters + int maxBufferedDocs = 10 + random().nextInt(16); // 10-25 docs per buffer + int totalDocuments = 80 + random().nextInt(81); // 80-160 documents + int segmentSize = 15 + random().nextInt(11); // 15-25 docs per segment + double vectorProbability = 0.65 + (random().nextDouble() * 0.25); // 65-90% have vectors + + log.info( + "Randomized sorting parameters: sortType=" + sortType + ", sortFieldName=" + sortFieldName); + log.info( + "Randomized config: maxBufferedDocs=" + + maxBufferedDocs + + ", totalDocuments=" + + totalDocuments + + ", segmentSize=" + + segmentSize + + ", vectorProbability=" + + vectorProbability); + + IndexWriterConfig config = + new IndexWriterConfig() + .setCodec(alwaysKnnVectorsFormat(new CuVSVectorsFormat())) + .setIndexSort(indexSort) // This automatically enables sorting during merges + .setMergePolicy(mergePolicy) + .setMaxBufferedDocs(maxBufferedDocs) + .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + + // List documents = new ArrayList<>(); + + try (IndexWriter writer = new IndexWriter(directory, config)) { + // Create documents with randomized sort keys + for (int i = 0; i < totalDocuments; i++) { + float[] vector = null; + + // Randomly decide if document has vector + if (random().nextDouble() < vectorProbability) { + vector = generateRandomVector(vectorDimension, random()); + } + + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(new StringField("original_order", String.valueOf(i), Field.Store.YES)); + + // Add sort field based on randomized type + if (sortType == SortField.Type.STRING) { + // Randomize text sort key length (4-12 characters) + int keyLength = 4 + random().nextInt(9); + String textSortKey = generateRandomText(random(), keyLength); + doc.add(new SortedDocValuesField(sortFieldName, new BytesRef(textSortKey))); + doc.add(new StringField(sortFieldName + "_stored", textSortKey, Field.Store.YES)); + } else { + // Use numeric sort key with wider range + long numericSortKey = random().nextLong() % 100000; // Can be negative for more variety + doc.add(new NumericDocValuesField(sortFieldName, numericSortKey)); + doc.add( + new StringField( + sortFieldName + "_stored", String.valueOf(numericSortKey), Field.Store.YES)); + } + + if (vector != null) { + doc.add(new KnnFloatVectorField("vector", vector, VectorSimilarityFunction.COSINE)); + } + + writer.addDocument(doc); + + // Commit based on randomized segment size + if ((i + 1) % segmentSize == 0) { + writer.commit(); + log.info( + "Committed segment " + + ((i + 1) / segmentSize) + + " with " + + (i + 1) + + " total documents"); + } + } + + log.info("Created " + totalDocuments + " documents with text-based index sorting"); + + // Force merge with sorting - this will use the sorting merge policy + writer.forceMerge(1); + log.info("Forced merge with text-based sorting completed"); + } + + // Verify the merged and sorted index + try (DirectoryReader reader = DirectoryReader.open(directory)) { + assertEquals("Should have exactly one segment after merge", 1, reader.leaves().size()); + + LeafReader leafReader = reader.leaves().get(0).reader(); + assertEquals("Total documents should match", totalDocuments, leafReader.maxDoc()); + + // Verify documents are sorted correctly by the randomized sort field + log.info( + "Verifying document sorting order using sortType: " + + sortType + + ", field: " + + sortFieldName); + + if (sortType == SortField.Type.STRING) { + // Verify string-based sorting + String previousSortKey = ""; + SortedDocValues sortedValues = leafReader.getSortedDocValues(sortFieldName); + + for (int docId = 0; docId < leafReader.maxDoc(); docId++) { + String currentSortKey = ""; + if (sortedValues != null && sortedValues.advanceExact(docId)) { + currentSortKey = sortedValues.lookupOrd(sortedValues.ordValue()).utf8ToString(); + } + + assertTrue( + "Documents should be sorted by " + + sortFieldName + + ": '" + + previousSortKey + + "' should be <= '" + + currentSortKey + + "'", + previousSortKey.compareTo(currentSortKey) <= 0); + previousSortKey = currentSortKey; + + // Log first 10 documents to verify sorting + if (docId < 10) { + IndexSearcher searcher = new IndexSearcher(reader); + String originalOrder = searcher.storedFields().document(docId).get("original_order"); + log.info( + "DocId: " + + docId + + ", OriginalOrder: " + + originalOrder + + ", SortKey: '" + + currentSortKey + + "'"); + } + } + } else { + // Verify numeric-based sorting + long previousSortKey = Long.MIN_VALUE; + var numericValues = leafReader.getNumericDocValues(sortFieldName); + + for (int docId = 0; docId < leafReader.maxDoc(); docId++) { + long currentSortKey = Long.MIN_VALUE; + if (numericValues != null && numericValues.advanceExact(docId)) { + currentSortKey = numericValues.longValue(); + } + + assertTrue( + "Documents should be sorted by " + + sortFieldName + + ": " + + previousSortKey + + " should be <= " + + currentSortKey, + previousSortKey <= currentSortKey); + previousSortKey = currentSortKey; + + // Log first 10 documents to verify sorting + if (docId < 10) { + IndexSearcher searcher = new IndexSearcher(reader); + String originalOrder = searcher.storedFields().document(docId).get("original_order"); + log.info( + "DocId: " + + docId + + ", OriginalOrder: " + + originalOrder + + ", SortKey: " + + currentSortKey); + } + } + } + + // Count total vectors by checking if vector field exists and has values + var vectorValues = leafReader.getFloatVectorValues("vector"); + int documentsWithVectors = vectorValues != null ? vectorValues.size() : 0; + + log.info("Found " + documentsWithVectors + " documents with vectors after sorted merge"); + + // Test vector search on sorted index + if (documentsWithVectors > 0) { + IndexSearcher searcher = new IndexSearcher(reader); + float[] queryVector = generateRandomVector(vectorDimension, random()); + + KnnFloatVectorQuery query = + new KnnFloatVectorQuery("vector", queryVector, Math.min(10, documentsWithVectors)); + TopDocs results = searcher.search(query, 10); + + assertTrue("Should find results in sorted index", results.scoreDocs.length > 0); + log.info("Vector search on sorted index returned " + results.scoreDocs.length + " results"); + + // Verify that returned documents maintain sort order if we check their sort keys + log.info("Verifying vector search results maintain sorting consistency..."); + for (int i = 0; i < Math.min(3, results.scoreDocs.length); i++) { + ScoreDoc scoreDoc = results.scoreDocs[i]; + String originalOrder = + searcher.storedFields().document(scoreDoc.doc).get("original_order"); + String sortKey = + searcher.storedFields().document(scoreDoc.doc).get(sortFieldName + "_stored"); + log.info( + "Result " + + i + + ": DocId=" + + scoreDoc.doc + + ", OriginalOrder=" + + originalOrder + + ", SortKey='" + + sortKey + + "', Score=" + + scoreDoc.score); + } + } + + log.info("Text-based index sorting verification completed successfully"); + } + } + + /** + * Test merging segments with various patterns of missing vectors + **/ + @Test + public void testMergeWithMissingVectors() throws IOException { + log.info("Starting testMergeWithMissingVectors"); + + // Randomize configuration + int maxBufferedDocs = 10 + random().nextInt(11); // 10-20 docs per buffer + int numSegments = 3 + random().nextInt(3); // 3-5 segments + + IndexWriterConfig config = + new IndexWriterConfig() + .setCodec(alwaysKnnVectorsFormat(new CuVSVectorsFormat())) + .setMaxBufferedDocs(maxBufferedDocs) + .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + + log.info( + "Randomized parameters: maxBufferedDocs=" + + maxBufferedDocs + + ", numSegments=" + + numSegments); + + int totalExpectedVectors = 0; + int totalDocuments = 0; + + try (IndexWriter writer = new IndexWriter(directory, config)) { + for (int seg = 0; seg < numSegments; seg++) { + // Randomize segment characteristics + int docsInSegment = 15 + random().nextInt(16); // 15-30 docs per segment + double vectorProbability = random().nextDouble(); // 0-100% vector probability + String segmentType = "seg_" + seg + "_prob_" + String.format("%.2f", vectorProbability); + + int segmentVectorCount = 0; + + for (int i = 0; i < docsInSegment; i++) { + Document doc = new Document(); + doc.add(new StringField("id", "seg" + seg + "_" + i, Field.Store.YES)); + doc.add(new StringField("segment", segmentType, Field.Store.YES)); + doc.add(new NumericDocValuesField("segment_num", seg)); + doc.add(new NumericDocValuesField("doc_in_segment", i)); + + // Randomly add vector based on segment's probability + if (random().nextDouble() < vectorProbability) { + float[] vector = generateRandomVector(vectorDimension, random()); + doc.add(new KnnFloatVectorField("vector", vector, VectorSimilarityFunction.COSINE)); + segmentVectorCount++; + } + + writer.addDocument(doc); + } + + writer.commit(); + totalDocuments += docsInSegment; + totalExpectedVectors += segmentVectorCount; + + log.info( + "Created segment " + + seg + + ": " + + docsInSegment + + " documents, " + + segmentVectorCount + + " with vectors (probability: " + + String.format("%.2f", vectorProbability) + + ")"); + } + + // Force merge all segments + writer.forceMerge(1); + log.info("Forced merge of " + numSegments + " segments completed"); + } + + // Verify the merged index handles missing vectors correctly + try (DirectoryReader reader = DirectoryReader.open(directory)) { + assertEquals("Should have exactly one segment after merge", 1, reader.leaves().size()); + + LeafReader leafReader = reader.leaves().get(0).reader(); + assertEquals("Total documents should match", totalDocuments, leafReader.maxDoc()); + + // Count actual vectors in merged index + var vectorValues = leafReader.getFloatVectorValues("vector"); + int actualVectorCount = vectorValues != null ? vectorValues.size() : 0; + + log.info( + "Total documents: " + + totalDocuments + + ", Expected vectors: " + + totalExpectedVectors + + ", Actual vectors: " + + actualVectorCount); + + assertEquals("Vector count should match expected", totalExpectedVectors, actualVectorCount); + + // Test vector search if we have vectors + if (actualVectorCount > 0) { + IndexSearcher searcher = new IndexSearcher(reader); + float[] queryVector = generateRandomVector(vectorDimension, random()); + + // Randomize search parameters + int searchK = Math.min(5 + random().nextInt(10), Math.min(actualVectorCount, TOP_K_LIMIT)); + + KnnFloatVectorQuery vectorQuery = new KnnFloatVectorQuery("vector", queryVector, searchK); + TopDocs vectorResults = searcher.search(vectorQuery, searchK); + + assertTrue("Should find some vector results", vectorResults.scoreDocs.length > 0); + assertTrue( + "Should not find more vectors than exist", + vectorResults.scoreDocs.length <= actualVectorCount); + + log.info( + "Found " + + vectorResults.scoreDocs.length + + " vector results out of " + + actualVectorCount + + " available"); + } else { + log.info("No vectors in merged index - skipping vector search"); + } + + log.info("Missing vectors test completed successfully"); + } + } + + /** + * Test merge behavior with document deletions + **/ + @Test + public void testMergeWithDeletions() throws IOException { + log.info("Starting testMergeWithDeletions"); + + // Randomize configuration parameters + int maxBufferedDocs = 15 + random().nextInt(11); // 15-25 docs per buffer + int numSegments = 3 + random().nextInt(4); // 3-6 segments + int docsPerSegment = 20 + random().nextInt(21); // 20-40 docs per segment + double vectorProbability = 0.7 + (random().nextDouble() * 0.25); // 70-95% have vectors + double deletionProbability = 0.2 + (random().nextDouble() * 0.3); // 20-50% deletion rate + + log.info( + "Randomized parameters: maxBufferedDocs=" + + maxBufferedDocs + + ", numSegments=" + + numSegments + + ", docsPerSegment=" + + docsPerSegment + + ", vectorProbability=" + + vectorProbability + + ", deletionProbability=" + + deletionProbability); + + IndexWriterConfig config = + new IndexWriterConfig() + .setCodec(alwaysKnnVectorsFormat(new CuVSVectorsFormat())) + .setMaxBufferedDocs(maxBufferedDocs) + .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + + List expectedRemainingDocs = new ArrayList<>(); + List deletedDocs = new ArrayList<>(); + int totalDocuments = numSegments * docsPerSegment; + + try (IndexWriter writer = new IndexWriter(directory, config)) { + // Create multiple segments with documents + for (int seg = 0; seg < numSegments; seg++) { + for (int i = 0; i < docsPerSegment; i++) { + int docId = seg * docsPerSegment + i; + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(docId), Field.Store.YES)); + doc.add(new StringField("segment", "seg_" + seg, Field.Store.YES)); + doc.add(new NumericDocValuesField("doc_num", docId)); + doc.add(new NumericDocValuesField("segment_num", seg)); + + // Randomly add vectors + if (random().nextDouble() < vectorProbability) { + float[] vector = generateRandomVector(vectorDimension, random()); + doc.add(new KnnFloatVectorField("vector", vector, VectorSimilarityFunction.COSINE)); + } + + writer.addDocument(doc); + } + writer.commit(); + } + + log.info( + "Created " + + numSegments + + " segments with " + + docsPerSegment + + " documents each (" + + totalDocuments + + " total)"); + + // Delete documents randomly and track which ones are deleted + int deletedCount = 0; + for (int docId = 0; docId < totalDocuments; docId++) { + if (random().nextDouble() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(docId))); + deletedDocs.add(docId); + deletedCount++; + } else { + expectedRemainingDocs.add(docId); + } + } + + log.info( + "Deleted " + + deletedCount + + " documents (" + + String.format("%.1f", (100.0 * deletedCount / totalDocuments)) + + "%), remaining: " + + expectedRemainingDocs.size()); + + writer.commit(); + + // Force merge to apply deletions + writer.forceMerge(1); + log.info("Forced merge with deletions completed"); + } + + // Verify the merged index correctly handles deletions + try (DirectoryReader reader = DirectoryReader.open(directory)) { + assertEquals("Should have exactly one segment after merge", 1, reader.leaves().size()); + + LeafReader leafReader = reader.leaves().get(0).reader(); + int expectedRemaining = expectedRemainingDocs.size(); + assertEquals( + "Should have correct number of documents after deletions", + expectedRemaining, + leafReader.maxDoc()); + + // Verify that deleted documents are not present + IndexSearcher searcher = new IndexSearcher(reader); + + // Test that we can find expected remaining documents + for (int i = 0; i < Math.min(10, expectedRemainingDocs.size()); i++) { + int docId = expectedRemainingDocs.get(i); + TopDocs result = searcher.search(new TermQuery(new Term("id", String.valueOf(docId))), 1); + assertEquals("Should find remaining document " + docId, 1, (int) result.totalHits.value()); + } + + // Test that actually deleted documents are not found + int deletedDocsToCheck = Math.min(10, deletedDocs.size()); // Check up to 10 deleted docs + for (int i = 0; i < deletedDocsToCheck; i++) { + int docId = deletedDocs.get(i); + TopDocs result = searcher.search(new TermQuery(new Term("id", String.valueOf(docId))), 1); + assertEquals( + "Should not find deleted document " + docId, 0, (int) result.totalHits.value()); + } + + // Test vector search works after deletions + float[] queryVector = generateRandomVector(vectorDimension, random()); + KnnFloatVectorQuery vectorQuery = new KnnFloatVectorQuery("vector", queryVector, 10); + TopDocs vectorResults = searcher.search(vectorQuery, 10); + + assertTrue( + "Should find some vector results after deletions", vectorResults.scoreDocs.length > 0); + + log.info("Found " + vectorResults.scoreDocs.length + " vector results after deletions"); + log.info("Deletion merge verification completed successfully"); + } + } + + /** + * Test merging segments for {@link IndexType#BRUTE_FORCE} + * */ + @Test + public void testMergeBruteForceIndex() throws IOException { + log.info("Starting testMergeBruteForceIndex"); + + // Randomize configuration parameters + int maxBufferedDocs = 8 + random().nextInt(8); // 8-15 docs per buffer + int numSegments = 3 + random().nextInt(3); // 3-5 segments + int docsPerSegment = 12 + random().nextInt(9); // 12-20 docs per segment + double vectorProbability = 0.8 + (random().nextDouble() * 0.2); // 80-100% have vectors + + log.info( + "Randomized parameters: maxBufferedDocs=" + + maxBufferedDocs + + ", numSegments=" + + numSegments + + ", docsPerSegment=" + + docsPerSegment + + ", vectorProbability=" + + vectorProbability); + + // Configure with brute force index type + CuVSVectorsFormat bruteForceFormat = + new CuVSVectorsFormat( + 32, // writer threads + 128, // intermediate graph degree + 64, // graph degree + IndexType.BRUTE_FORCE); // Use brute force index + + IndexWriterConfig config = + new IndexWriterConfig() + .setCodec(alwaysKnnVectorsFormat(bruteForceFormat)) + .setMaxBufferedDocs(maxBufferedDocs) + .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + + int totalDocuments = numSegments * docsPerSegment; + int totalExpectedVectors = 0; + + try (IndexWriter writer = new IndexWriter(directory, config)) { + // Create multiple segments with brute force index + for (int seg = 0; seg < numSegments; seg++) { + int segmentVectorCount = 0; + + for (int i = 0; i < docsPerSegment; i++) { + int docId = seg * docsPerSegment + i; + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(docId), Field.Store.YES)); + doc.add(new StringField("segment", "seg_" + seg, Field.Store.YES)); + doc.add(new NumericDocValuesField("segment_num", seg)); + doc.add(new NumericDocValuesField("doc_in_segment", i)); + + // Randomly add vectors based on probability + if (random().nextDouble() < vectorProbability) { + float[] vector = generateRandomVector(vectorDimension, random()); + doc.add(new KnnFloatVectorField("vector", vector, VectorSimilarityFunction.COSINE)); + segmentVectorCount++; + } + + writer.addDocument(doc); + } + + writer.commit(); + totalExpectedVectors += segmentVectorCount; + + log.info( + "Created brute force segment " + + seg + + ": " + + docsPerSegment + + " documents, " + + segmentVectorCount + + " with vectors"); + } + + log.info( + "Created " + + numSegments + + " brute force segments with " + + totalDocuments + + " total documents and " + + totalExpectedVectors + + " vectors"); + + // Force merge all brute force segments + writer.forceMerge(1); + log.info("Forced merge of brute force segments completed"); + } + + // Verify the merged brute force index + try (DirectoryReader reader = DirectoryReader.open(directory)) { + assertEquals("Should have exactly one segment after merge", 1, reader.leaves().size()); + + LeafReader leafReader = reader.leaves().get(0).reader(); + assertEquals("Total documents should match", totalDocuments, leafReader.maxDoc()); + + // Count actual vectors in merged index + var vectorValues = leafReader.getFloatVectorValues("vector"); + int actualVectorCount = vectorValues != null ? vectorValues.size() : 0; + + log.info( + "Brute force merge results: Total documents: " + + totalDocuments + + ", Expected vectors: " + + totalExpectedVectors + + ", Actual vectors: " + + actualVectorCount); + + assertEquals("Vector count should match expected", totalExpectedVectors, actualVectorCount); + + // Test brute force vector search (exact search) + if (actualVectorCount > 0) { + IndexSearcher searcher = new IndexSearcher(reader); + float[] queryVector = generateRandomVector(vectorDimension, random()); + + // Search for reasonable number of results + int searchK = Math.min(8 + random().nextInt(8), Math.min(actualVectorCount, TOP_K_LIMIT)); + + KnnFloatVectorQuery vectorQuery = new KnnFloatVectorQuery("vector", queryVector, searchK); + TopDocs vectorResults = searcher.search(vectorQuery, searchK); + + assertTrue( + "Should find some vector results in brute force index", + vectorResults.scoreDocs.length > 0); + assertTrue( + "Should not find more vectors than exist", + vectorResults.scoreDocs.length <= actualVectorCount); + + log.info( + "Brute force search found " + + vectorResults.scoreDocs.length + + " results out of " + + actualVectorCount + + " available vectors"); + + // Verify all returned documents are valid + for (ScoreDoc scoreDoc : vectorResults.scoreDocs) { + String docId = searcher.storedFields().document(scoreDoc.doc).get("id"); + assertNotNull("Document should have valid ID", docId); + assertTrue("Score should be positive", scoreDoc.score > 0); + } + } else { + log.info("No vectors in brute force merged index - skipping vector search"); + } + + log.info("Brute force merge verification completed successfully"); + } + } + + /** + * Test merging segments for {@link IndexType#CAGRA_AND_BRUTE_FORCE} + * */ + @Test + public void testMergeCagraAndBruteForceIndex() throws IOException { + log.info("Starting testMergeCagraAndBruteForceIndex"); + + // Use moderate dataset size + int maxBufferedDocs = 15 + random().nextInt(10); // 15-24 docs per buffer + int numSegments = + 4; // Fixed 4 segments: alternating CAGRA vs small segments (brute force fallback) + int docsPerSegment = 20 + random().nextInt(11); // 20-30 docs per segment + double vectorProbability = 0.9 + (random().nextDouble() * 0.1); // 90-100% have vectors + + log.info( + "Randomized parameters: maxBufferedDocs=" + + maxBufferedDocs + + ", numSegments=" + + numSegments + + ", docsPerSegment=" + + docsPerSegment + + ", vectorProbability=" + + vectorProbability); + + // Configure with CAGRA + brute force combined index type + CuVSVectorsFormat combinedFormat = + new CuVSVectorsFormat( + 32, // writer threads + 128, // intermediate graph degree + 64, // graph degree + IndexType.CAGRA_AND_BRUTE_FORCE); // Use combined CAGRA + brute force + + IndexWriterConfig config = + new IndexWriterConfig() + .setCodec(alwaysKnnVectorsFormat(combinedFormat)) + .setMaxBufferedDocs(maxBufferedDocs) + .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + + int totalDocuments = numSegments * docsPerSegment; + int totalExpectedVectors = 0; + + try (IndexWriter writer = new IndexWriter(directory, config)) { + // Create segments that will result in mixed index types during merge + for (int seg = 0; seg < numSegments; seg++) { + int segmentVectorCount = 0; + + for (int i = 0; i < docsPerSegment; i++) { + int docId = seg * docsPerSegment + i; + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(docId), Field.Store.YES)); + doc.add(new StringField("segment", "mixed_seg_" + seg, Field.Store.YES)); + doc.add(new StringField("index_type", "cagra_and_brute_force", Field.Store.YES)); + doc.add(new NumericDocValuesField("segment_num", seg)); + doc.add(new NumericDocValuesField("doc_in_segment", i)); + + // Add vectors based on probability + if (random().nextDouble() < vectorProbability) { + float[] vector = generateRandomVector(vectorDimension, random()); + doc.add(new KnnFloatVectorField("vector", vector, VectorSimilarityFunction.COSINE)); + segmentVectorCount++; + } + + writer.addDocument(doc); + } + + writer.commit(); + totalExpectedVectors += segmentVectorCount; + + log.info( + "Created CAGRA+brute force segment " + + seg + + ": " + + docsPerSegment + + " documents, " + + segmentVectorCount + + " with vectors"); + } + + log.info( + "Created " + + numSegments + + " CAGRA+brute force segments with " + + totalDocuments + + " total documents and " + + totalExpectedVectors + + " vectors"); + + // Force merge all CAGRA+brute force segments + writer.forceMerge(1); + log.info("Forced merge of CAGRA+brute force segments completed"); + } + + // Verify the merged CAGRA+brute force index + try (DirectoryReader reader = DirectoryReader.open(directory)) { + assertEquals("Should have exactly one segment after merge", 1, reader.leaves().size()); + + LeafReader leafReader = reader.leaves().get(0).reader(); + assertEquals("Total documents should match", totalDocuments, leafReader.maxDoc()); + + // Count actual vectors in merged index + var vectorValues = leafReader.getFloatVectorValues("vector"); + int actualVectorCount = vectorValues != null ? vectorValues.size() : 0; + + log.info( + "CAGRA+brute force merge results: Total documents: " + + totalDocuments + + ", Expected vectors: " + + totalExpectedVectors + + ", Actual vectors: " + + actualVectorCount); + + assertEquals("Vector count should match expected", totalExpectedVectors, actualVectorCount); + + // Test CAGRA+brute force index vector search + if (actualVectorCount > 0) { + IndexSearcher searcher = new IndexSearcher(reader); + float[] queryVector = generateRandomVector(vectorDimension, random()); + + // Search for reasonable number of results + int searchK = Math.min(12 + random().nextInt(8), Math.min(actualVectorCount, TOP_K_LIMIT)); + + KnnFloatVectorQuery vectorQuery = new KnnFloatVectorQuery("vector", queryVector, searchK); + TopDocs vectorResults = searcher.search(vectorQuery, searchK); + + assertTrue( + "Should find some vector results in CAGRA+brute force index", + vectorResults.scoreDocs.length > 0); + assertTrue( + "Should not find more vectors than exist", + vectorResults.scoreDocs.length <= actualVectorCount); + + log.info( + "CAGRA+brute force index search found " + + vectorResults.scoreDocs.length + + " results out of " + + actualVectorCount + + " available vectors"); + + // Verify all returned documents are valid and have expected metadata + for (ScoreDoc scoreDoc : vectorResults.scoreDocs) { + Document resultDoc = searcher.storedFields().document(scoreDoc.doc); + String docId = resultDoc.get("id"); + String indexType = resultDoc.get("index_type"); + + assertNotNull("Document should have valid ID", docId); + assertEquals( + "Document should be marked as CAGRA+brute force index type", + "cagra_and_brute_force", + indexType); + assertTrue("Score should be positive", scoreDoc.score > 0); + } + + // Test that the CAGRA+brute force index handles both approximate and exact search + // consistently + for (int trial = 0; trial < 3; trial++) { + float[] trialQueryVector = generateRandomVector(vectorDimension, random()); + KnnFloatVectorQuery trialQuery = + new KnnFloatVectorQuery("vector", trialQueryVector, Math.min(5, actualVectorCount)); + TopDocs trialResults = searcher.search(trialQuery, Math.min(5, actualVectorCount)); + + assertTrue("Trial " + trial + " should find results", trialResults.scoreDocs.length > 0); + log.info("Trial " + trial + " found " + trialResults.scoreDocs.length + " results"); + } + } else { + log.info("No vectors in CAGRA+brute force merged index - skipping vector search"); + } + + log.info("CAGRA+brute force merge verification completed successfully"); + } + } + + /** + * Test large scale merge to stress test the system + **/ + @Test + public void testLargeScaleMerge() throws IOException { + assumeTrue( + "testLargeScaleMerge requires -DlargeScale=true", + Boolean.parseBoolean(System.getProperty("largeScale", "false"))); + + log.info("Starting testLargeScaleMerge"); + + // Randomize large scale parameters + int maxBufferedDocs = 40 + random().nextInt(21); // 40-60 docs per buffer + int segmentCount = 15 + random().nextInt(11); // 15-25 segments + int docsPerSegment = 30 + random().nextInt(21); // 30-50 docs per segment + int totalDocuments = segmentCount * docsPerSegment; + + log.info( + "Randomized large scale parameters: maxBufferedDocs=" + + maxBufferedDocs + + ", segmentCount=" + + segmentCount + + ", docsPerSegment=" + + docsPerSegment + + ", totalDocuments=" + + totalDocuments); + + IndexWriterConfig config = + new IndexWriterConfig() + .setCodec(alwaysKnnVectorsFormat(new CuVSVectorsFormat())) + .setMaxBufferedDocs(maxBufferedDocs) + .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); + + try (IndexWriter writer = new IndexWriter(directory, config)) { + for (int seg = 0; seg < segmentCount; seg++) { + log.info("Creating segment " + (seg + 1) + "/" + segmentCount); + + // Randomize vector probability per segment + double vectorProbability = + 0.5 + (random().nextDouble() * 0.4); // 50-90% vectors per segment + + for (int i = 0; i < docsPerSegment; i++) { + int docId = seg * docsPerSegment + i; + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(docId), Field.Store.YES)); + doc.add(new NumericDocValuesField("segment", seg)); + doc.add(new NumericDocValuesField("position", i)); + + // Add vector based on segment's randomized probability + if (random().nextDouble() < vectorProbability) { + float[] vector = generateRandomVector(vectorDimension, random()); + doc.add(new KnnFloatVectorField("vector", vector, VectorSimilarityFunction.COSINE)); + } + + writer.addDocument(doc); + } + writer.commit(); + } + + log.info("Created " + segmentCount + " segments with " + totalDocuments + " total documents"); + + // Force merge all segments + long startTime = System.currentTimeMillis(); + writer.forceMerge(1); + long mergeTime = System.currentTimeMillis() - startTime; + + log.info("Large scale merge completed in " + mergeTime + "ms"); + } + + // Verify the large merged index + try (DirectoryReader reader = DirectoryReader.open(directory)) { + assertEquals("Should have exactly one segment after merge", 1, reader.leaves().size()); + + LeafReader leafReader = reader.leaves().get(0).reader(); + assertEquals("Total documents should match", totalDocuments, leafReader.maxDoc()); + + // Test vector search performance + var vectorValues = leafReader.getFloatVectorValues("vector"); + int actualVectorCount = vectorValues != null ? vectorValues.size() : 0; + + if (actualVectorCount > 0) { + IndexSearcher searcher = new IndexSearcher(reader); + float[] queryVector = generateRandomVector(vectorDimension, random()); + + // Randomize search parameters for large scale test + int searchK = + Math.min(20 + random().nextInt(31), Math.min(actualVectorCount, TOP_K_LIMIT)); // 20-50 + + long searchStart = System.currentTimeMillis(); + KnnFloatVectorQuery vectorQuery = new KnnFloatVectorQuery("vector", queryVector, searchK); + TopDocs vectorResults = searcher.search(vectorQuery, searchK); + long searchTime = System.currentTimeMillis() - searchStart; + + assertTrue("Should find vector results in large index", vectorResults.scoreDocs.length > 0); + log.info( + "Vector search in large index returned " + + vectorResults.scoreDocs.length + + " results out of " + + actualVectorCount + + " vectors in " + + searchTime + + "ms"); + } else { + log.info("No vectors in large merged index - skipping vector search"); + } + + log.info("Large scale merge verification completed successfully"); + } + } + + /** Helper method to generate random vectors */ + private float[] generateRandomVector(int dimension, Random random) { + float[] vector = new float[dimension]; + for (int i = 0; i < dimension; i++) { + vector[i] = (float) random().nextGaussian(); + } + // Normalize the vector + float norm = 0.0f; + for (float v : vector) { + norm += v * v; + } + norm = (float) Math.sqrt(norm); + if (norm > 0) { + for (int i = 0; i < dimension; i++) { + vector[i] /= norm; + } + } + return vector; + } + + /** Helper method to generate random text strings for sorting */ + private String generateRandomText(Random random, int length) { + StringBuilder sb = new StringBuilder(length); + String chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + for (int i = 0; i < length; i++) { + sb.append(chars.charAt(random().nextInt(chars.length()))); + } + return sb.toString(); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestUtils.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestUtils.java new file mode 100644 index 0000000000..8bd8339ffe --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestUtils.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.cuvs.lucene; + +import java.util.Random; + +public class TestUtils { + + public static float[][] generateDataset(Random random, int size, int dimensions) { + float[][] dataset = new float[size][dimensions]; + for (int i = 0; i < size; i++) { + for (int j = 0; j < dimensions; j++) { + dataset[i][j] = random.nextFloat() * 100; + } + } + return dataset; + } + + public static float[] generateRandomVector(int dimensions, Random random) { + float[] vector = new float[dimensions]; + for (int i = 0; i < dimensions; i++) { + vector[i] = random.nextFloat() * 100; + } + return vector; + } + + public static float[][] generateQueries(Random random, int dimensions, int numQueries) { + // Generate random query vectors + float[][] queries = new float[numQueries][dimensions]; + for (int i = 0; i < numQueries; i++) { + for (int j = 0; j < dimensions; j++) { + queries[i][j] = random.nextFloat() * 100; + } + } + return queries; + } +} From 19aadc1f34fc6fd52651a9ccfe714c65ba0b9778 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Fri, 26 Sep 2025 14:56:03 -0400 Subject: [PATCH 021/105] CAGRA to HNSW serialization and search on CPU (NVIDIA/cuvs-lucene#16) Introducing a new Codec that uses CAGRA for building the index on GPU and serializing to Lucene-compatible HNSW index segments. The Lucene-compatible segments are searchable via the `Lucene99HnswVectorsReader` (which is the default in Lucene 10.x). Note: This is based on top of NVIDIA/cuvs-lucene#14 and should be rebased once that is merged. TODO: - Benchmarks and more tests - Further refactoring to split the `CuVSVectorsFormat` into GPU and CPU-specific formats. Fixes NVIDIA/cuvs-lucene#13 Authors: - Vivek Narang (https://github.com/narangvivek10) - Puneet Ahuja (https://github.com/punAhuja) - Ishan Chattopadhyaya (https://github.com/chatman) Approvers: - Ishan Chattopadhyaya (https://github.com/chatman) - Corey J. Nolet (https://github.com/cjnolet) - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/cuvs-lucene/pull/16 --- java/cuvs-lucene/ci/release/update-version.sh | 4 + .../all_cuda-129_arch-aarch64.yaml | 2 +- .../all_cuda-129_arch-x86_64.yaml | 2 +- .../all_cuda-130_arch-aarch64.yaml | 2 +- .../all_cuda-130_arch-x86_64.yaml | 2 +- java/cuvs-lucene/dependencies.yaml | 2 +- java/cuvs-lucene/pom.xml | 48 +- ...Codec.java => CuVS2510GPUSearchCodec.java} | 44 +- ...mat.java => CuVS2510GPUVectorsFormat.java} | 107 ++- ...der.java => CuVS2510GPUVectorsReader.java} | 32 +- ...ter.java => CuVS2510GPUVectorsWriter.java} | 302 ++++---- .../nvidia/cuvs/lucene/CuVSSegmentFile.java | 57 -- .../nvidia/cuvs/lucene/GPUBuiltHnswGraph.java | 258 +++++++ ...VSFieldWriter.java => GPUFieldWriter.java} | 12 +- .../lucene/{CuVSIndex.java => GPUIndex.java} | 8 +- ...Query.java => GPUKnnFloatVectorQuery.java} | 9 +- ...r.java => GPUPerLeafCuVSKnnCollector.java} | 4 +- .../cuvs/lucene/IndexOutputOutputStream.java | 14 +- .../lucene/Lucene101AcceleratedHNSWCodec.java | 97 +++ .../Lucene99AcceleratedHNSWVectorsFormat.java | 161 ++++ .../Lucene99AcceleratedHNSWVectorsWriter.java | 702 ++++++++++++++++++ .../java/com/nvidia/cuvs/lucene/Utils.java | 35 + .../services/org.apache.lucene.codecs.Codec | 2 + .../org.apache.lucene.codecs.KnnVectorsFormat | 3 +- ...TestCagraToHnswSerializationAndSearch.java | 167 +++++ ...ializationAndSearchWithFallbackWriter.java | 173 +++++ .../cuvs/lucene/TestCuVSDeletedDocuments.java | 8 +- .../com/nvidia/cuvs/lucene/TestCuVSGaps.java | 12 +- .../TestCuVSRandomizedVectorSearch.java | 10 +- .../cuvs/lucene/TestCuVSVectorsFormat.java | 4 +- .../com/nvidia/cuvs/lucene/TestMerge.java | 24 +- 31 files changed, 1921 insertions(+), 386 deletions(-) rename java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/{CuVSCodec.java => CuVS2510GPUSearchCodec.java} (50%) rename java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/{CuVSVectorsFormat.java => CuVS2510GPUVectorsFormat.java} (56%) rename java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/{CuVSVectorsReader.java => CuVS2510GPUVectorsReader.java} (93%) rename java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/{CuVSVectorsWriter.java => CuVS2510GPUVectorsWriter.java} (78%) delete mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSSegmentFile.java create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java rename java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/{CuVSFieldWriter.java => GPUFieldWriter.java} (84%) rename java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/{CuVSIndex.java => GPUIndex.java} (92%) rename java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/{CuVSKnnFloatVectorQuery.java => GPUKnnFloatVectorQuery.java} (86%) rename java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/{PerLeafCuVSKnnCollector.java => GPUPerLeafCuVSKnnCollector.java} (93%) create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java create mode 100644 java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java diff --git a/java/cuvs-lucene/ci/release/update-version.sh b/java/cuvs-lucene/ci/release/update-version.sh index c7ea7a1bce..4a6bf41782 100755 --- a/java/cuvs-lucene/ci/release/update-version.sh +++ b/java/cuvs-lucene/ci/release/update-version.sh @@ -36,3 +36,7 @@ sed_runner "s/VERSION=\".*\"/VERSION=\"${NEXT_FULL_JAVA_TAG}\"/g" build.sh sed_runner "/.*/s//${NEXT_FULL_JAVA_TAG}<\/version>/g" pom.xml sed_runner "s| CuVS [[:digit:]]\{2\}\.[[:digit:]]\{2\} | CuVS ${NEXT_SHORT_TAG} |g" README.md + +for FILE in dependencies.yaml conda/environments/*.yaml; do + sed_runner "s/libcuvs==.*/libcuvs==${NEXT_SHORT_TAG}.*/g" "${FILE}" +done diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index 0b310207ce..05ec85b6ca 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -12,7 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs +- libcuvs==25.10.* - maven - openjdk=22.* name: all_cuda-129_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index a8f0444b68..f0a2538080 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -12,7 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs +- libcuvs==25.10.* - maven - openjdk=22.* name: all_cuda-129_arch-x86_64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml index 50e93bd652..6638808caf 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -12,7 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs +- libcuvs==25.10.* - maven - openjdk=22.* name: all_cuda-130_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml index c1e7ab3a9e..bb8a746f4a 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -12,7 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs +- libcuvs==25.10.* - maven - openjdk=22.* name: all_cuda-130_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index ee0613541d..b9eb3fcfd9 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -67,7 +67,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev - - libcuvs + - libcuvs==25.10.* java: common: - output_types: conda diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 6400cc49e3..dfd949758f 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -16,7 +16,7 @@ - cuvs-java + searchscale-maven SearchScale Maven https://maven.searchscale.com/snapshots @@ -40,36 +40,17 @@ 10.2.0 test - - com.opencsv - opencsv - 5.3 - commons-io commons-io - 2.15.1 - - - com.github.fommil - jniloader - 1.1 - - - com.fasterxml.jackson.core - jackson-databind - 2.17.0 - - - com.fasterxml.jackson.dataformat - jackson-dataformat-csv - 2.17.0 + 2.18.0 + test com.nvidia.cuvs cuvs-java - - 25.10.0-2c0e1-SNAPSHOT + + 25.10.0 @@ -104,6 +85,25 @@ + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + jar-with-dependencies + + + + + make-assembly + package + + single + + + + diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java similarity index 50% rename from java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodec.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index ec861765db..4027d3aba6 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -16,41 +16,57 @@ package com.nvidia.cuvs.lucene; import com.nvidia.cuvs.LibraryException; -import com.nvidia.cuvs.lucene.CuVSVectorsWriter.IndexType; +import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.lucene101.Lucene101Codec; -/** CuVS based codec for GPU based vector search */ -public class CuVSCodec extends FilterCodec { +/** CuVS based codec for GPU based vector search + * + * @apiNote cuVS serialization formats are in experimental phase and hence backward compatibility cannot be guaranteed. + * + * */ +public class CuVS2510GPUSearchCodec extends FilterCodec { + + private static final Logger log = Logger.getLogger(CuVS2510GPUSearchCodec.class.getName()); + private static final String NAME = "CuVS2510GPUSearchCodec"; - public CuVSCodec() { - this("CuVSCodec", new Lucene101Codec()); + private static final int DEFAULT_CUVS_WRITER_THREADS = 1; + private static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; + private static final int DEFAULT_GRAPH_DEGREE = 64; + private static final int DEFAULT_HNSW_LAYERS = 1; + private static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; + + private KnnVectorsFormat format; + + public CuVS2510GPUSearchCodec() { + this(NAME, new Lucene101Codec()); } - public CuVSCodec(String name, Codec delegate) { + public CuVS2510GPUSearchCodec(String name, Codec delegate) { super(name, delegate); - KnnVectorsFormat format; try { - // TODO: Remove this hard coded values. - format = new CuVSVectorsFormat(1, 128, 64, IndexType.CAGRA); + format = + new CuVS2510GPUVectorsFormat( + DEFAULT_CUVS_WRITER_THREADS, + DEFAULT_INTERMEDIATE_GRAPH_DEGREE, + DEFAULT_GRAPH_DEGREE, + DEFAULT_HNSW_LAYERS, + DEFAULT_INDEX_TYPE); setKnnFormat(format); } catch (LibraryException ex) { - Logger log = Logger.getLogger(CuVSCodec.class.getName()); log.severe("Couldn't load native library, possible classloader issue. " + ex.getMessage()); } } - KnnVectorsFormat knnFormat = null; - @Override public KnnVectorsFormat knnVectorsFormat() { - return knnFormat; + return format; } public void setKnnFormat(KnnVectorsFormat format) { - this.knnFormat = format; + this.format = format; } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java similarity index 56% rename from java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsFormat.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java index 1f7e4395d9..bccff5adf1 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java @@ -15,12 +15,15 @@ */ package com.nvidia.cuvs.lucene; +import static com.nvidia.cuvs.lucene.Utils.cuVSResourcesOrNull; + import com.nvidia.cuvs.CuVSResources; import com.nvidia.cuvs.LibraryException; -import com.nvidia.cuvs.lucene.CuVSVectorsWriter.IndexType; +import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; import java.io.IOException; import java.util.logging.Logger; import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; @@ -28,23 +31,24 @@ import org.apache.lucene.index.SegmentWriteState; /** CuVS based KnnVectorsFormat for GPU acceleration */ -public class CuVSVectorsFormat extends KnnVectorsFormat { +public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { - private static final Logger LOG = Logger.getLogger(CuVSVectorsFormat.class.getName()); + static final Logger log = Logger.getLogger(CuVS2510GPUVectorsFormat.class.getName()); // TODO: fix Lucene version in name, to the final targeted release, if any static final String CUVS_META_CODEC_NAME = "Lucene102CuVSVectorsFormatMeta"; - static final String CUVS_META_CODEC_EXT = "vemc"; // ""cagmf"; + static final String CUVS_META_CODEC_EXT = "vemc"; static final String CUVS_INDEX_CODEC_NAME = "Lucene102CuVSVectorsFormatIndex"; static final String CUVS_INDEX_EXT = "vcag"; static final int VERSION_START = 0; static final int VERSION_CURRENT = VERSION_START; - public static final int DEFAULT_WRITER_THREADS = 32; - public static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; - public static final int DEFAULT_GRAPH_DEGREE = 64; - public static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; + static final int DEFAULT_WRITER_THREADS = 32; + static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; + static final int DEFAULT_GRAPH_DEGREE = 64; + static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; + static final int HNSW_GRAPH_LAYERS = 1; static CuVSResources resources = cuVSResourcesOrNull(); @@ -56,80 +60,61 @@ public class CuVSVectorsFormat extends KnnVectorsFormat { final int cuvsWriterThreads; final int intGraphDegree; final int graphDegree; - final CuVSVectorsWriter.IndexType indexType; // the index type to build, when writing + final int hnswLayers; // Number of layers to create in CAGRA->HNSW conversion + final CuVS2510GPUVectorsWriter.IndexType indexType; // the index type to build, when writing /** - * Creates a CuVSVectorsFormat, with default values. + * Creates a CuVS2510GPUVectorsFormat, with default values. * * @throws LibraryException if the native library fails to load */ - public CuVSVectorsFormat() { + public CuVS2510GPUVectorsFormat() { this( DEFAULT_WRITER_THREADS, DEFAULT_INTERMEDIATE_GRAPH_DEGREE, DEFAULT_GRAPH_DEGREE, + HNSW_GRAPH_LAYERS, DEFAULT_INDEX_TYPE); } /** - * Creates a CuVSVectorsFormat, with the given threads, graph degree, etc. + * Creates a CuVS2510GPUVectorsFormat, with the given threads, graph degree, etc. * * @throws LibraryException if the native library fails to load */ - public CuVSVectorsFormat( - int cuvsWriterThreads, int intGraphDegree, int graphDegree, IndexType indexType) { - super("CuVSVectorsFormat"); + public CuVS2510GPUVectorsFormat( + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + IndexType indexType) { + super("CuVS2510GPUVectorsFormat"); this.cuvsWriterThreads = cuvsWriterThreads; this.intGraphDegree = intGraphDegree; this.graphDegree = graphDegree; + this.hnswLayers = hnswLayers; this.indexType = indexType; } - private static CuVSResources cuVSResourcesOrNull() { - try { - System.loadLibrary( - "cudart"); // nocommit: this is here so as to pass CI, should goto cuvs-java - } catch (UnsatisfiedLinkError e) { - LOG.warning("Could not load CUDA runtime library: " + e.getMessage()); - } - try { - resources = CuVSResources.create(); - return resources; - } catch (UnsupportedOperationException uoe) { - LOG.warning("cuvs is not supported on this platform or java version: " + uoe.getMessage()); - } catch (Throwable t) { - if (t instanceof ExceptionInInitializerError ex) { - t = ex.getCause(); - } - LOG.warning("Exception occurred during creation of cuvs resources. " + t); - } - return null; - } - - /** Tells whether the platform supports cuvs. */ - public static boolean supported() { - return resources != null; - } - - private static void checkSupported() { - if (!supported()) { - throw new UnsupportedOperationException(); - } - } - @Override - public CuVSVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { checkSupported(); var flatWriter = flatVectorsFormat.fieldsWriter(state); - return new CuVSVectorsWriter( - state, cuvsWriterThreads, intGraphDegree, graphDegree, indexType, resources, flatWriter); + return new CuVS2510GPUVectorsWriter( + state, + cuvsWriterThreads, + intGraphDegree, + graphDegree, + hnswLayers, + indexType, + resources, + flatWriter); } @Override - public CuVSVectorsReader fieldsReader(SegmentReadState state) throws IOException { + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { checkSupported(); - var flatReader = flatVectorsFormat.fieldsReader(state); - return new CuVSVectorsReader(state, resources, flatReader); + return new CuVS2510GPUVectorsReader(state, resources, flatVectorsFormat.fieldsReader(state)); } @Override @@ -139,12 +124,24 @@ public int getMaxDimensions(String fieldName) { @Override public String toString() { - StringBuilder sb = new StringBuilder("CuVSVectorsFormat("); - sb.append("cuvsWriterThreads=").append(cuvsWriterThreads); + StringBuilder sb = new StringBuilder(this.getClass().getSimpleName()); + sb.append("(cuvsWriterThreads=").append(cuvsWriterThreads); sb.append("intGraphDegree=").append(intGraphDegree); sb.append("graphDegree=").append(graphDegree); + sb.append("hnswLayers=").append(hnswLayers); sb.append("resources=").append(resources); sb.append(")"); return sb.toString(); } + + /** Tells whether the platform supports cuVS. */ + public static boolean supported() { + return resources != null; + } + + public static void checkSupported() { + if (!supported()) { + throw new UnsupportedOperationException(); + } + } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java similarity index 93% rename from java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java index 4118a0aac7..a9fe61a51b 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java @@ -15,12 +15,12 @@ */ package com.nvidia.cuvs.lucene; -import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_INDEX_CODEC_NAME; -import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_INDEX_EXT; -import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_META_CODEC_EXT; -import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_META_CODEC_NAME; -import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.VERSION_CURRENT; -import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.VERSION_START; +import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_INDEX_CODEC_NAME; +import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_INDEX_EXT; +import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_META_CODEC_EXT; +import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_META_CODEC_NAME; +import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.VERSION_CURRENT; +import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.VERSION_START; import com.nvidia.cuvs.BruteForceIndex; import com.nvidia.cuvs.BruteForceQuery; @@ -62,19 +62,19 @@ import org.apache.lucene.util.hnsw.IntToIntFunction; /** KnnVectorsReader instance associated with CuVS format */ -public class CuVSVectorsReader extends KnnVectorsReader { +public class CuVS2510GPUVectorsReader extends KnnVectorsReader { @SuppressWarnings("unused") - private static final Logger log = Logger.getLogger(CuVSVectorsReader.class.getName()); + private static final Logger log = Logger.getLogger(CuVS2510GPUVectorsReader.class.getName()); private final CuVSResources resources; private final FlatVectorsReader flatVectorsReader; // for reading the raw vectors private final FieldInfos fieldInfos; private final IntObjectHashMap fields; - private final IntObjectHashMap cuvsIndices; + private final IntObjectHashMap cuvsIndices; private final IndexInput cuvsIndexInput; - public CuVSVectorsReader( + public CuVS2510GPUVectorsReader( SegmentReadState state, CuVSResources resources, FlatVectorsReader flatReader) throws IOException { this.resources = resources; @@ -226,8 +226,8 @@ private FieldEntry getFieldEntry(String field, VectorEncoding expectedEncoding) return fieldEntry; } - private IntObjectHashMap loadCuVSIndices() throws IOException { - var indices = new IntObjectHashMap(); + private IntObjectHashMap loadCuVSIndices() throws IOException { + var indices = new IntObjectHashMap(); for (var e : fields) { var fieldEntry = e.value; int fieldNumber = e.key; @@ -237,7 +237,7 @@ private IntObjectHashMap loadCuVSIndices() throws IOException { return indices; } - private CuVSIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { + private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { CagraIndex cagraIndex = null; BruteForceIndex bruteForceIndex = null; HnswIndex hnswIndex = null; @@ -273,7 +273,7 @@ private CuVSIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { } catch (Throwable t) { Utils.handleThrowable(t); } - return new CuVSIndex(cagraIndex, bruteForceIndex, hnswIndex); + return new GPUIndex(cagraIndex, bruteForceIndex, hnswIndex); } @Override @@ -335,7 +335,7 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits var fieldNumber = fieldInfos.fieldInfo(field).number; - CuVSIndex cuvsIndex = cuvsIndices.get(fieldNumber); + GPUIndex cuvsIndex = cuvsIndices.get(fieldNumber); if (cuvsIndex == null) { throw new IllegalStateException("not index found for field:" + field); } @@ -476,7 +476,7 @@ public FieldInfos getFieldInfos() { return fieldInfos; } - public IntObjectHashMap getCuvsIndexes() { + public IntObjectHashMap getCuvsIndexes() { return cuvsIndices; } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java similarity index 78% rename from java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index 7861ad12b4..69d66db00f 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -15,11 +15,11 @@ */ package com.nvidia.cuvs.lucene; -import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_INDEX_CODEC_NAME; -import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_INDEX_EXT; -import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_META_CODEC_EXT; -import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.CUVS_META_CODEC_NAME; -import static com.nvidia.cuvs.lucene.CuVSVectorsFormat.VERSION_CURRENT; +import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_INDEX_CODEC_NAME; +import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_INDEX_EXT; +import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_META_CODEC_EXT; +import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_META_CODEC_NAME; +import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.VERSION_CURRENT; import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; import static org.apache.lucene.index.VectorEncoding.FLOAT32; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; @@ -36,11 +36,9 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.function.Supplier; import java.util.logging.Logger; import java.util.stream.IntStream; import org.apache.lucene.codecs.CodecUtil; @@ -69,15 +67,16 @@ * KnnVectorsWriter for CuVS, responsible for merge and flush of vectors into * GPU */ -public class CuVSVectorsWriter extends KnnVectorsWriter { +public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { - private static final long SHALLOW_RAM_BYTES_USED = shallowSizeOfInstance(CuVSVectorsWriter.class); + private static final long SHALLOW_RAM_BYTES_USED = + shallowSizeOfInstance(CuVS2510GPUVectorsWriter.class); @SuppressWarnings("unused") - private static final Logger log = Logger.getLogger(CuVSVectorsWriter.class.getName()); + private static final Logger log = Logger.getLogger(CuVS2510GPUVectorsWriter.class.getName()); /** The name of the CUVS component for the info-stream * */ - public static final String CUVS_COMPONENT = "CUVS"; + private static final String CUVS_COMPONENT = "CUVS"; // The minimum number of vectors in the dataset required before // we attempt to build a Cagra index @@ -91,19 +90,24 @@ public class CuVSVectorsWriter extends KnnVectorsWriter { private final IndexType indexType; private final FlatVectorsWriter flatVectorsWriter; // for writing the raw vectors - private final List fields = new ArrayList<>(); - private final IndexOutput meta, cuvsIndex; + private final List fields = new ArrayList<>(); + private IndexOutput meta = null, cuvsIndex = null; + private IndexOutput hnswMeta = null, hnswVectorIndex = null; private final InfoStream infoStream; private boolean finished; /** The CuVS index Type. */ public enum IndexType { + /** Builds a Cagra index. */ CAGRA(true, false, false), + /** Builds a Brute Force index. */ BRUTE_FORCE(false, true, false), + /** Builds an HSNW index - suitable for searching on CPU. */ HNSW(false, false, true), + /** Builds a Cagra and a Brute Force index. */ CAGRA_AND_BRUTE_FORCE(true, true, false); private final boolean cagra, bruteForce, hnsw; @@ -127,11 +131,12 @@ public boolean hnsw() { } } - public CuVSVectorsWriter( + public CuVS2510GPUVectorsWriter( SegmentWriteState state, int cuvsWriterThreads, int intGraphDegree, int graphDegree, + int hnswLayers, IndexType indexType, CuVSResources resources, FlatVectorsWriter flatVectorsWriter) @@ -153,6 +158,7 @@ public CuVSVectorsWriter( boolean success = false; try { + meta = state.directory.createOutput(metaFileName, state.context); cuvsIndex = state.directory.createOutput(cagraFileName, state.context); CodecUtil.writeIndexHeader( @@ -167,6 +173,7 @@ public CuVSVectorsWriter( VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); + success = true; } finally { if (success == false) { @@ -184,7 +191,7 @@ public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException var writer = Objects.requireNonNull(flatVectorsWriter.addField(fieldInfo)); @SuppressWarnings("unchecked") var flatWriter = (FlatFieldVectorsWriter) writer; - var cuvsFieldWriter = new CuVSFieldWriter(fieldInfo, flatWriter); + var cuvsFieldWriter = new GPUFieldWriter(fieldInfo, flatWriter); fields.add(cuvsFieldWriter); return writer; } @@ -213,25 +220,89 @@ private CagraIndexParams cagraIndexParams(int size) { .build(); } - static long nanosToMillis(long nanos) { - return Duration.ofNanos(nanos).toMillis(); - } - private void info(String msg) { if (infoStream.isEnabled(CUVS_COMPONENT)) { infoStream.message(CUVS_COMPONENT, msg); } } + private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOException { + if (vectors.size() == 0) { + writeEmpty(fieldInfo); + return; + } + long cagraIndexOffset, cagraIndexLength = 0L; + long bruteForceIndexOffset, bruteForceIndexLength = 0L; + long hnswIndexOffset, hnswIndexLength = 0L; + + // workaround for the minimum number of vectors for Cagra + IndexType indexType = + this.indexType.cagra() && vectors.size() < MIN_CAGRA_INDEX_SIZE + ? IndexType.BRUTE_FORCE + : this.indexType; + + try { + + cagraIndexOffset = cuvsIndex.getFilePointer(); + if (indexType.cagra()) { + try { + var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); + CuVSMatrix dataset = Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension()); + writeCagraIndex(cagraIndexOutputStream, dataset); + } catch (Throwable t) { + Utils.handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); + // workaround for cuVS issue + indexType = IndexType.BRUTE_FORCE; + } + cagraIndexLength = cuvsIndex.getFilePointer() - cagraIndexOffset; + } + + bruteForceIndexOffset = cuvsIndex.getFilePointer(); + if (indexType.bruteForce()) { + var bruteForceIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); + CuVSMatrix dataset = Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension()); + writeBruteForceIndex(bruteForceIndexOutputStream, dataset); + bruteForceIndexLength = cuvsIndex.getFilePointer() - bruteForceIndexOffset; + } + + hnswIndexOffset = cuvsIndex.getFilePointer(); + if (indexType.hnsw()) { + var hnswIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); + if (vectors.size() > MIN_CAGRA_INDEX_SIZE) { + try { + CuVSMatrix dataset = Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension()); + writeHNSWIndex(hnswIndexOutputStream, dataset); + } catch (Throwable t) { + Utils.handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); + } + } + hnswIndexLength = cuvsIndex.getFilePointer() - hnswIndexOffset; + } + + // Only write meta for non-HNSW_LUCENE modes + writeMeta( + fieldInfo, + vectors.size(), + cagraIndexOffset, + cagraIndexLength, + bruteForceIndexOffset, + bruteForceIndexLength, + hnswIndexOffset, + hnswIndexLength); + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + private void writeCagraIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { if (dataset.size() < 2) { throw new IllegalArgumentException(dataset.size() + " vectors, less than min [2] required"); } CagraIndexParams params = cagraIndexParams((int) dataset.size()); long startTime = System.nanoTime(); - var index = + CagraIndex index = CagraIndex.newBuilder(resources).withDataset(dataset).withIndexParams(params).build(); - long elapsedMillis = nanosToMillis(System.nanoTime() - startTime); + long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - startTime); info("Cagra index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); Path tmpFile = Files.createTempFile(resources.tempDirectory(), "tmpindex", "cag"); index.serialize(os, tmpFile); @@ -241,13 +312,12 @@ private void writeCagraIndex(OutputStream os, CuVSMatrix dataset) throws Throwab private void writeBruteForceIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { BruteForceIndexParams params = new BruteForceIndexParams.Builder() - .withNumWriterThreads(32) // TODO: Make this - // configurable later. + .withNumWriterThreads(32) // TODO: Make this configurable. .build(); long startTime = System.nanoTime(); var index = BruteForceIndex.newBuilder(resources).withIndexParams(params).withDataset(dataset).build(); - long elapsedMillis = nanosToMillis(System.nanoTime() - startTime); + long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - startTime); info("bf index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); index.serialize(os); index.close(); @@ -259,9 +329,9 @@ private void writeHNSWIndex(OutputStream os, CuVSMatrix dataset) throws Throwabl } CagraIndexParams indexParams = cagraIndexParams((int) dataset.size()); long startTime = System.nanoTime(); - var index = + CagraIndex index = CagraIndex.newBuilder(resources).withDataset(dataset).withIndexParams(indexParams).build(); - long elapsedMillis = nanosToMillis(System.nanoTime() - startTime); + long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - startTime); info("HNSW index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); Path tmpFile = Files.createTempFile("tmpindex", "hnsw"); index.serializeToHNSW(os, tmpFile); @@ -280,96 +350,23 @@ public void flush(int maxDoc, DocMap sortMap) throws IOException { } } - private void writeField(CuVSFieldWriter fieldData) throws IOException { - // TODO: Loading all vectors into memory is inefficient. Is there a way to stream the vectors - // from the flat writer to the CuVSMatrix? - List vectors = fieldData.getVectors(); - writeFieldInternal( - fieldData.fieldInfo(), - () -> Utils.createFloatMatrix(vectors, fieldData.fieldInfo().getVectorDimension()), - vectors.size()); + private void writeField(GPUFieldWriter fieldData) throws IOException { + writeFieldInternal(fieldData.fieldInfo(), fieldData.getVectors()); } - private void writeSortingField(CuVSFieldWriter fieldData, Sorter.DocMap sortMap) + private void writeSortingField(GPUFieldWriter fieldData, Sorter.DocMap sortMap) throws IOException { + DocsWithFieldSet oldDocsWithFieldSet = fieldData.getDocsWithFieldSet(); final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; // new ord to old ord mapOldOrdToNewOrd(oldDocsWithFieldSet, sortMap, null, new2OldOrd, null); - // TODO: Loading all vectors into memory is inefficient. Is there a way to stream the vectors - // from the flat writer to the CuVSMatrix? + List sortedVectors = new ArrayList(); for (int i = 0; i < fieldData.getVectors().size(); i++) { sortedVectors.add(fieldData.getVectors().get(new2OldOrd[i])); } - writeFieldInternal( - fieldData.fieldInfo(), - () -> Utils.createFloatMatrix(sortedVectors, fieldData.fieldInfo().getVectorDimension()), - sortedVectors.size()); - } - private void writeFieldInternal( - FieldInfo fieldInfo, Supplier datasetSupplier, int datasetSize) - throws IOException { - if (datasetSize == 0) { - writeEmpty(fieldInfo); - return; - } - long cagraIndexOffset, cagraIndexLength = 0L; - long bruteForceIndexOffset, bruteForceIndexLength = 0L; - long hnswIndexOffset, hnswIndexLength = 0L; - - // workaround for the minimum number of vectors for Cagra - IndexType indexType = - this.indexType.cagra() && datasetSize < MIN_CAGRA_INDEX_SIZE - ? IndexType.BRUTE_FORCE - : this.indexType; - - try { - cagraIndexOffset = cuvsIndex.getFilePointer(); - if (indexType.cagra()) { - try { - var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - writeCagraIndex(cagraIndexOutputStream, datasetSupplier.get()); - } catch (Throwable t) { - handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); - // workaround for cuVS issue - indexType = IndexType.BRUTE_FORCE; - } - cagraIndexLength = cuvsIndex.getFilePointer() - cagraIndexOffset; - } - - bruteForceIndexOffset = cuvsIndex.getFilePointer(); - if (indexType.bruteForce()) { - var bruteForceIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - writeBruteForceIndex(bruteForceIndexOutputStream, datasetSupplier.get()); - bruteForceIndexLength = cuvsIndex.getFilePointer() - bruteForceIndexOffset; - } - - hnswIndexOffset = cuvsIndex.getFilePointer(); - if (indexType.hnsw()) { - var hnswIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - if (datasetSize > MIN_CAGRA_INDEX_SIZE) { - try { - writeHNSWIndex(hnswIndexOutputStream, datasetSupplier.get()); - } catch (Throwable t) { - handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); - } - } - hnswIndexLength = cuvsIndex.getFilePointer() - hnswIndexOffset; - } - - writeMeta( - fieldInfo, - (int) datasetSize, - cagraIndexOffset, - cagraIndexLength, - bruteForceIndexOffset, - bruteForceIndexLength, - hnswIndexOffset, - hnswIndexLength); - } catch (Throwable t) { - Utils.handleThrowable(t); - } + writeFieldInternal(fieldData.fieldInfo(), sortedVectors); } private void writeEmpty(FieldInfo fieldInfo) throws IOException { @@ -417,13 +414,6 @@ static int distFuncToOrd(VectorSimilarityFunction func) { during the norm computation between the dataset vectors\ """; - static void handleThrowableWithIgnore(Throwable t, String msg) throws IOException { - if (t.getMessage().contains(msg)) { - return; - } - Utils.handleThrowable(t); - } - private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throws IOException { try { @@ -436,7 +426,7 @@ private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throw // Access the CAGRA index for this field from the reader if (knnReader != null) { - if (knnReader instanceof CuVSVectorsReader cvr) { + if (knnReader instanceof CuVS2510GPUVectorsReader cvr) { if (cvr != null) { totalVectorCount += cvr.getFieldEntries().get(fieldInfo.number).count(); CagraIndex cagraIndex = getCagraIndexFromReader(cvr, fieldInfo.name); @@ -463,6 +453,21 @@ private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throw } } + /** + * Creates List from merged vectors + * */ + private List createListFromMergedVectors(FloatVectorValues mergedVectorValues) + throws IOException { + List res = new ArrayList(); + KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); + for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { + int ordinal = iter.index(); + float[] vector = mergedVectorValues.vectorValue(ordinal); + res.add(vector); + } + return res; + } + /** * Fallback method that rebuilds indexes from merged vectors. * Used when native CAGRA merge() is not possible. Also used @@ -473,70 +478,27 @@ private void vectorBasedMerge(FieldInfo fieldInfo, MergeState mergeState) throws throw new AssertionError("Only Float32 supported"); } try { - // We need to compute the size of the number of merged documents up-front so that we can - // compute the CuVSMatrix capacity. TODO: Find a way to do this without merging twice. - final int numMergedDocs = getMergedDocsCount(fieldInfo, mergeState); - - if (numMergedDocs != 0) { - writeFieldInternal( - fieldInfo, - () -> { - try { - return createMatrixFromMergedVectors( - KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues( - fieldInfo, mergeState), - numMergedDocs); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, - numMergedDocs); - } else { - writeEmpty(fieldInfo); - } + List dataset = + createListFromMergedVectors( + KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState)); + writeFieldInternal(fieldInfo, dataset); } catch (Throwable t) { Utils.handleThrowable(t); } } - private int getMergedDocsCount(FieldInfo fieldInfo, MergeState mergeState) throws IOException { - KnnVectorValues.DocIndexIterator iter = - KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState) - .iterator(); - int numMergedDocs = 0; - for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { - numMergedDocs++; - } - return numMergedDocs; - } - - /** - * Creates CuVSMatrix from merged vectors - * */ - private CuVSMatrix createMatrixFromMergedVectors( - FloatVectorValues mergedVectorValues, int numMergedDocs) throws IOException { - List vectors = new ArrayList<>(numMergedDocs); - KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); - for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { - int ordinal = iter.index(); - float[] vector = mergedVectorValues.vectorValue(ordinal); - vectors.add(vector.clone()); - } - return Utils.createFloatMatrix(vectors, mergedVectorValues.dimension()); - } - /** * Extracts the CAGRA index for a specific field from a CuVSVectorsReader. */ - private CagraIndex getCagraIndexFromReader(CuVSVectorsReader reader, String fieldName) { + private CagraIndex getCagraIndexFromReader(CuVS2510GPUVectorsReader reader, String fieldName) { try { - IntObjectHashMap cuvsIndices = reader.getCuvsIndexes(); + IntObjectHashMap cuvsIndices = reader.getCuvsIndexes(); FieldInfos fieldInfos = reader.getFieldInfos(); FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldName); if (fieldInfo != null) { - CuVSIndex cuvsIndex = cuvsIndices.get(fieldInfo.number); + GPUIndex cuvsIndex = cuvsIndices.get(fieldInfo.number); if (cuvsIndex != null) { return cuvsIndex.getCagraIndex(); } @@ -562,7 +524,6 @@ private void writeMergedCagraIndex(FieldInfo fieldInfo, CagraIndex mergedIndex, mergedIndex.serialize(cagraIndexOutputStream, tmpFile); long cagraIndexLength = cuvsIndex.getFilePointer() - cagraIndexOffset; - // Write metadata (assuming no brute force or HNSW indexes for merged result) writeMeta(fieldInfo, vectorCount, cagraIndexOffset, cagraIndexLength, 0L, 0L, 0L, 0L); // Clean up the merged index @@ -618,11 +579,22 @@ public void finish() throws IOException { if (cuvsIndex != null) { CodecUtil.writeFooter(cuvsIndex); } + + { + if (hnswMeta != null) { + // write end of fields marker + hnswMeta.writeInt(-1); + CodecUtil.writeFooter(hnswMeta); + } + if (hnswVectorIndex != null) { + CodecUtil.writeFooter(hnswVectorIndex); + } + } } @Override public void close() throws IOException { - IOUtils.close(meta, cuvsIndex, flatVectorsWriter); + IOUtils.close(meta, cuvsIndex, hnswMeta, hnswVectorIndex, flatVectorsWriter); } @Override diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSSegmentFile.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSSegmentFile.java deleted file mode 100644 index 8a601b7eee..0000000000 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSSegmentFile.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.nvidia.cuvs.lucene; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; -import java.util.logging.Logger; -import java.util.zip.Deflater; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; - -/** Methods to deal with a CuVS composite file inside a segment */ -/*package-private*/ class CuVSSegmentFile implements AutoCloseable { - private final ZipOutputStream zos; - - private Set filesAdded = new HashSet(); - - public CuVSSegmentFile(OutputStream out) { - zos = new ZipOutputStream(out); - zos.setLevel(Deflater.NO_COMPRESSION); - } - - protected Logger log = Logger.getLogger(getClass().getName()); - - public void addFile(String name, byte[] bytes) throws IOException { - ZipEntry indexFileZipEntry = new ZipEntry(name); - zos.putNextEntry(indexFileZipEntry); - zos.write(bytes, 0, bytes.length); - zos.closeEntry(); - filesAdded.add(name); - } - - public Set getFilesAdded() { - return Collections.unmodifiableSet(filesAdded); - } - - @Override - public void close() throws IOException { - zos.close(); - } -} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java new file mode 100644 index 0000000000..7235ec97ab --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.cuvs.lucene; + +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + +import com.nvidia.cuvs.CuVSMatrix; +import com.nvidia.cuvs.RowView; +import java.util.ArrayList; +import java.util.List; +import org.apache.lucene.util.hnsw.HnswGraph; +import org.apache.lucene.util.hnsw.NeighborArray; + +public class GPUBuiltHnswGraph extends HnswGraph { + + private final int size; + private final int dimensions; + private final int numLevels; + + // Store layers data - each layer has its own nodes and adjacency lists + private final List layerNodes; + private final List layerNeighbors; + + // Layer 0 is special - it contains all nodes + private final NeighborArray[] layer0Neighbors; + + // Multi-layer constructor that supports arbitrary number of layers + public GPUBuiltHnswGraph( + int size, int dimensions, List layerNodes, List layerAdjacencies) { + + this.size = size; + this.dimensions = dimensions; + this.numLevels = layerAdjacencies.size(); + this.layerNodes = new ArrayList<>(); + this.layerNeighbors = new ArrayList<>(); + + // Process Layer 0 (base layer with all nodes) + CuVSMatrix layer0Adjacency = layerAdjacencies.get(0); + this.layer0Neighbors = fillNeighborArray(layer0Adjacency, size); + + // Process higher layers (1 to numLevels-1) + for (int level = 1; level < numLevels; level++) { + int[] nodes = layerNodes.get(level); + CuVSMatrix adjacency = layerAdjacencies.get(level); + this.layerNodes.add(nodes); + this.layerNeighbors.add(fillNeighborArray(adjacency, nodes.length)); + } + } + + private NeighborArray[] fillNeighborArray(CuVSMatrix adjacency, int size) { + NeighborArray[] neighbors = new NeighborArray[size]; + for (int i = 0; i < size; i++) { + RowView rv = adjacency.getRow(i); + if (rv != null && rv.size() > 0) { + neighbors[i] = new NeighborArray((int) rv.size(), true); + for (int j = 0; j < rv.size(); j++) { + neighbors[i].addInOrder(rv.getAsInt(j), 1.0f - (j * 0.001f)); + } + } else { + neighbors[i] = new NeighborArray(0, true); + } + } + return neighbors; + } + + public NodesIterator getNodesOnLevel(int level) { + if (level == 0) { + return new Level0NodesIterator(size); + } else if (level > 0 && level < numLevels) { + int[] nodes = layerNodes.get(level - 1); + return new HigherLevelNodesIterator(nodes); + } else { + return new Level0NodesIterator(0); + } + } + + public NeighborArray getNeighbors(int level, int node) { + if (level == 0 && node < size) { + return layer0Neighbors[node]; + } else if (level > 0 && level < numLevels) { + int[] nodes = layerNodes.get(level - 1); + NeighborArray[] neighbors = layerNeighbors.get(level - 1); + + // Find the index of this node in the layer + for (int i = 0; i < nodes.length; i++) { + if (nodes[i] == node) { + return neighbors[i]; + } + } + } + return null; + } + + // Implementation of abstract methods from HnswGraph + private int currentNode = -1; + private int currentLevel = -1; + private int neighborIndex = -1; + + @Override + public void seek(int level, int target) { + currentLevel = level; + currentNode = target; + neighborIndex = -1; + } + + @Override + public int nextNeighbor() { + if (currentLevel == 0 + && currentNode >= 0 + && currentNode < size + && layer0Neighbors[currentNode] != null) { + neighborIndex++; + if (neighborIndex < layer0Neighbors[currentNode].size()) { + int neighborNode = layer0Neighbors[currentNode].nodes()[neighborIndex]; + if (neighborNode >= 0 && neighborNode < size) { + return neighborNode; + } else { + return nextNeighbor(); // Skip invalid neighbor + } + } + } else if (currentLevel > 0 && currentLevel < numLevels) { + // Handle higher layers + NeighborArray neighbors = getNeighbors(currentLevel, currentNode); + if (neighbors != null) { + neighborIndex++; + if (neighborIndex < neighbors.size()) { + return neighbors.nodes()[neighborIndex]; + } + } + } + return NO_MORE_DOCS; + } + + @Override + public int entryNode() { + // Entry node should be from the highest layer + if (numLevels > 1) { + int topLevel = numLevels - 1; + int[] topLayerNodes = layerNodes.get(topLevel - 1); + if (topLayerNodes != null && topLayerNodes.length > 0) { + // Use random node from top layer with fixed seed for reproducibility + java.util.Random random = new java.util.Random(44); + int randomIndex = random.nextInt(topLayerNodes.length); + return topLayerNodes[randomIndex]; + } + } + return 0; // Default to node 0 for single-layer graphs + } + + @Override + public int maxConn() { + // Return the maximum degree across all nodes in layer 0 + int max = 0; + for (NeighborArray neighbor : layer0Neighbors) { + if (neighbor != null) { + max = Math.max(max, neighbor.size()); + } + } + return max; + } + + @Override + public int neighborCount() { + if (currentLevel == 0 + && currentNode >= 0 + && currentNode < size + && layer0Neighbors[currentNode] != null) { + return layer0Neighbors[currentNode].size(); + } else if (currentLevel > 0 && currentLevel < numLevels) { + NeighborArray neighbors = getNeighbors(currentLevel, currentNode); + return neighbors != null ? neighbors.size() : 0; + } + return 0; + } + + // NodesIterator for level 0 + private static class Level0NodesIterator extends NodesIterator { + private int current = -1; + + Level0NodesIterator(int size) { + super(size); + } + + @Override + public boolean hasNext() { + return current + 1 < size; + } + + @Override + public int nextInt() { + return ++current; + } + + @Override + public int consume(int[] dest) { + int numToCopy = Math.min(dest.length, size - (current + 1)); + for (int i = 0; i < numToCopy; i++) { + dest[i] = ++current; + } + return numToCopy; + } + } + + // NodesIterator for higher layers + private static class HigherLevelNodesIterator extends NodesIterator { + private final int[] nodeIds; + private int current = -1; + + HigherLevelNodesIterator(int[] nodeIds) { + super(nodeIds.length); + this.nodeIds = nodeIds; + } + + @Override + public boolean hasNext() { + return current + 1 < nodeIds.length; + } + + @Override + public int nextInt() { + return nodeIds[++current]; + } + + @Override + public int consume(int[] dest) { + int numToCopy = Math.min(dest.length, nodeIds.length - (current + 1)); + for (int i = 0; i < numToCopy; i++) { + dest[i] = nodeIds[++current]; + } + return numToCopy; + } + } + + public int size() { + return size; + } + + public int numLevels() { + return numLevels; + } + + public int dimensions() { + return dimensions; + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSFieldWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java similarity index 84% rename from java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSFieldWriter.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java index acd151817a..483e18d394 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSFieldWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java @@ -24,16 +24,16 @@ import org.apache.lucene.util.RamUsageEstimator; /** CuVS based fields writer */ -/*package-private*/ class CuVSFieldWriter extends KnnFieldVectorsWriter { +/*package-private*/ class GPUFieldWriter extends KnnFieldVectorsWriter { private static final long SHALLOW_SIZE = - RamUsageEstimator.shallowSizeOfInstance(CuVSFieldWriter.class); + RamUsageEstimator.shallowSizeOfInstance(GPUFieldWriter.class); private final FieldInfo fieldInfo; private final FlatFieldVectorsWriter flatFieldVectorsWriter; private int lastDocID = -1; - public CuVSFieldWriter( + public GPUFieldWriter( FieldInfo fieldInfo, FlatFieldVectorsWriter flatFieldVectorsWriter) { this.fieldInfo = fieldInfo; this.flatFieldVectorsWriter = flatFieldVectorsWriter; @@ -74,6 +74,10 @@ public long ramBytesUsed() { @Override public String toString() { - return "CuVSFieldWriter[field name=" + fieldInfo.name + ", number=" + fieldInfo.number + "]"; + StringBuilder sb = new StringBuilder(this.getClass().getSimpleName()); + sb.append("(field name=").append(fieldInfo.name); + sb.append("number=").append(fieldInfo.number); + sb.append(")"); + return sb.toString(); } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSIndex.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java similarity index 92% rename from java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSIndex.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java index 11519807bc..ea33c842ca 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSIndex.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java @@ -23,7 +23,7 @@ import java.util.Objects; /** This class holds references to the actual CuVS Index (Cagra, Brute force, etc.) */ -public class CuVSIndex implements Closeable { +public class GPUIndex implements Closeable { private final CagraIndex cagraIndex; private final BruteForceIndex bruteforceIndex; private final HnswIndex hnswIndex; @@ -33,7 +33,7 @@ public class CuVSIndex implements Closeable { private String segmentName; private volatile boolean closed; - public CuVSIndex( + public GPUIndex( String segmentName, String fieldName, CagraIndex cagraIndex, @@ -47,10 +47,10 @@ public CuVSIndex( throw new IllegalArgumentException("negative maxDocs:" + maxDocs); } this.maxDocs = maxDocs; - this.hnswIndex = null; // TODO: + this.hnswIndex = null; // TODO: remove hnswlib logic in a subsequent PR } - public CuVSIndex(CagraIndex cagraIndex, BruteForceIndex bruteforceIndex, HnswIndex hnswIndex) { + public GPUIndex(CagraIndex cagraIndex, BruteForceIndex bruteforceIndex, HnswIndex hnswIndex) { this.cagraIndex = cagraIndex; this.bruteforceIndex = bruteforceIndex; this.hnswIndex = hnswIndex; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSKnnFloatVectorQuery.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java similarity index 86% rename from java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSKnnFloatVectorQuery.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java index 8caf30aea1..64f70811a0 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVSKnnFloatVectorQuery.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java @@ -18,19 +18,20 @@ import java.io.IOException; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.KnnCollector; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.knn.KnnCollectorManager; import org.apache.lucene.util.Bits; -/** Query for CuVS */ -public class CuVSKnnFloatVectorQuery extends KnnFloatVectorQuery { +/** Query on GPU only */ +public class GPUKnnFloatVectorQuery extends KnnFloatVectorQuery { private final int iTopK; private final int searchWidth; - public CuVSKnnFloatVectorQuery( + public GPUKnnFloatVectorQuery( String field, float[] target, int k, Query filter, int iTopK, int searchWidth) { super(field, target, k, filter); this.iTopK = iTopK; @@ -45,7 +46,7 @@ protected TopDocs approximateSearch( KnnCollectorManager knnCollectorManager) throws IOException { - PerLeafCuVSKnnCollector results = new PerLeafCuVSKnnCollector(k, iTopK, searchWidth); + KnnCollector results = new GPUPerLeafCuVSKnnCollector(k, iTopK, searchWidth); LeafReader reader = context.reader(); reader.searchNearestVectors(field, this.getTargetCopy(), results, acceptDocs); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/PerLeafCuVSKnnCollector.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java similarity index 93% rename from java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/PerLeafCuVSKnnCollector.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java index 8e005570fa..ca30d3d3ed 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/PerLeafCuVSKnnCollector.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java @@ -24,7 +24,7 @@ import org.apache.lucene.search.knn.KnnSearchStrategy; /** KnnCollector for CuVS */ -/*package-private*/ class PerLeafCuVSKnnCollector implements KnnCollector { +/*package-private*/ class GPUPerLeafCuVSKnnCollector implements KnnCollector { public List scoreDocs; public int topK = 0; @@ -32,7 +32,7 @@ public int searchWidth = 1; // TODO getter, no setter public int results = 0; - public PerLeafCuVSKnnCollector(int topK, int iTopK, int searchWidth) { + public GPUPerLeafCuVSKnnCollector(int topK, int iTopK, int searchWidth) { super(); this.topK = topK; this.iTopK = iTopK; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java index b7866363fc..53283ebf24 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java @@ -27,7 +27,7 @@ final class IndexOutputOutputStream extends OutputStream { final IndexOutput out; final int bufferSize; final byte[] buffer; - int idx; + int pos; IndexOutputOutputStream(IndexOutput out) { this(out, DEFAULT_BUFFER_SIZE); @@ -41,16 +41,16 @@ final class IndexOutputOutputStream extends OutputStream { @Override public void write(int b) throws IOException { - buffer[idx] = (byte) b; - idx++; - if (idx == bufferSize) { + buffer[pos] = (byte) b; + pos++; + if (pos == bufferSize) { flush(); } } @Override public void write(byte[] b, int offset, int length) throws IOException { - if (idx != 0) { + if (pos != 0) { flush(); } out.writeBytes(b, offset, length); @@ -58,8 +58,8 @@ public void write(byte[] b, int offset, int length) throws IOException { @Override public void flush() throws IOException { - out.writeBytes(buffer, 0, idx); - idx = 0; + out.writeBytes(buffer, 0, pos); + pos = 0; } @Override diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java new file mode 100644 index 0000000000..facd93d8e2 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.cuvs.lucene; + +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; + +import com.nvidia.cuvs.LibraryException; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.lucene101.Lucene101Codec; + +/** CuVS based codec for GPU based vector search */ +public class Lucene101AcceleratedHNSWCodec extends FilterCodec { + + private static final Logger log = Logger.getLogger(Lucene101AcceleratedHNSWCodec.class.getName()); + + private static final int DEFAULT_CUVS_WRITER_THREADS = 1; + private static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; + private static final int DEFAULT_GRAPH_DEGREE = 64; + private static final int DEFAULT_HNSW_LAYERS = 1; + private static final String NAME = "Lucene101AcceleratedHNSWCodec"; + + private KnnVectorsFormat format; + + public Lucene101AcceleratedHNSWCodec() { + this(NAME, new Lucene101Codec()); + } + + public Lucene101AcceleratedHNSWCodec(String name, Codec delegate) { + super(name, delegate); + initializeFormatDefaultValues(); + } + + public Lucene101AcceleratedHNSWCodec( + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + int maxConn, + int beamWidth) { + this(NAME, new Lucene101Codec()); + initializeFormat( + cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); + } + + private void initializeFormatDefaultValues() { + initializeFormat( + DEFAULT_CUVS_WRITER_THREADS, + DEFAULT_INTERMEDIATE_GRAPH_DEGREE, + DEFAULT_GRAPH_DEGREE, + DEFAULT_HNSW_LAYERS, + DEFAULT_MAX_CONN, + DEFAULT_BEAM_WIDTH); + } + + private void initializeFormat( + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + int maxConn, + int beamWidth) { + try { + format = + new Lucene99AcceleratedHNSWVectorsFormat( + cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); + setKnnFormat(format); + } catch (LibraryException ex) { + log.severe("Couldn't load native library, possible classloader issue. " + ex.getMessage()); + } + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return format; + } + + public void setKnnFormat(KnnVectorsFormat format) { + this.format = format; + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java new file mode 100644 index 0000000000..3a18aaef41 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.Utils.cuVSResourcesOrNull; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; + +import com.nvidia.cuvs.CuVSResources; +import com.nvidia.cuvs.LibraryException; +import java.io.IOException; +import java.util.logging.Logger; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; + +/** CuVS based KnnVectorsFormat for GPU acceleration */ +public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { + + private static final Logger log = + Logger.getLogger(Lucene99AcceleratedHNSWVectorsFormat.class.getName()); + + static final int DEFAULT_WRITER_THREADS = 32; + static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; + static final int DEFAULT_GRAPH_DEGREE = 64; + static final int DEFAULT_HNSW_GRAPH_LAYERS = 1; + + static final String HNSW_META_CODEC_NAME = "Lucene99HnswVectorsFormatMeta"; + static final String HNSW_META_CODEC_EXT = "vem"; + static final String HNSW_INDEX_CODEC_NAME = "Lucene99HnswVectorsFormatIndex"; + static final String HNSW_INDEX_EXT = "vex"; + + private static CuVSResources resources = cuVSResourcesOrNull(); + + /** The format for storing, reading, and merging raw vectors on disk. */ + private static final FlatVectorsFormat flatVectorsFormat = + new Lucene99FlatVectorsFormat(DefaultFlatVectorScorer.INSTANCE); + + private final int maxDimensions = 4096; + private final int cuvsWriterThreads; + private final int intGraphDegree; + private final int graphDegree; + private final int hnswLayers; // Number of layers to create in CAGRA->HNSW conversion + + private final int maxConn; + private final int beamWidth; + + /** + * Creates a Lucene99AcceleratedHNSWVectorsFormat, with default values. + * + * @throws LibraryException if the native library fails to load + */ + public Lucene99AcceleratedHNSWVectorsFormat() { + this( + DEFAULT_WRITER_THREADS, + DEFAULT_INTERMEDIATE_GRAPH_DEGREE, + DEFAULT_GRAPH_DEGREE, + DEFAULT_HNSW_GRAPH_LAYERS, + DEFAULT_MAX_CONN, + DEFAULT_BEAM_WIDTH); + } + + /** + * Creates a Lucene99AcceleratedHNSWVectorsFormat, with the given threads, graph degree, etc. + * + * @throws LibraryException if the native library fails to load + */ + public Lucene99AcceleratedHNSWVectorsFormat( + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + int maxConn, + int beamWidth) { + super("Lucene99AcceleratedHNSWVectorsFormat"); + this.cuvsWriterThreads = cuvsWriterThreads; + this.intGraphDegree = intGraphDegree; + this.graphDegree = graphDegree; + this.hnswLayers = hnswLayers; + this.maxConn = maxConn; + this.beamWidth = beamWidth; + } + + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + var flatWriter = flatVectorsFormat.fieldsWriter(state); + if (supported()) { + log.info("cuVS is supported so using the Lucene99AcceleratedHNSWVectorsWriter"); + return new Lucene99AcceleratedHNSWVectorsWriter( + state, cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, resources, flatWriter); + } else { + log.warning( + "GPU based indexing not supported, falling back to using the Lucene99HnswVectorsWriter"); + // TODO: Make num merge workers configurable. + return new Lucene99HnswVectorsWriter( + state, maxConn, beamWidth, flatWriter, DEFAULT_NUM_MERGE_WORKER, null); + } + } + + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state)); + } + + @Override + public int getMaxDimensions(String fieldName) { + return maxDimensions; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(this.getClass().getSimpleName()); + sb.append("(cuvsWriterThreads=").append(cuvsWriterThreads); + sb.append("intGraphDegree=").append(intGraphDegree); + sb.append("graphDegree=").append(graphDegree); + sb.append("hnswLayers=").append(hnswLayers); + sb.append("resources=").append(resources); + sb.append(")"); + return sb.toString(); + } + + public static CuVSResources getResources() { + return resources; + } + + public static void setResources(CuVSResources resources) { + Lucene99AcceleratedHNSWVectorsFormat.resources = resources; + } + + /** Tells whether the platform supports cuVS. */ + public static boolean supported() { + return resources != null; + } + + public static void checkSupported() { + if (!supported()) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java new file mode 100644 index 0000000000..e15b743e52 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -0,0 +1,702 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_CODEC_NAME; +import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_EXT; +import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_EXT; +import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_NAME; +import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; +import static org.apache.lucene.index.VectorEncoding.FLOAT32; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; + +import com.nvidia.cuvs.CagraIndex; +import com.nvidia.cuvs.CagraIndexParams; +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.CuVSMatrix; +import com.nvidia.cuvs.CuVSResources; +import com.nvidia.cuvs.RowView; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Random; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.logging.Logger; +import java.util.stream.IntStream; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Sorter; +import org.apache.lucene.index.Sorter.DocMap; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.internal.hppc.IntObjectHashMap; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.hnsw.HnswGraph; +import org.apache.lucene.util.hnsw.HnswGraph.NodesIterator; +import org.apache.lucene.util.hnsw.NeighborArray; +import org.apache.lucene.util.packed.DirectMonotonicWriter; + +/** + * KnnVectorsWriter for CuVS, responsible for merge and flush of vectors into + * GPU + */ +public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { + + private static final long SHALLOW_RAM_BYTES_USED = + shallowSizeOfInstance(Lucene99AcceleratedHNSWVectorsWriter.class); + + @SuppressWarnings("unused") + private static final Logger log = + Logger.getLogger(Lucene99AcceleratedHNSWVectorsWriter.class.getName()); + + /** The name of the CUVS component for the info-stream * */ + private static final String CUVS_COMPONENT = "CUVS"; + + private final int cuvsWriterThreads; + private final int intGraphDegree; + private final int graphDegree; + private final int hnswLayers; // Number of layers to create in CAGRA->HNSW conversion + private final CuVSResources resources; + private final FlatVectorsWriter flatVectorsWriter; // for writing the raw vectors + private final List fields = new ArrayList<>(); + private final InfoStream infoStream; + private IndexOutput cuvsIndex = null; + private IndexOutput hnswMeta = null, hnswVectorIndex = null; + private boolean finished; + private String vemFileName; + private String vexFileName; + + public Lucene99AcceleratedHNSWVectorsWriter( + SegmentWriteState state, + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + CuVSResources resources, + FlatVectorsWriter flatVectorsWriter) + throws IOException { + super(); + this.cuvsWriterThreads = cuvsWriterThreads; + this.intGraphDegree = intGraphDegree; + this.graphDegree = graphDegree; + this.hnswLayers = hnswLayers; + this.resources = resources; + this.flatVectorsWriter = flatVectorsWriter; + this.infoStream = state.infoStream; + + vemFileName = + IndexFileNames.segmentFileName( + state.segmentInfo.name, state.segmentSuffix, HNSW_META_CODEC_EXT); + + vexFileName = + IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, HNSW_INDEX_EXT); + + boolean success = false; + try { + + hnswMeta = state.directory.createOutput(vemFileName, state.context); + hnswVectorIndex = state.directory.createOutput(vexFileName, state.context); + + CodecUtil.writeIndexHeader( + hnswMeta, + HNSW_META_CODEC_NAME, + Lucene99HnswVectorsFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix); + CodecUtil.writeIndexHeader( + hnswVectorIndex, + HNSW_INDEX_CODEC_NAME, + Lucene99HnswVectorsFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix); + + success = true; + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + @Override + public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { + var encoding = fieldInfo.getVectorEncoding(); + if (encoding != FLOAT32) { + throw new IllegalArgumentException("expected float32, got:" + encoding); + } + var writer = Objects.requireNonNull(flatVectorsWriter.addField(fieldInfo)); + @SuppressWarnings("unchecked") + var flatWriter = (FlatFieldVectorsWriter) writer; + var cuvsFieldWriter = new GPUFieldWriter(fieldInfo, flatWriter); + fields.add(cuvsFieldWriter); + return writer; + } + + static String indexMsg(int size, int... args) { + StringBuilder sb = new StringBuilder("cagra index params"); + sb.append(": size=").append(size); + sb.append(", intGraphDegree=").append(args[0]); + sb.append(", actualIntGraphDegree=").append(args[1]); + sb.append(", graphDegree=").append(args[2]); + sb.append(", actualGraphDegree=").append(args[3]); + return sb.toString(); + } + + private CagraIndexParams cagraIndexParams() { + return new CagraIndexParams.Builder() + .withNumWriterThreads(cuvsWriterThreads) + .withIntermediateGraphDegree(intGraphDegree) + .withGraphDegree(graphDegree) + .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) + .build(); + } + + private void info(String msg) { + if (infoStream.isEnabled(CUVS_COMPONENT)) { + infoStream.message(CUVS_COMPONENT, msg); + } + } + + private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOException { + + if (vectors.size() == 0) { + writeEmpty(fieldInfo); + return; + } + + try { + CuVSMatrix dataset = Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension()); + + if (dataset.size() < 2) { + throw new IllegalArgumentException(dataset.size() + " vectors, less than min [2] required"); + } + + long startTime = System.nanoTime(); + CagraIndexParams params = cagraIndexParams(); + CagraIndex cagraIndex = + CagraIndex.newBuilder(resources).withDataset(dataset).withIndexParams(params).build(); + + // Get the adjacency list from CAGRA index + CuVSMatrix adjacencyListMatrix = cagraIndex.getGraph(); + + int size = (int) dataset.size(); + int dimensions = fieldInfo.getVectorDimension(); + + // Create multi-layer HNSW graph from CAGRA + GPUBuiltHnswGraph hnswGraph = + createMultiLayerHnswGraph( + fieldInfo, size, dimensions, adjacencyListMatrix, vectors, hnswLayers); + + long vectorIndexOffset = hnswVectorIndex.getFilePointer(); + + // Write the graph to the vector index + int[][] graphLevelNodeOffsets = writeGraph(hnswGraph, hnswVectorIndex); + + long vectorIndexLength = hnswVectorIndex.getFilePointer() - vectorIndexOffset; + + // Write metadata + writeMeta( + hnswVectorIndex, + hnswMeta, + fieldInfo, + vectorIndexOffset, + vectorIndexLength, + size, + hnswGraph, + graphLevelNodeOffsets); + + long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - startTime); + info("HNSW graph created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); + + cagraIndex.close(); + + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + + /** + * Creates a multi-layer HNSW graph with dynamic number of layers. + * M = cagraGraphDegree/2 + * Each layer contains 1/M nodes from the previous layer + * Creates layers until the highest layer has ≤ M nodes + */ + private GPUBuiltHnswGraph createMultiLayerHnswGraph( + FieldInfo fieldInfo, + int size, + int dimensions, + CuVSMatrix adjacencyListMatrix, + List vectors, + int hnswLayers) + throws Throwable { + + // Calculate M as cagraGraphDegree/2 + int M = graphDegree / 2; + + // Store all layers data + List layerNodes = new ArrayList<>(); + List layerAdjacencies = new ArrayList<>(); + + // Layer 0: Use full CAGRA adjacency list + layerNodes.add(null); // Layer 0 contains all nodes, so we don't need to store node list + layerAdjacencies.add(adjacencyListMatrix); + + int currentLayerSize = size; + int layerIndex = 1; + Random random = new Random(); + + while (layerIndex < hnswLayers && currentLayerSize > 1) { + // Calculate size for next layer (1/M of current layer) + int nextLayerSize = Math.max(2, currentLayerSize / M); + // Select nodes for this layer + SortedSet selectedNodesSet = new TreeSet<>(); + + if (layerIndex == 1) { + // Select from all nodes (Layer 0) + while (selectedNodesSet.size() < nextLayerSize) { + selectedNodesSet.add(random.nextInt(size)); + } + } else { + // Select from previous layer nodes + int[] prevLayerNodes = layerNodes.get(layerNodes.size() - 1); + while (selectedNodesSet.size() < nextLayerSize) { + int idx = random.nextInt(prevLayerNodes.length); + selectedNodesSet.add(prevLayerNodes[idx]); + } + } + + // Convert to sorted array + int[] selectedNodes = + selectedNodesSet.stream().mapToInt(Integer::intValue).sorted().toArray(); + + layerNodes.add(selectedNodes); + + // Extract vectors for selected nodes + float[][] selectedVectors = new float[nextLayerSize][]; + for (int i = 0; i < nextLayerSize; i++) { + selectedVectors[i] = vectors.get(selectedNodes[i]); + } + + // Build CAGRA graph for this layer + layerAdjacencies.add(buildCagraGraphForSubset(selectedVectors, selectedNodes)); + + // Update for next iteration + currentLayerSize = nextLayerSize; + layerIndex++; + + // Use different seed for each layer + random = new Random(new Random().nextLong()); + } + + // Create the multi-layer graph with all layers + return new GPUBuiltHnswGraph(size, dimensions, layerNodes, layerAdjacencies); + } + + /** + * Builds a CAGRA graph for a subset of vectors + */ + private CuVSMatrix buildCagraGraphForSubset(float[][] vectors, int[] selectedNodes) + throws Throwable { + // Create CuVSMatrix from the subset vectors + CuVSMatrix subsetDataset = CuVSMatrix.ofArray(vectors); + + // Build CAGRA index for the subset + CagraIndexParams params = cagraIndexParams(); + CagraIndex subsetIndex = + CagraIndex.newBuilder(resources).withDataset(subsetDataset).withIndexParams(params).build(); + + // Get adjacency list from subset CAGRA index + CuVSMatrix cagraGraph = subsetIndex.getGraph(); + + long numNodes = cagraGraph.size(); + long degree = cagraGraph.columns(); + + // Create a re-mapped adjacency list + int[][] remappedAdjacency = new int[(int) numNodes][(int) degree]; + + for (int i = 0; i < numNodes; i++) { + RowView rv = cagraGraph.getRow(i); + for (int j = 0; j < degree && j < rv.size(); j++) { + int subsetIndex1 = rv.getAsInt(j); + // Map subset index to original node ID + if (subsetIndex1 >= 0 && subsetIndex1 < selectedNodes.length) { + remappedAdjacency[i][j] = selectedNodes[subsetIndex1]; + } else { + // Invalid index, use self-reference + remappedAdjacency[i][j] = selectedNodes[i]; + } + } + } + + subsetIndex.close(); + return CuVSMatrix.ofArray(remappedAdjacency); + } + + private void writeMeta( + IndexOutput vectorIndex, + IndexOutput meta, + FieldInfo field, + long vectorIndexOffset, + long vectorIndexLength, + int count, + HnswGraph graph, + int[][] graphLevelNodeOffsets) + throws IOException { + + meta.writeInt(field.number); + meta.writeInt(field.getVectorEncoding().ordinal()); + meta.writeInt(distFuncToOrd(field.getVectorSimilarityFunction())); + meta.writeVLong(vectorIndexOffset); + meta.writeVLong(vectorIndexLength); + meta.writeVInt(field.getVectorDimension()); + meta.writeInt(count); + meta.writeVInt(graphDegree / 2); // M = cagraGraphDegree/2 + + // write graph nodes on each level + if (graph == null) { + meta.writeVInt(0); + } else { + meta.writeVInt(graph.numLevels()); + long valueCount = 0; + for (int level = 0; level < graph.numLevels(); level++) { + NodesIterator nodesOnLevel = graph.getNodesOnLevel(level); + valueCount += nodesOnLevel.size(); + if (level > 0) { + int[] nol = new int[nodesOnLevel.size()]; + int numberConsumed = nodesOnLevel.consume(nol); + Arrays.sort(nol); + assert numberConsumed == nodesOnLevel.size(); + meta.writeVInt(nol.length); // number of nodes on a level + for (int i = nodesOnLevel.size() - 1; i > 0; --i) { + nol[i] -= nol[i - 1]; + } + for (int n : nol) { + meta.writeVInt(n); + } + } else { + assert nodesOnLevel.size() == count : "Level 0 expects to have all nodes"; + } + } + + long start = vectorIndex.getFilePointer(); + meta.writeLong(start); + meta.writeVInt(16); // DIRECT_MONOTONIC_BLOCK_SHIFT); + + final DirectMonotonicWriter memoryOffsetsWriter = + DirectMonotonicWriter.getInstance(meta, vectorIndex, valueCount, 16); + long cumulativeOffsetSum = 0; + for (int[] levelOffsets : graphLevelNodeOffsets) { + for (int v : levelOffsets) { + memoryOffsetsWriter.add(cumulativeOffsetSum); + cumulativeOffsetSum += v; + } + } + + memoryOffsetsWriter.finish(); + + meta.writeLong(vectorIndex.getFilePointer() - start); + } + } + + private int[][] writeGraph(GPUBuiltHnswGraph graph, IndexOutput vectorIndex) throws IOException { + // write vectors' neighbors on each level into the vectorIndex file + int countOnLevel0 = graph.size(); + int[][] offsets = new int[graph.numLevels()][]; + int[] scratch = new int[graph.maxConn() * 2]; + for (int level = 0; level < graph.numLevels(); level++) { + int[] sortedNodes = NodesIterator.getSortedNodes(graph.getNodesOnLevel(level)); + offsets[level] = new int[sortedNodes.length]; + int nodeOffsetId = 0; + + for (int node : sortedNodes) { + // Get node neighbors + NeighborArray neighbors = graph.getNeighbors(level, node); + // Get the size of the neighbor array + int size = neighbors.size(); + // Write size in VInt as the neighbors list is typically small + long offsetStart = vectorIndex.getFilePointer(); + // Get neighbors + int[] nnodes = neighbors.nodes(); + // Sort them + Arrays.sort(nnodes, 0, size); + // Now that we have sorted, do delta encoding to minimize the required bits to store the + // information + int actualSize = 0; + if (size > 0) { + scratch[0] = nnodes[0]; + actualSize = 1; + } + // De-duplication + for (int i = 1; i < size; i++) { + assert nnodes[i] < countOnLevel0 : "node too large: " + nnodes[i] + ">=" + countOnLevel0; + // Sorting step helps here + if (nnodes[i - 1] == nnodes[i]) { + continue; + } + scratch[actualSize++] = nnodes[i] - nnodes[i - 1]; + } + // Write the size after duplicates are removed + vectorIndex.writeVInt(actualSize); + // Write de-duplicated neighbors + for (int i = 0; i < actualSize; i++) { + vectorIndex.writeVInt(scratch[i]); + } + offsets[level][nodeOffsetId++] = + Math.toIntExact(vectorIndex.getFilePointer() - offsetStart); + } + } + // Return offsets (information written while writing the meta info) + return offsets; + } + + @Override + public void flush(int maxDoc, DocMap sortMap) throws IOException { + flatVectorsWriter.flush(maxDoc, sortMap); + for (var field : fields) { + if (sortMap == null) { + writeField(field); + } else { + writeSortingField(field, sortMap); + } + } + } + + private void writeField(GPUFieldWriter fieldData) throws IOException { + writeFieldInternal(fieldData.fieldInfo(), fieldData.getVectors()); + } + + private void writeSortingField(GPUFieldWriter fieldData, Sorter.DocMap sortMap) + throws IOException { + + DocsWithFieldSet oldDocsWithFieldSet = fieldData.getDocsWithFieldSet(); + final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; // new ord to old ord + mapOldOrdToNewOrd(oldDocsWithFieldSet, sortMap, null, new2OldOrd, null); + + List sortedVectors = new ArrayList(); + for (int i = 0; i < fieldData.getVectors().size(); i++) { + sortedVectors.add(fieldData.getVectors().get(new2OldOrd[i])); + } + + writeFieldInternal(fieldData.fieldInfo(), sortedVectors); + } + + private void writeEmpty(FieldInfo fieldInfo) throws IOException { + writeMeta(null, hnswMeta, fieldInfo, 0, 0, 0, null, null); + } + + static int distFuncToOrd(VectorSimilarityFunction func) { + for (int i = 0; i < SIMILARITY_FUNCTIONS.size(); i++) { + if (SIMILARITY_FUNCTIONS.get(i).equals(func)) { + return (byte) i; + } + } + throw new IllegalArgumentException("invalid distance function: " + func); + } + + private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + try { + + List cagraIndexes = new ArrayList<>(); + // We need this count so that the merged segment's meta information has the vector count. + int totalVectorCount = 0; + + for (int i = 0; i < mergeState.knnVectorsReaders.length; i++) { + KnnVectorsReader knnReader = mergeState.knnVectorsReaders[i]; + // Access the CAGRA index for this field from the reader + + if (knnReader != null) { + if (knnReader instanceof CuVS2510GPUVectorsReader cvr) { + if (cvr != null) { + totalVectorCount += cvr.getFieldEntries().get(fieldInfo.number).count(); + CagraIndex cagraIndex = getCagraIndexFromReader(cvr, fieldInfo.name); + if (cagraIndex != null) { + cagraIndexes.add(cagraIndex); + } + } + } else { + // This should never happen + throw new RuntimeException( + "Reader is not of CuVSVectorsReader type. Instead it is: " + knnReader.getClass()); + } + } + } + assert cagraIndexes.size() > 1; + + CagraIndex mergedIndex = + CagraIndex.merge(cagraIndexes.toArray(new CagraIndex[cagraIndexes.size()])); + writeMergedCagraIndex(fieldInfo, mergedIndex, totalVectorCount); + info("Successfully merged " + cagraIndexes.size() + " CAGRA indexes using native merge API"); + + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + + /** + * Creates List from merged vectors + * */ + private List createListFromMergedVectors(FloatVectorValues mergedVectorValues) + throws IOException { + List vectors = new ArrayList(); + KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); + for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { + float[] vector = mergedVectorValues.vectorValue(iter.index()); + vectors.add(vector); + } + return vectors; + } + + /** + * Fallback method that rebuilds indexes from merged vectors. + * Used when native CAGRA merge() is not possible. Also used + * when non-CAGRA index types are used (for e.g. Brute Force index). + */ + private void vectorBasedMerge(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + if (fieldInfo.getVectorEncoding() != FLOAT32) { + throw new AssertionError("Only Float32 supported"); + } + try { + List dataset = + createListFromMergedVectors( + KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState)); + writeFieldInternal(fieldInfo, dataset); + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + + /** + * Extracts the CAGRA index for a specific field from a CuVSVectorsReader. + */ + private CagraIndex getCagraIndexFromReader(CuVS2510GPUVectorsReader reader, String fieldName) { + try { + IntObjectHashMap cuvsIndices = reader.getCuvsIndexes(); + FieldInfos fieldInfos = reader.getFieldInfos(); + FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldName); + + if (fieldInfo != null) { + GPUIndex cuvsIndex = cuvsIndices.get(fieldInfo.number); + if (cuvsIndex != null) { + return cuvsIndex.getCagraIndex(); + } + } + } catch (Exception e) { + e.printStackTrace(); + info("Failed to extract CAGRA index for field " + fieldName + ": " + e.getMessage()); + } + return null; + } + + /** + * Writes a pre-built merged CAGRA index to the output. + */ + private void writeMergedCagraIndex(FieldInfo fieldInfo, CagraIndex mergedIndex, int vectorCount) + throws IOException { + try { + var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); + + // Serialize the merged index + Path tmpFile = Files.createTempFile(resources.tempDirectory(), "mergedindex", "cag"); + mergedIndex.serialize(cagraIndexOutputStream, tmpFile); + + // TODO: Path to writeFieldInternal missing. Fix this. + + // Clean up the merged index + mergedIndex.close(); + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + + @Override + public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + flatVectorsWriter.mergeOneField(fieldInfo, mergeState); + + // Since CAGRA merge does not support merging of indexes with purging of deletes, + // we fallback to vector-based re-indexing. Issue: + // https://github.com/rapidsai/cuvs/issues/1253 + boolean hasDeletions = + IntStream.range(0, mergeState.liveDocs.length) + .anyMatch( + i -> + mergeState.liveDocs[i] == null + || IntStream.range(0, mergeState.maxDocs[i]) + .anyMatch(j -> !mergeState.liveDocs[i].get(j))); + + if (mergeState.knnVectorsReaders.length > 1 && !hasDeletions) { + mergeCagraIndexes(fieldInfo, mergeState); + } else { + // CAGRA's merge API does not handle the trivial case of merging 1 index. + vectorBasedMerge(fieldInfo, mergeState); + } + } + + @Override + public void finish() throws IOException { + if (finished) { + throw new IllegalStateException("already finished"); + } + finished = true; + flatVectorsWriter.finish(); + + if (cuvsIndex != null) { + CodecUtil.writeFooter(cuvsIndex); + } + + if (hnswMeta != null) { + // write end of fields marker + hnswMeta.writeInt(-1); + CodecUtil.writeFooter(hnswMeta); + } + if (hnswVectorIndex != null) { + CodecUtil.writeFooter(hnswVectorIndex); + } + } + + @Override + public void close() throws IOException { + IOUtils.close(cuvsIndex, hnswMeta, hnswVectorIndex, flatVectorsWriter); + } + + @Override + public long ramBytesUsed() { + long total = SHALLOW_RAM_BYTES_USED; + for (var field : fields) { + total += field.ramBytesUsed(); + } + return total; + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java index 8d1d4bd015..dfb9aa0c49 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java @@ -16,11 +16,16 @@ package com.nvidia.cuvs.lucene; import com.nvidia.cuvs.CuVSMatrix; +import com.nvidia.cuvs.CuVSResources; import java.io.IOException; +import java.time.Duration; import java.util.List; +import java.util.logging.Logger; public class Utils { + static final Logger log = Logger.getLogger(Utils.class.getName()); + static void handleThrowable(Throwable t) throws IOException { switch (t) { case IOException ioe -> throw ioe; @@ -45,4 +50,34 @@ static CuVSMatrix createFloatMatrix(List data, int dimensions) { float[][] vectors = data.toArray(new float[0][]); return CuVSMatrix.ofArray(vectors); } + + static long nanosToMillis(long nanos) { + return Duration.ofNanos(nanos).toMillis(); + } + + static CuVSResources cuVSResourcesOrNull() { + try { + System.loadLibrary("cudart"); + } catch (UnsatisfiedLinkError e) { + log.warning("Could not load CUDA runtime library: " + e.getMessage()); + } + try { + return CuVSResources.create(); + } catch (UnsupportedOperationException uoe) { + log.warning("cuVS is not supported on this platform or java version: " + uoe.getMessage()); + } catch (Throwable t) { + if (t instanceof ExceptionInInitializerError ex) { + t = ex.getCause(); + } + log.warning("Exception occurred during creation of cuVS resources. " + t); + } + return null; + } + + static void handleThrowableWithIgnore(Throwable t, String msg) throws IOException { + if (t.getMessage().contains(msg)) { + return; + } + handleThrowable(t); + } } diff --git a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec new file mode 100644 index 0000000000..9865a33d13 --- /dev/null +++ b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -0,0 +1,2 @@ +com.nvidia.cuvs.lucene.Lucene101AcceleratedHNSWCodec +com.nvidia.cuvs.lucene.CuVS2510GPUSearchCodec diff --git a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index 6e486ee194..747a157595 100644 --- a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -15,4 +15,5 @@ org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat -com.nvidia.cuvs.lucene.CuVSVectorsFormat +com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat +com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java new file mode 100644 index 0000000000..4f5f7aba6a --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java @@ -0,0 +1,167 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Random; +import java.util.UUID; +import java.util.logging.Logger; +import org.apache.commons.io.FileUtils; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestCagraToHnswSerializationAndSearch extends LuceneTestCase { + + private static Logger log = + Logger.getLogger(TestCagraToHnswSerializationAndSearch.class.getName()); + private static Random random; + private static Path indexDirPath; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + // Fixed seed so that we can validate against the same result. + random = new Random(222); + indexDirPath = Paths.get(UUID.randomUUID().toString()); + } + + @Test + public void testCagraToHnswSerializationAndSearch() throws IOException { + Codec codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); + IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); + + final int COMMIT_FREQ = 2000; + final String ID_FIELD = "id"; + final String VECTOR_FIELD = "vector_field"; + + int numDocs = 2000; + int dimension = 32; + int topK = 5; + int count = COMMIT_FREQ; + float[][] dataset = generateDataset(random, numDocs, dimension); + + // Indexing + try (Directory indexDirectory = FSDirectory.open(indexDirPath); + IndexWriter indexWriter = new IndexWriter(indexDirectory, config)) { + for (int i = 0; i < numDocs; i++) { + Document document = new Document(); + document.add(new StringField(ID_FIELD, Integer.toString(i), Field.Store.YES)); + document.add(new KnnFloatVectorField(VECTOR_FIELD, dataset[i], EUCLIDEAN)); + indexWriter.addDocument(document); + count -= 1; + if (count == 0) { + indexWriter.commit(); + count = COMMIT_FREQ; + } + } + } + + // Searching + try (Directory indexDirectory = FSDirectory.open(indexDirPath)) { + try (DirectoryReader reader = DirectoryReader.open(indexDirectory)) { + log.info("Successfully opened index"); + + int vectorCount = 0; + for (LeafReaderContext leafReaderContext : reader.leaves()) { + LeafReader leafReader = leafReaderContext.reader(); + FloatVectorValues knnValues = leafReader.getFloatVectorValues(VECTOR_FIELD); + assertNotNull(knnValues); + log.info( + VECTOR_FIELD + + " field: " + + knnValues.size() + + " vectors, " + + knnValues.dimension() + + " dimensions"); + vectorCount += knnValues.size(); + assertTrue("Vector dimension mismatch", knnValues.dimension() == dimension); + } + assertTrue("Dataset size mismatch", vectorCount == numDocs); + + log.info("Testing vector search queries..."); + IndexSearcher searcher = new IndexSearcher(reader); + + float[] queryVector = generateDataset(random, 1, dimension)[0]; + log.info("Query vector: " + Arrays.toString(queryVector)); + + KnnFloatVectorQuery query = new KnnFloatVectorQuery(VECTOR_FIELD, queryVector, topK); + TopDocs results = searcher.search(query, topK); + + log.info("Search results (" + results.totalHits + " total hits):"); + Integer[] expected = new Integer[] {1869, 1803, 1302, 59, 1497, 108, 1411, 351, 1982}; + HashSet expectedIds = new HashSet(Arrays.asList(expected)); + + for (int i = 0; i < results.scoreDocs.length; i++) { + ScoreDoc scoreDoc = results.scoreDocs[i]; + Document doc = searcher.storedFields().document(scoreDoc.doc); + String id = doc.get(ID_FIELD); + log.info( + " Rank " + + (i + 1) + + ": doc " + + scoreDoc.doc + + " (id=" + + id + + "), score=" + + scoreDoc.score); + assertTrue( + "Id: " + id + " expected but not found", expectedIds.contains(Integer.valueOf(id))); + } + assertTrue("TopK results not returned", results.scoreDocs.length == topK); + + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + @AfterClass + public static void afterClass() throws Exception { + File indexDirPathFile = indexDirPath.toFile(); + if (indexDirPathFile.exists() && indexDirPathFile.isDirectory()) { + FileUtils.deleteDirectory(indexDirPathFile); + } + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java new file mode 100644 index 0000000000..792aca9654 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2025, NVIDIA CORPORATION. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.Utils.cuVSResourcesOrNull; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Random; +import java.util.UUID; +import java.util.logging.Logger; +import org.apache.commons.io.FileUtils; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestCagraToHnswSerializationAndSearchWithFallbackWriter extends LuceneTestCase { + + private static Logger log = + Logger.getLogger(TestCagraToHnswSerializationAndSearchWithFallbackWriter.class.getName()); + + private static Random random; + private static Path indexDirPath; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + // Set resources to null to simulate that cuVS is not supported. + Lucene99AcceleratedHNSWVectorsFormat.setResources(null); + // Fixed seed so that we can validate against the same result. + random = new Random(222); + indexDirPath = Paths.get(UUID.randomUUID().toString()); + } + + @Test + public void testCagraToHnswSerializationAndSearchWithFallbackWriter() throws IOException { + Codec codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); + IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); + + final int COMMIT_FREQ = 2000; + final String ID_FIELD = "id"; + final String VECTOR_FIELD = "vector_field"; + + int numDocs = 2000; + int dimension = 32; + int topK = 5; + int count = COMMIT_FREQ; + float[][] dataset = generateDataset(random, numDocs, dimension); + + // Indexing + try (Directory indexDirectory = FSDirectory.open(indexDirPath); + IndexWriter indexWriter = new IndexWriter(indexDirectory, config)) { + for (int i = 0; i < numDocs; i++) { + Document document = new Document(); + document.add(new StringField(ID_FIELD, Integer.toString(i), Field.Store.YES)); + document.add(new KnnFloatVectorField(VECTOR_FIELD, dataset[i], EUCLIDEAN)); + indexWriter.addDocument(document); + count -= 1; + if (count == 0) { + indexWriter.commit(); + count = COMMIT_FREQ; + } + } + } + + // Searching + try (Directory indexDirectory = FSDirectory.open(indexDirPath)) { + try (DirectoryReader reader = DirectoryReader.open(indexDirectory)) { + log.info("Successfully opened index"); + + int vectorCount = 0; + for (LeafReaderContext leafReaderContext : reader.leaves()) { + LeafReader leafReader = leafReaderContext.reader(); + FloatVectorValues knnValues = leafReader.getFloatVectorValues(VECTOR_FIELD); + assertNotNull(knnValues); + log.info( + VECTOR_FIELD + + " field: " + + knnValues.size() + + " vectors, " + + knnValues.dimension() + + " dimensions"); + vectorCount += knnValues.size(); + assertTrue("Vector dimension mismatch", knnValues.dimension() == dimension); + } + assertTrue("Dataset size mismatch", vectorCount == numDocs); + + log.info("Testing vector search queries..."); + IndexSearcher searcher = new IndexSearcher(reader); + + float[] queryVector = generateDataset(random, 1, dimension)[0]; + log.info("Query vector: " + Arrays.toString(queryVector)); + + KnnFloatVectorQuery query = new KnnFloatVectorQuery(VECTOR_FIELD, queryVector, topK); + TopDocs results = searcher.search(query, topK); + + log.info("Search results (" + results.totalHits + " total hits):"); + Integer[] expected = new Integer[] {1869, 1411, 1497, 351, 554}; + HashSet expectedIds = new HashSet(Arrays.asList(expected)); + + for (int i = 0; i < results.scoreDocs.length; i++) { + ScoreDoc scoreDoc = results.scoreDocs[i]; + Document doc = searcher.storedFields().document(scoreDoc.doc); + String id = doc.get(ID_FIELD); + log.info( + " Rank " + + (i + 1) + + ": doc " + + scoreDoc.doc + + " (id=" + + id + + "), score=" + + scoreDoc.score); + assertTrue( + "Id: " + id + " expected but not found", expectedIds.contains(Integer.valueOf(id))); + } + assertTrue("TopK results not returned", results.scoreDocs.length == topK); + + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + @AfterClass + public static void afterClass() throws Exception { + // Reset resources for other tests to work + Lucene99AcceleratedHNSWVectorsFormat.setResources(cuVSResourcesOrNull()); + File indexDirPathFile = indexDirPath.toFile(); + if (indexDirPathFile.exists() && indexDirPathFile.isDirectory()) { + FileUtils.deleteDirectory(indexDirPathFile); + } + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java index ef17136e8e..1774587f25 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java @@ -51,17 +51,17 @@ import org.junit.BeforeClass; import org.junit.Test; -@SuppressSysoutChecks(bugUrl = "prints info from within cuvs") +@SuppressSysoutChecks(bugUrl = "prints info from within cuVS") public class TestCuVSDeletedDocuments extends LuceneTestCase { protected static Logger log = Logger.getLogger(TestCuVSDeletedDocuments.class.getName()); - static final Codec codec = TestUtil.alwaysKnnVectorsFormat(new CuVSVectorsFormat()); + static final Codec codec = TestUtil.alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat()); private static Random random; @BeforeClass public static void beforeClass() throws Exception { - assumeTrue("cuvs not supported", CuVSVectorsFormat.supported()); + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); random = random(); } @@ -194,7 +194,7 @@ public void testVectorSearchWithMixedDeletedAndMissingVectors() throws IOExcepti // Test filtered search with deletions Query filter = new TermQuery(new Term("category", "A")); Query filteredQuery = - new CuVSKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); + new GPUKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; for (ScoreDoc hit : filteredHits) { diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java index e27568b5e3..bba34bfaf3 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java @@ -48,12 +48,12 @@ import org.junit.BeforeClass; import org.junit.Test; -@SuppressSysoutChecks(bugUrl = "prints info from within cuvs") +@SuppressSysoutChecks(bugUrl = "prints info from within cuVS") public class TestCuVSGaps extends LuceneTestCase { protected static Logger log = Logger.getLogger(TestCuVSGaps.class.getName()); - static final Codec codec = TestUtil.alwaysKnnVectorsFormat(new CuVSVectorsFormat()); + static final Codec codec = TestUtil.alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat()); static IndexSearcher searcher; static IndexReader reader; static Directory directory; @@ -70,7 +70,7 @@ public class TestCuVSGaps extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { - assertTrue("cuvs not supported", CuVSVectorsFormat.supported()); + assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); directory = newDirectory(); random = random(); @@ -120,7 +120,7 @@ public static void afterClass() throws Exception { @Test public void testVectorSearchWithAlternatingDocuments() throws IOException { - assertTrue("cuvs not supported", CuVSVectorsFormat.supported()); + assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); // Use the first vector (from document 0) as query float[] queryVector = dataset[0]; @@ -153,7 +153,7 @@ public void testVectorSearchWithAlternatingDocuments() throws IOException { @Test public void testVectorSearchWithFilterAndAlternatingDocuments() throws IOException { - assumeTrue("cuvs not supported", CuVSVectorsFormat.supported()); + assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); // Use the first vector (from document 0) as query float[] queryVector = dataset[0]; @@ -163,7 +163,7 @@ public void testVectorSearchWithFilterAndAlternatingDocuments() throws IOExcepti // This should further restrict our results to even numbers 0, 2, 4, 6, 8 Query filter = new TermQuery(new Term("id", "8")); // Only match document 8 - Query filteredQuery = new CuVSKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); + Query filteredQuery = new GPUKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; // Should only get document 8 (the only one that matches the filter and has a vector) diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java index 8802373466..add0fe2043 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java @@ -51,12 +51,12 @@ import org.junit.BeforeClass; import org.junit.Test; -@SuppressSysoutChecks(bugUrl = "prints info from within cuvs") +@SuppressSysoutChecks(bugUrl = "prints info from within cuVS") public class TestCuVSRandomizedVectorSearch extends LuceneTestCase { protected static Logger log = Logger.getLogger(TestCuVSRandomizedVectorSearch.class.getName()); - static final Codec codec = TestUtil.alwaysKnnVectorsFormat(new CuVSVectorsFormat()); + static final Codec codec = TestUtil.alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat()); static IndexSearcher searcher; static IndexReader reader; static Directory directory; @@ -69,7 +69,7 @@ public class TestCuVSRandomizedVectorSearch extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { - assertTrue("cuvs not supported", CuVSVectorsFormat.supported()); + assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); directory = newDirectory(); RandomIndexWriter writer = @@ -184,7 +184,7 @@ private static List> generateExpectedResults( @Test public void testVectorSearchWithFilter() throws IOException { - assertTrue("cuvs not supported", CuVSVectorsFormat.supported()); + assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); Random random = random(); int topK = Math.min(random.nextInt(TOP_K_LIMIT) + 1, dataset.length); @@ -206,7 +206,7 @@ public void testVectorSearchWithFilter() throws IOException { Query filter = new TermQuery(new Term("id", targetDocId)); // Test the new constructor with filter - Query filteredQuery = new CuVSKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); + Query filteredQuery = new GPUKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java index 69aeb29082..cf78b05171 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java @@ -41,12 +41,12 @@ public class TestCuVSVectorsFormat extends BaseKnnVectorsFormatTestCase { @BeforeClass public static void beforeClass() { - assertTrue("cuvs is not supported", CuVSVectorsFormat.supported()); + assumeTrue("cuVS is not supported", CuVS2510GPUVectorsFormat.supported()); } @Override protected Codec getCodec() { - return TestUtil.alwaysKnnVectorsFormat(new CuVSVectorsFormat()); + return TestUtil.alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat()); } public void testMergeTwoSegsWithASingleDocPerSeg() throws Exception { diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java index a424492800..0ad4861d57 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java @@ -17,7 +17,7 @@ import static org.apache.lucene.tests.util.TestUtil.alwaysKnnVectorsFormat; -import com.nvidia.cuvs.lucene.CuVSVectorsWriter.IndexType; +import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -70,7 +70,7 @@ public class TestMerge extends LuceneTestCase { @BeforeClass public static void beforeClass() { - assertTrue("cuVS is not supported", CuVSVectorsFormat.supported()); + assumeTrue("cuVS is not supported", CuVS2510GPUVectorsFormat.supported()); } private Directory directory; @@ -128,7 +128,7 @@ public void testMergeManyDocumentsMultipleSegments() throws IOException { IndexWriterConfig config = new IndexWriterConfig() - .setCodec(alwaysKnnVectorsFormat(new CuVSVectorsFormat())) + .setCodec(alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat())) .setMaxBufferedDocs(maxBufferedDocs) // Randomized buffer size .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); @@ -252,7 +252,7 @@ public void testMergeWithIndexSorting() throws IOException { IndexWriterConfig config = new IndexWriterConfig() - .setCodec(alwaysKnnVectorsFormat(new CuVSVectorsFormat())) + .setCodec(alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat())) .setIndexSort(indexSort) // This automatically enables sorting during merges .setMergePolicy(mergePolicy) .setMaxBufferedDocs(maxBufferedDocs) @@ -458,7 +458,7 @@ public void testMergeWithMissingVectors() throws IOException { IndexWriterConfig config = new IndexWriterConfig() - .setCodec(alwaysKnnVectorsFormat(new CuVSVectorsFormat())) + .setCodec(alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat())) .setMaxBufferedDocs(maxBufferedDocs) .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); @@ -597,7 +597,7 @@ public void testMergeWithDeletions() throws IOException { IndexWriterConfig config = new IndexWriterConfig() - .setCodec(alwaysKnnVectorsFormat(new CuVSVectorsFormat())) + .setCodec(alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat())) .setMaxBufferedDocs(maxBufferedDocs) .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); @@ -730,11 +730,12 @@ public void testMergeBruteForceIndex() throws IOException { + vectorProbability); // Configure with brute force index type - CuVSVectorsFormat bruteForceFormat = - new CuVSVectorsFormat( + CuVS2510GPUVectorsFormat bruteForceFormat = + new CuVS2510GPUVectorsFormat( 32, // writer threads 128, // intermediate graph degree 64, // graph degree + 1, IndexType.BRUTE_FORCE); // Use brute force index IndexWriterConfig config = @@ -881,11 +882,12 @@ public void testMergeCagraAndBruteForceIndex() throws IOException { + vectorProbability); // Configure with CAGRA + brute force combined index type - CuVSVectorsFormat combinedFormat = - new CuVSVectorsFormat( + CuVS2510GPUVectorsFormat combinedFormat = + new CuVS2510GPUVectorsFormat( 32, // writer threads 128, // intermediate graph degree 64, // graph degree + 1, IndexType.CAGRA_AND_BRUTE_FORCE); // Use combined CAGRA + brute force IndexWriterConfig config = @@ -1056,7 +1058,7 @@ public void testLargeScaleMerge() throws IOException { IndexWriterConfig config = new IndexWriterConfig() - .setCodec(alwaysKnnVectorsFormat(new CuVSVectorsFormat())) + .setCodec(alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat())) .setMaxBufferedDocs(maxBufferedDocs) .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); From 1bed6b278da2c76812467c64df57d8550e9ff15c Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Wed, 1 Oct 2025 16:15:36 -0400 Subject: [PATCH 022/105] Remove hnswlib index (NVIDIA/cuvs-lucene#24) Fixes NVIDIA/cuvs-lucene#7 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/24 --- .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 2 - .../cuvs/lucene/CuVS2510GPUVectorsFormat.java | 20 +---- .../cuvs/lucene/CuVS2510GPUVectorsReader.java | 25 +----- .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 78 +++---------------- .../java/com/nvidia/cuvs/lucene/GPUIndex.java | 14 +--- .../com/nvidia/cuvs/lucene/TestMerge.java | 2 - 6 files changed, 16 insertions(+), 125 deletions(-) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index 4027d3aba6..1e811e9728 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -36,7 +36,6 @@ public class CuVS2510GPUSearchCodec extends FilterCodec { private static final int DEFAULT_CUVS_WRITER_THREADS = 1; private static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; private static final int DEFAULT_GRAPH_DEGREE = 64; - private static final int DEFAULT_HNSW_LAYERS = 1; private static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; private KnnVectorsFormat format; @@ -53,7 +52,6 @@ public CuVS2510GPUSearchCodec(String name, Codec delegate) { DEFAULT_CUVS_WRITER_THREADS, DEFAULT_INTERMEDIATE_GRAPH_DEGREE, DEFAULT_GRAPH_DEGREE, - DEFAULT_HNSW_LAYERS, DEFAULT_INDEX_TYPE); setKnnFormat(format); } catch (LibraryException ex) { diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java index bccff5adf1..10213bb647 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java @@ -48,7 +48,6 @@ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; static final int DEFAULT_GRAPH_DEGREE = 64; static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; - static final int HNSW_GRAPH_LAYERS = 1; static CuVSResources resources = cuVSResourcesOrNull(); @@ -60,7 +59,6 @@ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { final int cuvsWriterThreads; final int intGraphDegree; final int graphDegree; - final int hnswLayers; // Number of layers to create in CAGRA->HNSW conversion final CuVS2510GPUVectorsWriter.IndexType indexType; // the index type to build, when writing /** @@ -73,7 +71,6 @@ public CuVS2510GPUVectorsFormat() { DEFAULT_WRITER_THREADS, DEFAULT_INTERMEDIATE_GRAPH_DEGREE, DEFAULT_GRAPH_DEGREE, - HNSW_GRAPH_LAYERS, DEFAULT_INDEX_TYPE); } @@ -83,16 +80,11 @@ public CuVS2510GPUVectorsFormat() { * @throws LibraryException if the native library fails to load */ public CuVS2510GPUVectorsFormat( - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - int hnswLayers, - IndexType indexType) { + int cuvsWriterThreads, int intGraphDegree, int graphDegree, IndexType indexType) { super("CuVS2510GPUVectorsFormat"); this.cuvsWriterThreads = cuvsWriterThreads; this.intGraphDegree = intGraphDegree; this.graphDegree = graphDegree; - this.hnswLayers = hnswLayers; this.indexType = indexType; } @@ -101,14 +93,7 @@ public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOE checkSupported(); var flatWriter = flatVectorsFormat.fieldsWriter(state); return new CuVS2510GPUVectorsWriter( - state, - cuvsWriterThreads, - intGraphDegree, - graphDegree, - hnswLayers, - indexType, - resources, - flatWriter); + state, cuvsWriterThreads, intGraphDegree, graphDegree, indexType, resources, flatWriter); } @Override @@ -128,7 +113,6 @@ public String toString() { sb.append("(cuvsWriterThreads=").append(cuvsWriterThreads); sb.append("intGraphDegree=").append(intGraphDegree); sb.append("graphDegree=").append(graphDegree); - sb.append("hnswLayers=").append(hnswLayers); sb.append("resources=").append(resources); sb.append(")"); return sb.toString(); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java index a9fe61a51b..b90055ab90 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java @@ -28,8 +28,6 @@ import com.nvidia.cuvs.CagraQuery; import com.nvidia.cuvs.CagraSearchParams; import com.nvidia.cuvs.CuVSResources; -import com.nvidia.cuvs.HnswIndex; -import com.nvidia.cuvs.HnswIndexParams; import java.io.IOException; import java.util.Iterator; import java.util.List; @@ -240,7 +238,6 @@ private IntObjectHashMap loadCuVSIndices() throws IOException { private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { CagraIndex cagraIndex = null; BruteForceIndex bruteForceIndex = null; - HnswIndex hnswIndex = null; try { long len = fieldEntry.cagraIndexLength(); @@ -260,20 +257,10 @@ private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { bruteForceIndex = BruteForceIndex.newBuilder(resources).from(in).build(); } } - - len = fieldEntry.hnswIndexLength(); - if (len > 0) { - long off = fieldEntry.hnswIndexOffset(); - try (var slice = cuvsIndexInput.slice("hnsw index", off, len); - var in = new IndexInputInputStream(slice)) { - var params = new HnswIndexParams.Builder().build(); - hnswIndex = HnswIndex.newBuilder(resources).withIndexParams(params).from(in).build(); - } - } } catch (Throwable t) { Utils.handleThrowable(t); } - return new GPUIndex(cagraIndex, bruteForceIndex, hnswIndex); + return new GPUIndex(cagraIndex, bruteForceIndex); } @Override @@ -427,9 +414,7 @@ record FieldEntry( long cagraIndexOffset, long cagraIndexLength, long bruteForceIndexOffset, - long bruteForceIndexLength, - long hnswIndexOffset, - long hnswIndexLength) { + long bruteForceIndexLength) { static FieldEntry readEntry( IndexInput input, @@ -442,8 +427,6 @@ static FieldEntry readEntry( var cagraIndexLength = input.readVLong(); var bruteForceIndexOffset = input.readVLong(); var bruteForceIndexLength = input.readVLong(); - var hnswIndexOffset = input.readVLong(); - var hnswIndexLength = input.readVLong(); return new FieldEntry( vectorEncoding, similarityFunction, @@ -452,9 +435,7 @@ static FieldEntry readEntry( cagraIndexOffset, cagraIndexLength, bruteForceIndexOffset, - bruteForceIndexLength, - hnswIndexOffset, - hnswIndexLength); + bruteForceIndexLength); } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index 69d66db00f..9c3690e064 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -92,7 +92,6 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { private final FlatVectorsWriter flatVectorsWriter; // for writing the raw vectors private final List fields = new ArrayList<>(); private IndexOutput meta = null, cuvsIndex = null; - private IndexOutput hnswMeta = null, hnswVectorIndex = null; private final InfoStream infoStream; private boolean finished; @@ -100,22 +99,18 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { public enum IndexType { /** Builds a Cagra index. */ - CAGRA(true, false, false), + CAGRA(true, false), /** Builds a Brute Force index. */ - BRUTE_FORCE(false, true, false), - - /** Builds an HSNW index - suitable for searching on CPU. */ - HNSW(false, false, true), + BRUTE_FORCE(false, true), /** Builds a Cagra and a Brute Force index. */ - CAGRA_AND_BRUTE_FORCE(true, true, false); - private final boolean cagra, bruteForce, hnsw; + CAGRA_AND_BRUTE_FORCE(true, true); + private final boolean cagra, bruteForce; - IndexType(boolean cagra, boolean bruteForce, boolean hnsw) { + IndexType(boolean cagra, boolean bruteForce) { this.cagra = cagra; this.bruteForce = bruteForce; - this.hnsw = hnsw; } public boolean cagra() { @@ -125,10 +120,6 @@ public boolean cagra() { public boolean bruteForce() { return bruteForce; } - - public boolean hnsw() { - return hnsw; - } } public CuVS2510GPUVectorsWriter( @@ -136,7 +127,6 @@ public CuVS2510GPUVectorsWriter( int cuvsWriterThreads, int intGraphDegree, int graphDegree, - int hnswLayers, IndexType indexType, CuVSResources resources, FlatVectorsWriter flatVectorsWriter) @@ -233,7 +223,6 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro } long cagraIndexOffset, cagraIndexLength = 0L; long bruteForceIndexOffset, bruteForceIndexLength = 0L; - long hnswIndexOffset, hnswIndexLength = 0L; // workaround for the minimum number of vectors for Cagra IndexType indexType = @@ -265,30 +254,13 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro bruteForceIndexLength = cuvsIndex.getFilePointer() - bruteForceIndexOffset; } - hnswIndexOffset = cuvsIndex.getFilePointer(); - if (indexType.hnsw()) { - var hnswIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - if (vectors.size() > MIN_CAGRA_INDEX_SIZE) { - try { - CuVSMatrix dataset = Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension()); - writeHNSWIndex(hnswIndexOutputStream, dataset); - } catch (Throwable t) { - Utils.handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); - } - } - hnswIndexLength = cuvsIndex.getFilePointer() - hnswIndexOffset; - } - - // Only write meta for non-HNSW_LUCENE modes writeMeta( fieldInfo, vectors.size(), cagraIndexOffset, cagraIndexLength, bruteForceIndexOffset, - bruteForceIndexLength, - hnswIndexOffset, - hnswIndexLength); + bruteForceIndexLength); } catch (Throwable t) { Utils.handleThrowable(t); } @@ -323,21 +295,6 @@ private void writeBruteForceIndex(OutputStream os, CuVSMatrix dataset) throws Th index.close(); } - private void writeHNSWIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { - if (dataset.size() < 2) { - throw new IllegalArgumentException(dataset.size() + " vectors, less than min [2] required"); - } - CagraIndexParams indexParams = cagraIndexParams((int) dataset.size()); - long startTime = System.nanoTime(); - CagraIndex index = - CagraIndex.newBuilder(resources).withDataset(dataset).withIndexParams(indexParams).build(); - long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - startTime); - info("HNSW index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); - Path tmpFile = Files.createTempFile("tmpindex", "hnsw"); - index.serializeToHNSW(os, tmpFile); - index.close(); - } - @Override public void flush(int maxDoc, DocMap sortMap) throws IOException { flatVectorsWriter.flush(maxDoc, sortMap); @@ -370,7 +327,7 @@ private void writeSortingField(GPUFieldWriter fieldData, Sorter.DocMap sortMap) } private void writeEmpty(FieldInfo fieldInfo) throws IOException { - writeMeta(fieldInfo, 0, 0L, 0L, 0L, 0L, 0L, 0L); + writeMeta(fieldInfo, 0, 0L, 0L, 0L, 0L); } private void writeMeta( @@ -379,9 +336,7 @@ private void writeMeta( long cagraIndexOffset, long cagraIndexLength, long bruteForceIndexOffset, - long bruteForceIndexLength, - long hnswIndexOffset, - long hnswIndexLength) + long bruteForceIndexLength) throws IOException { meta.writeInt(field.number); meta.writeInt(field.getVectorEncoding().ordinal()); @@ -392,8 +347,6 @@ private void writeMeta( meta.writeVLong(cagraIndexLength); meta.writeVLong(bruteForceIndexOffset); meta.writeVLong(bruteForceIndexLength); - meta.writeVLong(hnswIndexOffset); - meta.writeVLong(hnswIndexLength); } static int distFuncToOrd(VectorSimilarityFunction func) { @@ -524,7 +477,7 @@ private void writeMergedCagraIndex(FieldInfo fieldInfo, CagraIndex mergedIndex, mergedIndex.serialize(cagraIndexOutputStream, tmpFile); long cagraIndexLength = cuvsIndex.getFilePointer() - cagraIndexOffset; - writeMeta(fieldInfo, vectorCount, cagraIndexOffset, cagraIndexLength, 0L, 0L, 0L, 0L); + writeMeta(fieldInfo, vectorCount, cagraIndexOffset, cagraIndexLength, 0L, 0L); // Clean up the merged index mergedIndex.close(); @@ -579,22 +532,11 @@ public void finish() throws IOException { if (cuvsIndex != null) { CodecUtil.writeFooter(cuvsIndex); } - - { - if (hnswMeta != null) { - // write end of fields marker - hnswMeta.writeInt(-1); - CodecUtil.writeFooter(hnswMeta); - } - if (hnswVectorIndex != null) { - CodecUtil.writeFooter(hnswVectorIndex); - } - } } @Override public void close() throws IOException { - IOUtils.close(meta, cuvsIndex, hnswMeta, hnswVectorIndex, flatVectorsWriter); + IOUtils.close(meta, cuvsIndex, flatVectorsWriter); } @Override diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java index ea33c842ca..6d86744dd3 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java @@ -17,7 +17,6 @@ import com.nvidia.cuvs.BruteForceIndex; import com.nvidia.cuvs.CagraIndex; -import com.nvidia.cuvs.HnswIndex; import java.io.Closeable; import java.io.IOException; import java.util.Objects; @@ -26,7 +25,6 @@ public class GPUIndex implements Closeable { private final CagraIndex cagraIndex; private final BruteForceIndex bruteforceIndex; - private final HnswIndex hnswIndex; private int maxDocs; private String fieldName; @@ -47,13 +45,11 @@ public GPUIndex( throw new IllegalArgumentException("negative maxDocs:" + maxDocs); } this.maxDocs = maxDocs; - this.hnswIndex = null; // TODO: remove hnswlib logic in a subsequent PR } - public GPUIndex(CagraIndex cagraIndex, BruteForceIndex bruteforceIndex, HnswIndex hnswIndex) { + public GPUIndex(CagraIndex cagraIndex, BruteForceIndex bruteforceIndex) { this.cagraIndex = cagraIndex; this.bruteforceIndex = bruteforceIndex; - this.hnswIndex = hnswIndex; } public CagraIndex getCagraIndex() { @@ -66,11 +62,6 @@ public BruteForceIndex getBruteforceIndex() { return bruteforceIndex; } - public HnswIndex getHNSWIndex() { - ensureOpen(); - return hnswIndex; - } - public String getFieldName() { return fieldName; } @@ -106,9 +97,6 @@ private void destroyIndices() throws IOException { if (bruteforceIndex != null) { bruteforceIndex.close(); } - if (hnswIndex != null) { - hnswIndex.close(); - } } catch (Throwable t) { Utils.handleThrowable(t); } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java index 0ad4861d57..27b01e0469 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java @@ -735,7 +735,6 @@ public void testMergeBruteForceIndex() throws IOException { 32, // writer threads 128, // intermediate graph degree 64, // graph degree - 1, IndexType.BRUTE_FORCE); // Use brute force index IndexWriterConfig config = @@ -887,7 +886,6 @@ public void testMergeCagraAndBruteForceIndex() throws IOException { 32, // writer threads 128, // intermediate graph degree 64, // graph degree - 1, IndexType.CAGRA_AND_BRUTE_FORCE); // Use combined CAGRA + brute force IndexWriterConfig config = From d52d2503031304236e9a6f803462044f7a3548ea Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Thu, 2 Oct 2025 08:51:27 -0400 Subject: [PATCH 023/105] Update pom.xml - Add meta information, configure Maven Surefire Plugin to give detailed stack traces and information on tests (NVIDIA/cuvs-lucene#25) Fixes NVIDIA/cuvs-lucene#26 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/25 --- java/cuvs-lucene/pom.xml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index dfd949758f..372ce73688 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -9,6 +9,26 @@ cuvs-lucene jar + + This project provides pluggable codecs for Apache Lucene to enable GPU-based accelerated vector search. + + https://rapids.ai + + + + Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + repo + A business-friendly OSS license + + + + scm:git:https://github.com/rapidsai/cuvs-lucene.git + scm:git:git@github.com:rapidsai/cuvs-lucene.git + HEAD + https://github.com/rapidsai/cuvs-lucene + + 21 21 @@ -104,6 +124,18 @@ + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.4 + + plain + false + false + false + 0 + + From 29f8dfd2041a3df703f9e7a67004983022b6320d Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Fri, 3 Oct 2025 15:34:34 -0400 Subject: [PATCH 024/105] Removing 10.2 codecs to avoid backward compatibility issues during Lucene upgrades (NVIDIA/cuvs-lucene#29) As per [last night's comment on the Solr Jira issue](https://issues.apache.org/jira/browse/SOLR-17892?focusedCommentId=18024203&page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel#comment-18024203) we realized that exposing the Lucene Codecs in the SPI registry prevents any calling application from being able to upgrade the codec because, under the hood, the next version of Lucene moves the older codec (Lucene101) into a different package name. As a result, we should only provide the relevant vector format, reader, and writer, and the calling application needs to wrap them into its own FilterCodec. For the sake of the current tests, we have decided to move the codec classes into the tests so that they are not bundled in the artifacts. This issue is a blocker for the inclusion of cuvs Lucene in Solr or any other Lucene-based search engine. Authors: - Vivek Narang (https://github.com/narangvivek10) - Ishan Chattopadhyaya (https://github.com/chatman) Approvers: - Ishan Chattopadhyaya (https://github.com/chatman) - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cuvs-lucene/pull/29 --- .../cuvs/lucene/CuVS2510GPUVectorsReader.java | 3 +- .../cuvs/lucene/FilterCuVSProvider.java | 11 ++ .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 0 .../lucene/Lucene101AcceleratedHNSWCodec.java | 0 ...TestCagraToHnswSerializationAndSearch.java | 104 +++++++++--------- .../services/org.apache.lucene.codecs.Codec | 0 6 files changed, 63 insertions(+), 55 deletions(-) rename java/cuvs-lucene/src/{main => test}/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java (100%) rename java/cuvs-lucene/src/{main => test}/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java (100%) rename java/cuvs-lucene/src/{main => test}/resources/META-INF/services/org.apache.lucene.codecs.Codec (100%) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java index b90055ab90..296df53b7c 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java @@ -27,6 +27,7 @@ import com.nvidia.cuvs.CagraIndex; import com.nvidia.cuvs.CagraQuery; import com.nvidia.cuvs.CagraSearchParams; +import com.nvidia.cuvs.CuVSMatrix; import com.nvidia.cuvs.CuVSResources; import java.io.IOException; import java.util.Iterator; @@ -346,7 +347,7 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits new CagraQuery.Builder(resources) .withTopK(topK) .withSearchParams(searchParams) - .withQueryVectors(new float[][] {target}) + .withQueryVectors(CuVSMatrix.ofArray(new float[][] {target})) .build(); CagraIndex cagraIndex = cuvsIndex.getCagraIndex(); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index 05acbeef53..81d248b2bc 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -77,6 +77,12 @@ public Builder newHostMatrixBuilder(long rows, long cols, DataType dataType) { return delegate.newHostMatrixBuilder(rows, cols, dataType); } + @Override + public Builder newHostMatrixBuilder( + long rows, long cols, int maxRows, int maxCols, DataType dataType) { + return delegate.newHostMatrixBuilder(rows, cols, maxRows, maxCols, dataType); + } + @Override public Builder newDeviceMatrixBuilder( CuVSResources resources, long rows, long cols, DataType dataType) { @@ -94,6 +100,11 @@ public MethodHandle newNativeMatrixBuilder() { return delegate.newNativeMatrixBuilder(); } + @Override + public MethodHandle newNativeMatrixBuilderWithStrides() { + return delegate.newNativeMatrixBuilderWithStrides(); + } + @Override public CuVSMatrix newMatrixFromArray(float[][] vectors) { return delegate.newMatrixFromArray(vectors); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java similarity index 100% rename from java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java rename to java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java similarity index 100% rename from java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java rename to java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java index 4f5f7aba6a..4295880442 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java @@ -99,61 +99,57 @@ public void testCagraToHnswSerializationAndSearch() throws IOException { } // Searching - try (Directory indexDirectory = FSDirectory.open(indexDirPath)) { - try (DirectoryReader reader = DirectoryReader.open(indexDirectory)) { - log.info("Successfully opened index"); - - int vectorCount = 0; - for (LeafReaderContext leafReaderContext : reader.leaves()) { - LeafReader leafReader = leafReaderContext.reader(); - FloatVectorValues knnValues = leafReader.getFloatVectorValues(VECTOR_FIELD); - assertNotNull(knnValues); - log.info( - VECTOR_FIELD - + " field: " - + knnValues.size() - + " vectors, " - + knnValues.dimension() - + " dimensions"); - vectorCount += knnValues.size(); - assertTrue("Vector dimension mismatch", knnValues.dimension() == dimension); - } - assertTrue("Dataset size mismatch", vectorCount == numDocs); - - log.info("Testing vector search queries..."); - IndexSearcher searcher = new IndexSearcher(reader); - - float[] queryVector = generateDataset(random, 1, dimension)[0]; - log.info("Query vector: " + Arrays.toString(queryVector)); - - KnnFloatVectorQuery query = new KnnFloatVectorQuery(VECTOR_FIELD, queryVector, topK); - TopDocs results = searcher.search(query, topK); - - log.info("Search results (" + results.totalHits + " total hits):"); - Integer[] expected = new Integer[] {1869, 1803, 1302, 59, 1497, 108, 1411, 351, 1982}; - HashSet expectedIds = new HashSet(Arrays.asList(expected)); - - for (int i = 0; i < results.scoreDocs.length; i++) { - ScoreDoc scoreDoc = results.scoreDocs[i]; - Document doc = searcher.storedFields().document(scoreDoc.doc); - String id = doc.get(ID_FIELD); - log.info( - " Rank " - + (i + 1) - + ": doc " - + scoreDoc.doc - + " (id=" - + id - + "), score=" - + scoreDoc.score); - assertTrue( - "Id: " + id + " expected but not found", expectedIds.contains(Integer.valueOf(id))); - } - assertTrue("TopK results not returned", results.scoreDocs.length == topK); - - } catch (Exception e) { - e.printStackTrace(); + try (Directory indexDirectory = FSDirectory.open(indexDirPath); + DirectoryReader reader = DirectoryReader.open(indexDirectory)) { + log.info("Successfully opened index"); + + int vectorCount = 0; + for (LeafReaderContext leafReaderContext : reader.leaves()) { + LeafReader leafReader = leafReaderContext.reader(); + FloatVectorValues knnValues = leafReader.getFloatVectorValues(VECTOR_FIELD); + assertNotNull(knnValues); + log.info( + VECTOR_FIELD + + " field: " + + knnValues.size() + + " vectors, " + + knnValues.dimension() + + " dimensions"); + vectorCount += knnValues.size(); + assertTrue("Vector dimension mismatch", knnValues.dimension() == dimension); + } + assertTrue("Dataset size mismatch", vectorCount == numDocs); + + log.info("Testing vector search queries..."); + IndexSearcher searcher = new IndexSearcher(reader); + + float[] queryVector = generateDataset(random, 1, dimension)[0]; + log.info("Query vector: " + Arrays.toString(queryVector)); + + KnnFloatVectorQuery query = new KnnFloatVectorQuery(VECTOR_FIELD, queryVector, topK); + TopDocs results = searcher.search(query, topK); + + log.info("Search results (" + results.totalHits + " total hits):"); + Integer[] expected = new Integer[] {1869, 1803, 1302, 59, 1497, 108, 1411, 351, 1982}; + HashSet expectedIds = new HashSet(Arrays.asList(expected)); + + for (int i = 0; i < results.scoreDocs.length; i++) { + ScoreDoc scoreDoc = results.scoreDocs[i]; + Document doc = searcher.storedFields().document(scoreDoc.doc); + String id = doc.get(ID_FIELD); + log.info( + " Rank " + + (i + 1) + + ": doc " + + scoreDoc.doc + + " (id=" + + id + + "), score=" + + scoreDoc.score); + assertTrue( + "Id: " + id + " expected but not found", expectedIds.contains(Integer.valueOf(id))); } + assertTrue("TopK results not returned", results.scoreDocs.length == topK); } } diff --git a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/java/cuvs-lucene/src/test/resources/META-INF/services/org.apache.lucene.codecs.Codec similarity index 100% rename from java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec rename to java/cuvs-lucene/src/test/resources/META-INF/services/org.apache.lucene.codecs.Codec From d642fec8fce4beb4d9fc86a60a07589082d09765 Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Fri, 10 Oct 2025 01:12:26 +0530 Subject: [PATCH 025/105] Avoid copying vectors to heap before creating dataset matrix (NVIDIA/cuvs-lucene#30) We were inadvertently copying over the vectors from a list to an array before creating the CuVSMatrix instance. This is memory inefficient. Authors: - Ishan Chattopadhyaya (https://github.com/chatman) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/30 --- .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 6 +++-- .../Lucene99AcceleratedHNSWVectorsWriter.java | 3 ++- .../java/com/nvidia/cuvs/lucene/Utils.java | 25 ++++++++++++++----- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index 9c3690e064..4773edfcd7 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -236,7 +236,8 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro if (indexType.cagra()) { try { var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - CuVSMatrix dataset = Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension()); + CuVSMatrix dataset = + Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension(), resources); writeCagraIndex(cagraIndexOutputStream, dataset); } catch (Throwable t) { Utils.handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); @@ -249,7 +250,8 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro bruteForceIndexOffset = cuvsIndex.getFilePointer(); if (indexType.bruteForce()) { var bruteForceIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - CuVSMatrix dataset = Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension()); + CuVSMatrix dataset = + Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension(), resources); writeBruteForceIndex(bruteForceIndexOutputStream, dataset); bruteForceIndexLength = cuvsIndex.getFilePointer() - bruteForceIndexOffset; } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index e15b743e52..d70673566a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -198,7 +198,8 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro } try { - CuVSMatrix dataset = Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension()); + CuVSMatrix dataset = + Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension(), resources); if (dataset.size() < 2) { throw new IllegalArgumentException(dataset.size() + " vectors, less than min [2] required"); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java index dfb9aa0c49..a44dc03a6d 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java @@ -38,17 +38,30 @@ static void handleThrowable(Throwable t) throws IOException { /** * A method to build a {@link CuVSMatrix} from a list of float vectors. * - * Note: This could be a memory-intensive operation and should therefore be avoided. - * Consider using this {@link CuVSMatrix.Builder} instead for copying the vectors without loading them in heap. + * Uses {@link CuVSMatrix.Builder} to copy vectors directly to device memory + * without creating intermediate heap arrays. * * @param data The float vectors * @param dimensions The number float elements in each vector + * @param resources The CuVS resources for device matrix creation * @return an instance of {@link CuVSMatrix} */ - static CuVSMatrix createFloatMatrix(List data, int dimensions) { - // Convert List to float[][] for the ofArray method - float[][] vectors = data.toArray(new float[0][]); - return CuVSMatrix.ofArray(vectors); + static CuVSMatrix createFloatMatrix(List data, int dimensions, CuVSResources resources) { + // Use Builder pattern to avoid intermediate float[][] allocation + // and copy directly from List to device memory + CuVSMatrix.Builder builder = + CuVSMatrix.deviceBuilder( + resources, + data.size(), // rows (number of vectors) + dimensions, // columns (vector dimension) + CuVSMatrix.DataType.FLOAT); + + // Add vectors one by one - builder copies directly to device memory + for (float[] vector : data) { + builder.addVector(vector); + } + + return builder.build(); } static long nanosToMillis(long nanos) { From f4bd68254a1aa26922fcc28a794a5f0231ef04be Mon Sep 17 00:00:00 2001 From: Ishan Chattopadhyaya Date: Fri, 17 Oct 2025 20:26:37 +0530 Subject: [PATCH 026/105] Upgrade to cuvs-java maven central artifacts, single-vector bugfix (NVIDIA/cuvs-lucene#31) * cuvs-java 25.10 is released, use that from Maven Central * There's a blocker bug for Solr integration where single vector indexes cause a Solr crash. Adding a graceful handling for that. * Updated README.md with a more accurate description of the project. Authors: - Ishan Chattopadhyaya (https://github.com/chatman) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/31 --- java/cuvs-lucene/README.md | 4 +- java/cuvs-lucene/pom.xml | 9 +-- .../Lucene99AcceleratedHNSWVectorsWriter.java | 65 ++++++++++++++++++- ...TestCagraToHnswSerializationAndSearch.java | 46 +++++++++++++ 4 files changed, 113 insertions(+), 11 deletions(-) diff --git a/java/cuvs-lucene/README.md b/java/cuvs-lucene/README.md index cc4da5a1d1..bae113a8fc 100644 --- a/java/cuvs-lucene/README.md +++ b/java/cuvs-lucene/README.md @@ -1,10 +1,10 @@ # Lucene cuVS -This is a codec for connecting [cuVS](https://github.com/rapidsai/cuvs), NVIDIA's GPU accelerated vector search library, into [Apache Lucene](https://github.com/apache/lucene). +This is a project for using [cuVS](https://github.com/rapidsai/cuvs), NVIDIA's GPU accelerated vector search library, with [Apache Lucene](https://github.com/apache/lucene). ## Overview -The cuVS library is plugged in as a new `KnnVectorFormat` via a custom codec. +This library provides a new [KnnVectorFormat](https://lucene.apache.org/core/10_3_1/core/org/apache/lucene/codecs/KnnVectorsFormat.html) which can be plugged into a Lucene codec. > [!CAUTION] > This is not production ready yet. diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 372ce73688..dd86ddeaed 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -34,13 +34,7 @@ 21 - - - searchscale-maven - SearchScale Maven - https://maven.searchscale.com/snapshots - - + @@ -69,7 +63,6 @@ com.nvidia.cuvs cuvs-java - 25.10.0 diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index d70673566a..14c2ddbed9 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -202,7 +202,10 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension(), resources); if (dataset.size() < 2) { - throw new IllegalArgumentException(dataset.size() + " vectors, less than min [2] required"); + // Handle single vector case by creating a dummy HNSW graph + // TODO: Remove this workaround once https://github.com/rapidsai/cuvs/pull/1256 is merged + writeSingleVectorGraph(fieldInfo, vectors); + return; } long startTime = System.nanoTime(); @@ -514,6 +517,66 @@ private void writeSortingField(GPUFieldWriter fieldData, Sorter.DocMap sortMap) writeFieldInternal(fieldData.fieldInfo(), sortedVectors); } + private void writeSingleVectorGraph(FieldInfo fieldInfo, List vectors) + throws IOException { + // Workaround for CAGRA not supporting single vector indexes + // TODO: Remove this method once https://github.com/rapidsai/cuvs/pull/1256 is merged + try { + int size = 1; + int dimensions = fieldInfo.getVectorDimension(); + + // Create a dummy HNSW graph for a single vector + GPUBuiltHnswGraph hnswGraph = createSingleVectorHnswGraph(size, dimensions); + + long vectorIndexOffset = hnswVectorIndex.getFilePointer(); + + // Write the graph to the vector index + int[][] graphLevelNodeOffsets = writeGraph(hnswGraph, hnswVectorIndex); + + long vectorIndexLength = hnswVectorIndex.getFilePointer() - vectorIndexOffset; + + // Write metadata + writeMeta( + hnswVectorIndex, + hnswMeta, + fieldInfo, + vectorIndexOffset, + vectorIndexLength, + size, + hnswGraph, + graphLevelNodeOffsets); + + long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - System.nanoTime()); + info("Single vector HNSW graph created in " + elapsedMillis + "ms, with " + size + " vector"); + + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + + /** + * Creates a dummy HNSW graph for a single vector. + * The graph will have 1 level with 1 node and no neighbors. + */ + private GPUBuiltHnswGraph createSingleVectorHnswGraph(int size, int dimensions) throws Throwable { + // Create adjacency list for single node with no neighbors + int[][] singleNodeAdjacency = new int[][] {{-1}}; // -1 indicates no neighbors + + // Create CuVSMatrix from the adjacency list + CuVSMatrix adjacencyMatrix = CuVSMatrix.ofArray(singleNodeAdjacency); + + // Create layer data for single-level graph + List layerNodes = new ArrayList<>(); + List layerAdjacencies = new ArrayList<>(); + + // Layer 0: contains all nodes (just the single node) + layerNodes.add(null); // Layer 0 contains all nodes, so we don't need to store node list + layerAdjacencies.add(adjacencyMatrix); + + // Create the single-layer graph + return new GPUBuiltHnswGraph(size, dimensions, layerNodes, layerAdjacencies); + } + private void writeEmpty(FieldInfo fieldInfo) throws IOException { writeMeta(null, hnswMeta, fieldInfo, 0, 0, 0, null, null); } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java index 4295880442..a2c2e1f2a3 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java @@ -153,6 +153,52 @@ public void testCagraToHnswSerializationAndSearch() throws IOException { } } + @Test + public void testSingleVectorIndex() throws Exception { + // Test single vector index support with dummy HNSW graph + // TODO: This test can be removed once https://github.com/rapidsai/cuvs/pull/1256 is merged + // and CAGRA natively supports single vector indexes + Codec codec = new Lucene101AcceleratedHNSWCodec(); + + final String ID_FIELD = "id"; + final String VECTOR_FIELD = "vector_field"; + + int dimension = 32; + float[] vector = generateDataset(random, 1, dimension)[0]; + + // Index a single document with a vector - this should now work with dummy HNSW graph + try (Directory indexDirectory = newDirectory()) { + IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); + try (IndexWriter indexWriter = new IndexWriter(indexDirectory, config)) { + Document document = new Document(); + document.add(new StringField(ID_FIELD, "0", Field.Store.YES)); + document.add(new KnnFloatVectorField(VECTOR_FIELD, vector, EUCLIDEAN)); + indexWriter.addDocument(document); + + // This should now succeed by creating a dummy HNSW graph for the single vector + indexWriter.commit(); + } + + // Verify the index can be opened and searched + try (DirectoryReader reader = DirectoryReader.open(indexDirectory)) { + assertEquals(1, reader.numDocs()); + LeafReader leafReader = getOnlyLeafReader(reader); + FloatVectorValues knnValues = leafReader.getFloatVectorValues(VECTOR_FIELD); + assertNotNull(knnValues); + assertEquals(1, knnValues.size()); + assertEquals(dimension, knnValues.dimension()); + + // Test search functionality + IndexSearcher searcher = new IndexSearcher(reader); + KnnFloatVectorQuery query = new KnnFloatVectorQuery(VECTOR_FIELD, vector, 1); + TopDocs results = searcher.search(query, 1); + assertEquals(1, results.totalHits.value()); + assertEquals(1, results.scoreDocs.length); + assertEquals(0, results.scoreDocs[0].doc); + } + } + } + @AfterClass public static void afterClass() throws Exception { File indexDirPathFile = indexDirPath.toFile(); From e3c53e3d66cca7f07dd2a2df740fff2674ed3799 Mon Sep 17 00:00:00 2001 From: Nate Rock Date: Tue, 21 Oct 2025 17:16:55 -0500 Subject: [PATCH 027/105] Use main branch (NVIDIA/cuvs-lucene#32) ## Description Supports rollout of new branching strategy. https://docs.rapids.ai/notices/rsn0047/ xref: https://github.com/rapidsai/build-planning/issues/224 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index f1085f3388..721ff56322 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -3,7 +3,7 @@ name: build on: push: branches: - - "branch-*" + - "main" tags: - v[0-9][0-9].[0-9][0-9].[0-9][0-9] workflow_dispatch: From d0649a79a8d47ee61b2187755aae43af75c8a304 Mon Sep 17 00:00:00 2001 From: Nate Rock Date: Tue, 21 Oct 2025 17:18:50 -0500 Subject: [PATCH 028/105] Use main shared-workflows branch (NVIDIA/cuvs-lucene#33) Contributes to https://github.com/rapidsai/build-planning/issues/224 ## Notes for Reviewers This is safe to admin-merge because the change is a no-op... configs on those 2 branches are identical. --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 8 ++++---- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/ci/release/update-version.sh | 7 +++++++ 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 721ff56322..5494c906b4 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -34,7 +34,7 @@ concurrency: jobs: java-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index e7fffb5949..95bf5e4706 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-25.10 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main if: always() with: needs: ${{ toJSON(needs) }} @@ -42,7 +42,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-25.10 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main with: files_yaml: | test_java: @@ -52,14 +52,14 @@ jobs: checks: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-25.10 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" conda-java-tests: needs: [changed-files, checks] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 4a8f5ae497..cbecd1f038 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -25,7 +25,7 @@ on: jobs: conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main strategy: fail-fast: false matrix: diff --git a/java/cuvs-lucene/ci/release/update-version.sh b/java/cuvs-lucene/ci/release/update-version.sh index 4a6bf41782..e1e769073b 100755 --- a/java/cuvs-lucene/ci/release/update-version.sh +++ b/java/cuvs-lucene/ci/release/update-version.sh @@ -30,6 +30,9 @@ function sed_runner() { sed -i.bak ''"$1"'' "$2" && rm -f "${2}".bak } +# Centralized version file update +echo "${NEXT_FULL_TAG}" > VERSION + # Update Java version NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}" sed_runner "s/VERSION=\".*\"/VERSION=\"${NEXT_FULL_JAVA_TAG}\"/g" build.sh @@ -40,3 +43,7 @@ sed_runner "s| CuVS [[:digit:]]\{2\}\.[[:digit:]]\{2\} | CuVS ${NEXT_SHORT_TAG} for FILE in dependencies.yaml conda/environments/*.yaml; do sed_runner "s/libcuvs==.*/libcuvs==${NEXT_SHORT_TAG}.*/g" "${FILE}" done + +for FILE in .github/workflows/*.yaml; do + sed_runner "s/:[0-9]*\\.[0-9]*-/:${NEXT_SHORT_TAG}-/g" "${FILE}" +done From 2e28888245b46d3de2d6b44f2879ac0a2ac478a7 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Fri, 24 Oct 2025 19:03:02 +0530 Subject: [PATCH 029/105] Add Javadocs and configure the Apache Maven Javadoc Plugin (NVIDIA/cuvs-lucene#28) Fixes NVIDIA/cuvs-lucene#27 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Ishan Chattopadhyaya (https://github.com/chatman) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/28 --- java/cuvs-lucene/pom.xml | 18 +++ .../cuvs/lucene/CuVS2510GPUVectorsFormat.java | 38 ++++- .../cuvs/lucene/CuVS2510GPUVectorsReader.java | 144 +++++++++++++++++- .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 130 +++++++++++++++- .../lucene/FilterCuVSServiceProvider.java | 10 +- .../nvidia/cuvs/lucene/GPUBuiltHnswGraph.java | 59 ++++++- .../nvidia/cuvs/lucene/GPUFieldWriter.java | 33 +++- .../java/com/nvidia/cuvs/lucene/GPUIndex.java | 57 ++++++- .../cuvs/lucene/GPUKnnFloatVectorQuery.java | 16 +- .../lucene/GPUPerLeafCuVSKnnCollector.java | 15 +- .../cuvs/lucene/IndexInputInputStream.java | 17 ++- .../cuvs/lucene/IndexOutputOutputStream.java | 29 +++- .../Lucene99AcceleratedHNSWVectorsFormat.java | 50 +++++- .../Lucene99AcceleratedHNSWVectorsWriter.java | 116 +++++++++++++- .../java/com/nvidia/cuvs/lucene/Utils.java | 35 ++++- .../com/nvidia/cuvs/lucene/package-info.java | 6 +- .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 28 +++- .../lucene/Lucene101AcceleratedHNSWCodec.java | 6 +- ...TestCagraToHnswSerializationAndSearch.java | 12 +- 19 files changed, 775 insertions(+), 44 deletions(-) diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index dd86ddeaed..38ab629b26 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -129,6 +129,24 @@ 0 + + org.apache.maven.plugins + maven-javadoc-plugin + 3.12.0 + + + attach-javadocs + + jar + + + com.nvidia.cuvs.lucene + all + ${project.build.directory}/javadocs + + + + diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java index 10213bb647..d79cb45678 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java @@ -30,12 +30,16 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; -/** CuVS based KnnVectorsFormat for GPU acceleration */ +/** + * Extends upon the KnnVectorsFormat - Encodes/decodes per-document vector and any associated indexing structures required to support + * GPU-based accelerated nearest-neighbor search. + * + * @since 25.10 + */ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { static final Logger log = Logger.getLogger(CuVS2510GPUVectorsFormat.class.getName()); - // TODO: fix Lucene version in name, to the final targeted release, if any static final String CUVS_META_CODEC_NAME = "Lucene102CuVSVectorsFormatMeta"; static final String CUVS_META_CODEC_EXT = "vemc"; static final String CUVS_INDEX_CODEC_NAME = "Lucene102CuVSVectorsFormatIndex"; @@ -62,7 +66,7 @@ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { final CuVS2510GPUVectorsWriter.IndexType indexType; // the index type to build, when writing /** - * Creates a CuVS2510GPUVectorsFormat, with default values. + * Initializes the {@link CuVS2510GPUVectorsFormat} with default parameter values. * * @throws LibraryException if the native library fails to load */ @@ -75,7 +79,12 @@ public CuVS2510GPUVectorsFormat() { } /** - * Creates a CuVS2510GPUVectorsFormat, with the given threads, graph degree, etc. + * Initializes the {@link CuVS2510GPUVectorsFormat} with the given threads, graph degree, etc. + * + * @param cuvsWriterThreads the number of cuVS writer threads to use + * @param intGraphDegree the intermediate graph degree for building the CAGRA index + * @param graphDegree the graph degree for building the CAGRA index + * @param indexType the {@link com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType} * * @throws LibraryException if the native library fails to load */ @@ -88,6 +97,9 @@ public CuVS2510GPUVectorsFormat( this.indexType = indexType; } + /** + * Returns a {@link CuVS2510GPUVectorsWriter} to write the vectors to the index. + */ @Override public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { checkSupported(); @@ -96,17 +108,26 @@ public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOE state, cuvsWriterThreads, intGraphDegree, graphDegree, indexType, resources, flatWriter); } + /** + * Returns a KnnVectorsReader to read the vectors from the index. + */ @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { checkSupported(); return new CuVS2510GPUVectorsReader(state, resources, flatVectorsFormat.fieldsReader(state)); } + /** + * Returns the maximum number of vector dimensions supported by this codec for the given field name. + */ @Override public int getMaxDimensions(String fieldName) { return maxDimensions; } + /** + * Returns a string containing information like cuvsWriterThreads, intGraphDegree, etc. + */ @Override public String toString() { StringBuilder sb = new StringBuilder(this.getClass().getSimpleName()); @@ -118,11 +139,18 @@ public String toString() { return sb.toString(); } - /** Tells whether the platform supports cuVS. */ + /** + * Tells whether the platform supports cuVS. + * + * @return if cuVS is supported or not + */ public static boolean supported() { return resources != null; } + /** + * Checks if cuVS is supported and throws {@link UnsupportedOperationException} otherwise. + */ public static void checkSupported() { if (!supported()) { throw new UnsupportedOperationException(); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java index 296df53b7c..ae9ef94909 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java @@ -60,7 +60,11 @@ import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.hnsw.IntToIntFunction; -/** KnnVectorsReader instance associated with CuVS format */ +/** + * KnnVectorsReader instance associated with cuVS format for reading vectors from an index. + * + * @since 25.10 + */ public class CuVS2510GPUVectorsReader extends KnnVectorsReader { @SuppressWarnings("unused") @@ -73,6 +77,15 @@ public class CuVS2510GPUVectorsReader extends KnnVectorsReader { private final IntObjectHashMap cuvsIndices; private final IndexInput cuvsIndexInput; + /** + * Initializes the {@link CuVS2510GPUVectorsReader}, checks and loads the index. + * + * @param state instance of the SegmentReadState + * @param resources instance of the CuVSResources + * @param flatReader instance of the FlatVectorsReader + * + * @throws IOException I/O exception + */ public CuVS2510GPUVectorsReader( SegmentReadState state, CuVSResources resources, FlatVectorsReader flatReader) throws IOException { @@ -114,6 +127,15 @@ public CuVS2510GPUVectorsReader( } } + /** + * Opens and returns the IndexInput for the segment file. + * + * @param state instance of the SegmentReadState + * @param versionMeta the version number + * @param context instance of the IOContext + * @return an instance of the IndexInput + * @throws IOException + */ private static IndexInput openCuVSInput( SegmentReadState state, int versionMeta, IOContext context) throws IOException { String fileName = @@ -140,6 +162,12 @@ private static IndexInput openCuVSInput( } } + /** + * Confirms that the vector dimensions are as expected. + * + * @param info instance of the FieldInfo that describes document fields + * @param fieldEntry instance of the FieldEntry that holds the meta information for the field + */ private void validateFieldEntry(FieldInfo info, FieldEntry fieldEntry) { int dimension = info.getVectorDimension(); if (dimension != fieldEntry.dims()) { @@ -153,6 +181,12 @@ private void validateFieldEntry(FieldInfo info, FieldEntry fieldEntry) { } } + /** + * Reads the fieldInfo for each index field and loads FieldEntry in a map. + * + * @param meta intance of the ChecksumIndexInput + * @throws IOException + */ private void readFields(ChecksumIndexInput meta) throws IOException { for (int fieldNumber = meta.readInt(); fieldNumber != -1; fieldNumber = meta.readInt()) { FieldInfo info = fieldInfos.fieldInfo(fieldNumber); @@ -176,6 +210,13 @@ private void readFields(ChecksumIndexInput meta) throws IOException { VectorSimilarityFunction.COSINE, VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT); + /** + * Checks the distance function validity and returns it. + * + * @param input instance of DataInput + * @return an instance of VectorSimilarityFunction + * @throws IOException + */ static VectorSimilarityFunction readSimilarityFunction(DataInput input) throws IOException { int i = input.readInt(); if (i < 0 || i >= SIMILARITY_FUNCTIONS.size()) { @@ -184,6 +225,13 @@ static VectorSimilarityFunction readSimilarityFunction(DataInput input) throws I return SIMILARITY_FUNCTIONS.get(i); } + /** + * Reads the vector encoding (The numeric datatype of the vector values) from the DataInput. + * + * @param input instance of DataInput + * @return the vector encoding + * @throws IOException + */ static VectorEncoding readVectorEncoding(DataInput input) throws IOException { int encodingId = input.readInt(); if (encodingId < 0 || encodingId >= VectorEncoding.values().length) { @@ -192,6 +240,14 @@ static VectorEncoding readVectorEncoding(DataInput input) throws IOException { return VectorEncoding.values()[encodingId]; } + /** + * Reads the field from IndexInput using FieldInfo. + * + * @param input instance of IndexInput + * @param info instance of FieldInfo + * @return the field entry + * @throws IOException + */ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOException { VectorEncoding vectorEncoding = readVectorEncoding(input); VectorSimilarityFunction similarityFunction = readSimilarityFunction(input); @@ -207,6 +263,13 @@ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOExceptio return FieldEntry.readEntry(input, vectorEncoding, info.getVectorSimilarityFunction()); } + /** + * Gets the FieldEntry from the map using the field name. Check the encoding as well. + * + * @param field name of the field + * @param expectedEncoding expected encoding + * @return an instance of FieldEntry that Holds the meta information for the field + */ private FieldEntry getFieldEntry(String field, VectorEncoding expectedEncoding) { final FieldInfo info = fieldInfos.fieldInfo(field); final FieldEntry fieldEntry; @@ -225,6 +288,12 @@ private FieldEntry getFieldEntry(String field, VectorEncoding expectedEncoding) return fieldEntry; } + /** + * Invokes loadCuVSIndex for each field and returns the map of {@link GPUIndex}. + * + * @return the map containing {@link GPUIndex} objects + * @throws IOException + */ private IntObjectHashMap loadCuVSIndices() throws IOException { var indices = new IntObjectHashMap(); for (var e : fields) { @@ -236,6 +305,13 @@ private IntObjectHashMap loadCuVSIndices() throws IOException { return indices; } + /** + * Loads the CAGRA and bruteforce index (if exists) onto the GPU. + * + * @param fieldEntry instance of {@link FieldEntry} + * @return return the instance of {@link GPUIndex} + * @throws IOException + */ private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { CagraIndex cagraIndex = null; BruteForceIndex bruteForceIndex = null; @@ -264,6 +340,9 @@ private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { return new GPUIndex(cagraIndex, bruteForceIndex); } + /** + * Closes the resources. + */ @Override public void close() throws IOException { var closeableStream = @@ -277,16 +356,27 @@ static Stream stream(Iterator iterator) { return StreamSupport.stream(((Iterable) () -> iterator).spliterator(), false); } + /** + * Checks consistency of this reader. + */ @Override public void checkIntegrity() throws IOException { // TODO: Pending implementation } + /** + * Returns the FloatVectorValues for the given field. + */ @Override public FloatVectorValues getFloatVectorValues(String field) throws IOException { return flatVectorsReader.getFloatVectorValues(field); } + /** + * Returns the FloatVectorValues for the given field. + * + * This is not supported. + */ @Override public ByteVectorValues getByteVectorValues(String field) { throw new UnsupportedOperationException("byte vectors not supported"); @@ -297,6 +387,9 @@ public interface FloatToFloatFunction { float apply(float v); } + /** + * Returns a long array from bits. + */ static long[] bitsToLongArray(Bits bits) { if (bits instanceof FixedBitSet fixedBitSet) { return fixedBitSet.getBits(); @@ -305,6 +398,12 @@ static long[] bitsToLongArray(Bits bits) { } } + /** + * Get the score normalization function. + * + * @param sim instance of VectorSimilarityFunction + * @return an instance of the FloatToFloatFunction + */ static FloatToFloatFunction getScoreNormalizationFunc(VectorSimilarityFunction sim) { // TODO: check for different similarities return score -> (1f / (1f + score)); @@ -313,6 +412,9 @@ static FloatToFloatFunction getScoreNormalizationFunc(VectorSimilarityFunction s // This is a hack - https://github.com/rapidsai/cuvs/issues/696 static final int FILTER_OVER_SAMPLE = 10; + /** + * Returns the k nearest neighbor documents using cuVS's CAGRA or Bruteforce algorithm for this field, to the given vector. + */ @Override public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { @@ -401,12 +503,20 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits } } + /** + * Return the k nearest neighbor documents as determined by comparison of their vector values for this field, to the given vector. + * + * This is not supported. + */ @Override public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { throw new UnsupportedOperationException("byte vectors not supported"); } + /** + * Holds the meta information for the field. + */ record FieldEntry( VectorEncoding vectorEncoding, VectorSimilarityFunction similarityFunction, @@ -417,6 +527,15 @@ record FieldEntry( long bruteForceIndexOffset, long bruteForceIndexLength) { + /** + * Returns an instance of FieldEntry. + * + * @param input instance of IndexInput + * @param vectorEncoding The numeric datatype of the vector values + * @param similarityFunction Vector similarity function; used in search to return top K most similar vectors to a target vector + * @return an instance of FieldEntry + * @throws IOException I/O Exceptions + */ static FieldEntry readEntry( IndexInput input, VectorEncoding vectorEncoding, @@ -440,6 +559,14 @@ static FieldEntry readEntry( } } + /** + * Checks the version and throws CorruptIndexException on mismatch. + * + * @param versionMeta + * @param versionVectorData + * @param in + * @throws CorruptIndexException + */ static void checkVersion(int versionMeta, int versionVectorData, IndexInput in) throws CorruptIndexException { if (versionMeta != versionVectorData) { @@ -454,14 +581,29 @@ static void checkVersion(int versionMeta, int versionVectorData, IndexInput in) } } + /** + * Gets the instance of FieldInfos. + * + * @return the instance of FieldInfos + */ public FieldInfos getFieldInfos() { return fieldInfos; } + /** + * Gets the map of {@link GPUIndex} objects. + * + * @return the map of gpu index objects + */ public IntObjectHashMap getCuvsIndexes() { return cuvsIndices; } + /** + * Gets the map of FieldEntry objects that hold the meta information for the field. + * + * @return the map of FieldEntry objects + */ public IntObjectHashMap getFieldEntries() { return fields; } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index 4773edfcd7..c55835cab9 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -64,8 +64,9 @@ import org.apache.lucene.util.InfoStream; /** - * KnnVectorsWriter for CuVS, responsible for merge and flush of vectors into - * GPU + * extends upon KnnVectorsWriter and has implementation for critical methods like flush, merge etc. + * + * @since 25.10 */ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { @@ -95,7 +96,9 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { private final InfoStream infoStream; private boolean finished; - /** The CuVS index Type. */ + /** + * The cuVS index Types. + */ public enum IndexType { /** Builds a Cagra index. */ @@ -113,15 +116,38 @@ public enum IndexType { this.bruteForce = bruteForce; } + /** + * Check if cagra is set + * + * @return is cagra set + */ public boolean cagra() { return cagra; } + /** + * Check if bruteforce is set + * + * @return is bruteforce set + */ public boolean bruteForce() { return bruteForce; } } + /** + * Initializes {@link CuVS2510GPUVectorsWriter}. + * + * @param state instance of the SegmentWriteState + * @param cuvsWriterThreads the number of cuVS writer threads + * @param intGraphDegree the intermediate graph degree for building the CAGRA index + * @param graphDegree the graph degree for building the CAGRA index + * @param indexType the IndexType + * @param resources instance of the CuVSResources + * @param flatVectorsWriter instance of FlatVectorsWriter + * + * @throws IOException I/O exceptions + */ public CuVS2510GPUVectorsWriter( SegmentWriteState state, int cuvsWriterThreads, @@ -172,6 +198,9 @@ public CuVS2510GPUVectorsWriter( } } + /** + * Add new field for indexing. + */ @Override public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { var encoding = fieldInfo.getVectorEncoding(); @@ -186,6 +215,13 @@ public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException return writer; } + /** + * Returns a string containing meta information like graph degree etc. + * + * @param size index size + * @param args other parameters like graph degree, Intermediate graph degree, etc. + * @return the string containing the meta information + */ static String indexMsg(int size, int... args) { StringBuilder sb = new StringBuilder("cagra index params"); sb.append(": size=").append(size); @@ -196,6 +232,12 @@ static String indexMsg(int size, int... args) { return sb.toString(); } + /** + * Builds and returns an instance of CagraIndexParams. + * + * @param size the size of the index + * @return an instance of CagraIndexParams + */ private CagraIndexParams cagraIndexParams(int size) { if (size < 2) { // https://github.com/rapidsai/cuvs/issues/666 @@ -210,12 +252,24 @@ private CagraIndexParams cagraIndexParams(int size) { .build(); } + /** + * Utility to print info/debug messages via InfoStream. + * + * @param msg + */ private void info(String msg) { if (infoStream.isEnabled(CUVS_COMPONENT)) { infoStream.message(CUVS_COMPONENT, msg); } } + /** + * Creates CAGRA and/or Bruteforce indexes and writes them. + * + * @param fieldInfo Instance of the FieldInFo to use + * @param vectors list of float vectors to index + * @throws IOException + */ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOException { if (vectors.size() == 0) { writeEmpty(fieldInfo); @@ -268,6 +322,13 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro } } + /** + * Builds and writes the CAGRA index. + * + * @param os Instance of the OutputStream + * @param dataset The instance of CuVSMatrix holding the dataset + * @throws Throwable + */ private void writeCagraIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { if (dataset.size() < 2) { throw new IllegalArgumentException(dataset.size() + " vectors, less than min [2] required"); @@ -283,6 +344,13 @@ private void writeCagraIndex(OutputStream os, CuVSMatrix dataset) throws Throwab index.close(); } + /** + * Builds and writes the Bruteforce index. + * + * @param os Instance of OutputStream to write the index to + * @param dataset Instance of CuVSMatrix that holds the dataset + * @throws Throwable + */ private void writeBruteForceIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { BruteForceIndexParams params = new BruteForceIndexParams.Builder() @@ -297,6 +365,9 @@ private void writeBruteForceIndex(OutputStream os, CuVSMatrix dataset) throws Th index.close(); } + /** + * Creates the CAGRA and/or Bruteforce indexes and writes them to the disk. + */ @Override public void flush(int maxDoc, DocMap sortMap) throws IOException { flatVectorsWriter.flush(maxDoc, sortMap); @@ -309,10 +380,23 @@ public void flush(int maxDoc, DocMap sortMap) throws IOException { } } + /** + * Calls the method that builds indexes and writes them to the disk. + * + * @param fieldData reference to the {@link GPUFieldWriter} + * @throws IOException + */ private void writeField(GPUFieldWriter fieldData) throws IOException { writeFieldInternal(fieldData.fieldInfo(), fieldData.getVectors()); } + /** + * Builds indexes and writes them to the disk. + * + * @param fieldData reference to the {@link GPUFieldWriter} + * @param sortMap reference to DocMap + * @throws IOException I/O Exceptions + */ private void writeSortingField(GPUFieldWriter fieldData, Sorter.DocMap sortMap) throws IOException { @@ -328,10 +412,27 @@ private void writeSortingField(GPUFieldWriter fieldData, Sorter.DocMap sortMap) writeFieldInternal(fieldData.fieldInfo(), sortedVectors); } + /** + * Writes empty meta information for the field. + * + * @param fieldInfo instance of the FieldInfo + * @throws IOException I/O Exceptions + */ private void writeEmpty(FieldInfo fieldInfo) throws IOException { writeMeta(fieldInfo, 0, 0L, 0L, 0L, 0L); } + /** + * Writes the meta information for the index. + * + * @param field instance of FieldInfo + * @param count number of vectors + * @param cagraIndexOffset CAGRA index offset + * @param cagraIndexLength CAGRA index length + * @param bruteForceIndexOffset Bruteforce index offset + * @param bruteForceIndexLength Bruteforce index length + * @throws IOException I/O Exceptions + */ private void writeMeta( FieldInfo field, int count, @@ -369,6 +470,13 @@ static int distFuncToOrd(VectorSimilarityFunction func) { during the norm computation between the dataset vectors\ """; + /** + * Uses the cuVS API to merge CAGRA indexes. + * + * @param fieldInfo instance of the FieldInfo + * @param mergeState instance of the MergeState + * @throws IOException I/O Exceptions + */ private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throws IOException { try { @@ -409,8 +517,8 @@ private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throw } /** - * Creates List from merged vectors - * */ + * Creates List from merged vectors. + */ private List createListFromMergedVectors(FloatVectorValues mergedVectorValues) throws IOException { List res = new ArrayList(); @@ -488,6 +596,9 @@ private void writeMergedCagraIndex(FieldInfo fieldInfo, CagraIndex mergedIndex, } } + /** + * Write field for merging. + */ @Override public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { flatVectorsWriter.mergeOneField(fieldInfo, mergeState); @@ -518,6 +629,9 @@ public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOE } } + /** + * Called once at the end before close. + */ @Override public void finish() throws IOException { if (finished) { @@ -536,11 +650,17 @@ public void finish() throws IOException { } } + /** + * Close the applicable resources. + */ @Override public void close() throws IOException { IOUtils.close(meta, cuvsIndex, flatVectorsWriter); } + /** + * Returns the memory usage of this object in bytes. + */ @Override public long ramBytesUsed() { long total = SHALLOW_RAM_BYTES_USED; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java index 9416f082b0..ea0139c4ea 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java @@ -18,8 +18,16 @@ import com.nvidia.cuvs.spi.CuVSProvider; import com.nvidia.cuvs.spi.CuVSServiceProvider; -/** A provider that creates instances of FilterCuVSProvider. */ +/** + * A provider that creates instances of FilterCuVSProvider. + * + * @since 25.10 + */ public class FilterCuVSServiceProvider extends CuVSServiceProvider { + + /** + * Initialize and return an CuVSProvider provided by this provider. + */ @Override public CuVSProvider get(CuVSProvider builtinProvider) { return new FilterCuVSProvider(builtinProvider); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java index 7235ec97ab..2febd27e63 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java @@ -24,6 +24,11 @@ import org.apache.lucene.util.hnsw.HnswGraph; import org.apache.lucene.util.hnsw.NeighborArray; +/** + * This class holds the in-memory representation of the HNSW graph + * + * @since 25.10 + */ public class GPUBuiltHnswGraph extends HnswGraph { private final int size; @@ -37,7 +42,14 @@ public class GPUBuiltHnswGraph extends HnswGraph { // Layer 0 is special - it contains all nodes private final NeighborArray[] layer0Neighbors; - // Multi-layer constructor that supports arbitrary number of layers + /** + * Multi-layer constructor that supports arbitrary number of layers. + * + * @param size the size of the dataset + * @param dimensions the vector dimension + * @param layerNodes the nodes on the layer + * @param layerAdjacencies adjacency list + */ public GPUBuiltHnswGraph( int size, int dimensions, List layerNodes, List layerAdjacencies) { @@ -60,6 +72,13 @@ public GPUBuiltHnswGraph( } } + /** + * Fills the neighbor array using the adjacency matrix. + * + * @param adjacency instance of adjacency CuVSMatrix + * @param size the number of nodes + * @return the NeighborArray + */ private NeighborArray[] fillNeighborArray(CuVSMatrix adjacency, int size) { NeighborArray[] neighbors = new NeighborArray[size]; for (int i = 0; i < size; i++) { @@ -76,6 +95,9 @@ private NeighborArray[] fillNeighborArray(CuVSMatrix adjacency, int size) { return neighbors; } + /** + * Get all nodes on a given level as node 0th ordinals. + */ public NodesIterator getNodesOnLevel(int level) { if (level == 0) { return new Level0NodesIterator(size); @@ -87,6 +109,13 @@ public NodesIterator getNodesOnLevel(int level) { } } + /** + * Get the neighbors for the node and the level it resides. + * + * @param level the level + * @param node the node + * @return an instance of NeighborArray + */ public NeighborArray getNeighbors(int level, int node) { if (level == 0 && node < size) { return layer0Neighbors[node]; @@ -109,6 +138,9 @@ public NeighborArray getNeighbors(int level, int node) { private int currentLevel = -1; private int neighborIndex = -1; + /** + * Move the pointer to exactly the given level's target. + */ @Override public void seek(int level, int target) { currentLevel = level; @@ -116,6 +148,9 @@ public void seek(int level, int target) { neighborIndex = -1; } + /** + * Iterates over the neighbor list. + */ @Override public int nextNeighbor() { if (currentLevel == 0 @@ -144,6 +179,9 @@ public int nextNeighbor() { return NO_MORE_DOCS; } + /** + * Returns graph's entry point on the top level. + */ @Override public int entryNode() { // Entry node should be from the highest layer @@ -160,6 +198,9 @@ public int entryNode() { return 0; // Default to node 0 for single-layer graphs } + /** + * returns M, the maximum number of connections for a node. + */ @Override public int maxConn() { // Return the maximum degree across all nodes in layer 0 @@ -172,6 +213,9 @@ public int maxConn() { return max; } + /** + * Returns the neighbor count. + */ @Override public int neighborCount() { if (currentLevel == 0 @@ -244,14 +288,27 @@ public int consume(int[] dest) { } } + /** + * Returns the number of nodes in the graph. + */ public int size() { return size; } + /** + * Returns the number of levels in the HNSW graph. + * + * @return the number of levels + */ public int numLevels() { return numLevels; } + /** + * Gets the vector dimension. + * + * @return the vector dimension + */ public int dimensions() { return dimensions; } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java index 483e18d394..105fe7c748 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java @@ -23,7 +23,11 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.util.RamUsageEstimator; -/** CuVS based fields writer */ +/** + * cuVS based fields writer + * + * @since 25.10 + */ /*package-private*/ class GPUFieldWriter extends KnnFieldVectorsWriter { private static final long SHALLOW_SIZE = @@ -39,6 +43,9 @@ public GPUFieldWriter( this.flatFieldVectorsWriter = flatFieldVectorsWriter; } + /** + * Add new docID with its vector value to the given field for indexing. + */ @Override public void addValue(int docID, float[] vectorValue) throws IOException { if (docID == lastDocID) { @@ -50,28 +57,52 @@ public void addValue(int docID, float[] vectorValue) throws IOException { flatFieldVectorsWriter.addValue(docID, vectorValue); } + /** + * Gets the list of float vectors. + * + * @return a list of float vectors + */ List getVectors() { return flatFieldVectorsWriter.getVectors(); } + /** + * Gets the field info that holds the description of the field. + * + * @return an instance of FieldInfo + */ FieldInfo fieldInfo() { return fieldInfo; } + /** + * Gets the docsWithFieldSet for the field writer. + * + * @return an instance of DocsWithFieldSet + */ DocsWithFieldSet getDocsWithFieldSet() { return flatFieldVectorsWriter.getDocsWithFieldSet(); } + /** + * Used to copy values being indexed to internal storage. + */ @Override public float[] copyValue(float[] vectorValue) { throw new UnsupportedOperationException(); } + /** + * Returns the memory usage of this object in bytes. + */ @Override public long ramBytesUsed() { return SHALLOW_SIZE + flatFieldVectorsWriter.ramBytesUsed(); } + /** + * Returns a string containing the field name and number. + */ @Override public String toString() { StringBuilder sb = new StringBuilder(this.getClass().getSimpleName()); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java index 6d86744dd3..00ca336d20 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java @@ -21,7 +21,11 @@ import java.io.IOException; import java.util.Objects; -/** This class holds references to the actual CuVS Index (Cagra, Brute force, etc.) */ +/** + * This class holds references to the cuVS Index (Cagra, Brute force, etc.) + * + * @since 25.10 + */ public class GPUIndex implements Closeable { private final CagraIndex cagraIndex; private final BruteForceIndex bruteforceIndex; @@ -31,6 +35,15 @@ public class GPUIndex implements Closeable { private String segmentName; private volatile boolean closed; + /** + * Initializes an instance of {@link GPUIndex} + * + * @param segmentName the name of the segment + * @param fieldName the field name + * @param cagraIndex reference to the CagraIndex + * @param maxDocs the maximum documents + * @param bruteforceIndex reference to the BruteForceIndex + */ public GPUIndex( String segmentName, String fieldName, @@ -47,39 +60,76 @@ public GPUIndex( this.maxDocs = maxDocs; } + /** + * Initializes an instance of {@link GPUIndex} + * + * @param cagraIndex reference to the CagraIndex instance + * @param bruteforceIndex reference to the BruteForceIndex instance + */ public GPUIndex(CagraIndex cagraIndex, BruteForceIndex bruteforceIndex) { this.cagraIndex = cagraIndex; this.bruteforceIndex = bruteforceIndex; } + /** + * Gets the reference to the CAGRA index + * + * @return an instance of CagraIndex + */ public CagraIndex getCagraIndex() { ensureOpen(); return cagraIndex; } + /** + * Gets the reference to the Bruteforce index + * + * @return an instance of BruteForceIndex + */ public BruteForceIndex getBruteforceIndex() { ensureOpen(); return bruteforceIndex; } + /** + * Gets the field name + * + * @return field name + */ public String getFieldName() { return fieldName; } + /** + * Gets the segment name + * + * @return segment name + */ public String getSegmentName() { return segmentName; } + /** + * Gets the max docs + * + * @return the max docs + */ public int getMaxDocs() { return maxDocs; } + /** + * Throws {@link IllegalArgumentException} if the index is closed + */ private void ensureOpen() { if (closed) { throw new IllegalStateException("index is closed"); } } + /** + * Closes this stream and releases any resources associated with it. + */ @Override public void close() throws IOException { if (closed) { @@ -89,6 +139,11 @@ public void close() throws IOException { destroyIndices(); } + /** + * Closes the cuVS indexes. + * + * @throws IOException + */ private void destroyIndices() throws IOException { try { if (cagraIndex != null) { diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java index 64f70811a0..d69e9fe733 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java @@ -25,12 +25,26 @@ import org.apache.lucene.search.knn.KnnCollectorManager; import org.apache.lucene.util.Bits; -/** Query on GPU only */ +/** + * Extends upon KnnFloatVectorQuery for GPU-only search. + * + * @since 25.10 + */ public class GPUKnnFloatVectorQuery extends KnnFloatVectorQuery { private final int iTopK; private final int searchWidth; + /** + * Initializes {@link GPUKnnFloatVectorQuery} + * + * @param field the vector field name + * @param target the vector target query + * @param k the topK value + * @param filter instance of the Query + * @param iTopK the iTopK value + * @param searchWidth the search width + */ public GPUKnnFloatVectorQuery( String field, float[] target, int k, Query filter, int iTopK, int searchWidth) { super(field, target, k, filter); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java index ca30d3d3ed..64bdcc9676 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java @@ -23,8 +23,12 @@ import org.apache.lucene.search.TotalHits; import org.apache.lucene.search.knn.KnnSearchStrategy; -/** KnnCollector for CuVS */ -/*package-private*/ class GPUPerLeafCuVSKnnCollector implements KnnCollector { +/** + * KnnCollector for cuVS used for search on the GPU. + * + * @since 25.10 + */ +class GPUPerLeafCuVSKnnCollector implements KnnCollector { public List scoreDocs; public int topK = 0; @@ -32,6 +36,13 @@ public int searchWidth = 1; // TODO getter, no setter public int results = 0; + /** + * Initializes {@link GPUPerLeafCuVSKnnCollector} + * + * @param topK the topk value + * @param iTopK the iTopK value + * @param searchWidth the search width + */ public GPUPerLeafCuVSKnnCollector(int topK, int iTopK, int searchWidth) { super(); this.topK = topK; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java index 1beead66b7..a156614db4 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java @@ -19,18 +19,30 @@ import java.io.InputStream; import org.apache.lucene.store.IndexInput; -/** InputStream for reading from an IndexInput. */ +/** + * InputStream for reading from an IndexInput. + * + * @since 25.10 + */ final class IndexInputInputStream extends InputStream { final IndexInput in; long pos = 0; final long limit; + /** + * Initializes the {@link IndexInputInputStream} + * + * @param in instance of IndexInput + */ IndexInputInputStream(IndexInput in) { this.in = in; this.limit = in.length(); } + /** + * Reads the next byte of data from the input stream. + */ @Override public int read() throws IOException { if (pos >= limit) { @@ -40,6 +52,9 @@ public int read() throws IOException { return in.readByte(); } + /** + * Reads up to len bytes of data from the input stream into an array of bytes. + */ @Override public int read(byte[] b, int off, int len) throws IOException { if (len <= 0) { diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java index 53283ebf24..982ca6e130 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java @@ -19,7 +19,11 @@ import java.io.OutputStream; import org.apache.lucene.store.IndexOutput; -/** OutputStream for writing into an IndexOutput */ +/** + * OutputStream for writing into an IndexOutput + * + * @since 25.10 + */ final class IndexOutputOutputStream extends OutputStream { static final int DEFAULT_BUFFER_SIZE = 8192; @@ -29,16 +33,30 @@ final class IndexOutputOutputStream extends OutputStream { final byte[] buffer; int pos; + /** + * Initializes the {@link IndexOutputOutputStream}. + * + * @param out instance of IndexOutput + */ IndexOutputOutputStream(IndexOutput out) { this(out, DEFAULT_BUFFER_SIZE); } + /** + * Initializes the {@link IndexOutputOutputStream}. + * + * @param out instance of IndexOutput + * @param bufferSize the size of buffer to use + */ IndexOutputOutputStream(IndexOutput out, int bufferSize) { this.out = out; this.bufferSize = bufferSize; this.buffer = new byte[bufferSize]; } + /** + * Writes the specified byte to this output stream. + */ @Override public void write(int b) throws IOException { buffer[pos] = (byte) b; @@ -48,6 +66,9 @@ public void write(int b) throws IOException { } } + /** + * Writes len bytes from the specified byte array starting at offset off to this output stream. + */ @Override public void write(byte[] b, int offset, int length) throws IOException { if (pos != 0) { @@ -56,12 +77,18 @@ public void write(byte[] b, int offset, int length) throws IOException { out.writeBytes(b, offset, length); } + /** + * Flushes this output stream and forces any buffered output bytes to be written out. + */ @Override public void flush() throws IOException { out.writeBytes(buffer, 0, pos); pos = 0; } + /** + * Closes this output stream and releases any system resources associated with this stream. + */ @Override public void close() throws IOException { this.flush(); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java index 3a18aaef41..b7ceaa3f49 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java @@ -35,7 +35,11 @@ import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; -/** CuVS based KnnVectorsFormat for GPU acceleration */ +/** + * cuVS based KnnVectorsFormat for indexing on GPU and searching on the CPU. + * + * @since 25.10 + */ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { private static final Logger log = @@ -61,13 +65,13 @@ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { private final int cuvsWriterThreads; private final int intGraphDegree; private final int graphDegree; - private final int hnswLayers; // Number of layers to create in CAGRA->HNSW conversion + private final int hnswLayers; private final int maxConn; private final int beamWidth; /** - * Creates a Lucene99AcceleratedHNSWVectorsFormat, with default values. + * Initializes {@link Lucene99AcceleratedHNSWVectorsFormat} with default values. * * @throws LibraryException if the native library fails to load */ @@ -82,9 +86,14 @@ public Lucene99AcceleratedHNSWVectorsFormat() { } /** - * Creates a Lucene99AcceleratedHNSWVectorsFormat, with the given threads, graph degree, etc. + * Initializes {@link Lucene99AcceleratedHNSWVectorsFormat} with the given threads, graph degree, etc. * - * @throws LibraryException if the native library fails to load + * @param cuvsWriterThreads number of cuVS threads to use while building the CAGRA index + * @param intGraphDegree the intermediate graph degree while building the CAGRA index + * @param graphDegree the graph degree to use while building the CAGRA index + * @param hnswLayers the number of HNSW layers to construct in the HNSW graph + * @param maxConn the maximum connections for the HNSW graph + * @param beamWidth the beam width to use while building the HNSW graph */ public Lucene99AcceleratedHNSWVectorsFormat( int cuvsWriterThreads, @@ -102,6 +111,9 @@ public Lucene99AcceleratedHNSWVectorsFormat( this.beamWidth = beamWidth; } + /** + * Returns a KnnVectorsWriter to write the vectors to the index. + */ @Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { var flatWriter = flatVectorsFormat.fieldsWriter(state); @@ -118,16 +130,25 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException } } + /** + * Returns a KnnVectorsReader to read the vectors from the index. + */ @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state)); } + /** + * Returns the maximum number of vector dimensions supported by this codec for the given field name. + */ @Override public int getMaxDimensions(String fieldName) { return maxDimensions; } + /** + * Returns a string containing the meta information like hnsw layers, graph degree etc. + */ @Override public String toString() { StringBuilder sb = new StringBuilder(this.getClass().getSimpleName()); @@ -140,19 +161,36 @@ public String toString() { return sb.toString(); } + /** + * Gets the instance of CuVSResources + * + * @return the instance of CuVSResources + */ public static CuVSResources getResources() { return resources; } + /** + * Sets the instance of CuVSResources + * + * @param resources the instance of CuVSResources to set + */ public static void setResources(CuVSResources resources) { Lucene99AcceleratedHNSWVectorsFormat.resources = resources; } - /** Tells whether the platform supports cuVS. */ + /** + * Tells whether the platform supports cuVS. + * + * @return if cuVS supported or not + */ public static boolean supported() { return resources != null; } + /** + * Checks if cuVS supported and throws {@link UnsupportedOperationException} otherwise. + */ public static void checkSupported() { if (!supported()) { throw new UnsupportedOperationException(); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index 14c2ddbed9..f516a7c209 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -70,8 +70,10 @@ import org.apache.lucene.util.packed.DirectMonotonicWriter; /** - * KnnVectorsWriter for CuVS, responsible for merge and flush of vectors into - * GPU + * This class extends upon the KnnVectorsWriter to + * enable the creation of GPU-based accelerated vector search indexes. + * + * @since 25.10 */ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { @@ -99,6 +101,18 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { private String vemFileName; private String vexFileName; + /** + * Initializes {@link Lucene99AcceleratedHNSWVectorsWriter} + * + * @param state instance of the {@link org.apache.lucene.index.SegmentWriteState} + * @param cuvsWriterThreads number of cuVS threads to use while building the intermediate CAGRA index + * @param intGraphDegree the intermediate graph degree to use while building the CAGRA index + * @param graphDegree the graph degree to use while building the CAGRA index + * @param hnswLayers the number of hnsw layers to construct while building the HNSW graph + * @param resources instance of the {@link com.nvidia.cuvs.CuVSResources} + * @param flatVectorsWriter instance of the {@link org.apache.lucene.codecs.hnsw.FlatVectorsWriter} + * @throws IOException IOException + */ public Lucene99AcceleratedHNSWVectorsWriter( SegmentWriteState state, int cuvsWriterThreads, @@ -151,6 +165,9 @@ public Lucene99AcceleratedHNSWVectorsWriter( } } + /** + * Add new field for indexing. + */ @Override public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { var encoding = fieldInfo.getVectorEncoding(); @@ -165,6 +182,13 @@ public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException return writer; } + /** + * Utility method for building index metadata information string object. + * + * @param size index size + * @param args additional metadata information + * @return the string representation of the metadata information + */ static String indexMsg(int size, int... args) { StringBuilder sb = new StringBuilder("cagra index params"); sb.append(": size=").append(size); @@ -175,6 +199,11 @@ static String indexMsg(int size, int... args) { return sb.toString(); } + /** + * Builds an instance of CagraIndexParams. + * + * @return instance of CagraIndexParams + */ private CagraIndexParams cagraIndexParams() { return new CagraIndexParams.Builder() .withNumWriterThreads(cuvsWriterThreads) @@ -184,12 +213,24 @@ private CagraIndexParams cagraIndexParams() { .build(); } + /** + * A utility method to print info/debugging messages using InfoStream. + * + * @param msg the debugging message to print + */ private void info(String msg) { if (infoStream.isEnabled(CUVS_COMPONENT)) { infoStream.message(CUVS_COMPONENT, msg); } } + /** + * Builds the intermediate CAGRA index and builds and writes the HNSW index + * + * @param fieldInfo instance of FieldInfo that has the field description + * @param vectors vectors to index + * @throws IOException + */ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOException { if (vectors.size() == 0) { @@ -369,6 +410,19 @@ private CuVSMatrix buildCagraGraphForSubset(float[][] vectors, int[] selectedNod return CuVSMatrix.ofArray(remappedAdjacency); } + /** + * Writes the meta information for the index. + * + * @param vectorIndex instance of IndexOutput + * @param meta instance of IndexOutput + * @param field instance of FieldInfo + * @param vectorIndexOffset vector index offset + * @param vectorIndexLength vector index length + * @param count the count of vectors + * @param graph instance of HnswGraph + * @param graphLevelNodeOffsets graph level node offsets + * @throws IOException I/O Exceptions + */ private void writeMeta( IndexOutput vectorIndex, IndexOutput meta, @@ -435,6 +489,14 @@ private void writeMeta( } } + /** + * Returns a 2D array of offsets (information written while writing the meta info) + * + * @param graph instance of GPUBuiltHnswGraph + * @param vectorIndex instance of IndexOutput + * @return a 2D array of offsets + * @throws IOException I/O Exceptions + */ private int[][] writeGraph(GPUBuiltHnswGraph graph, IndexOutput vectorIndex) throws IOException { // write vectors' neighbors on each level into the vectorIndex file int countOnLevel0 = graph.size(); @@ -486,6 +548,9 @@ private int[][] writeGraph(GPUBuiltHnswGraph graph, IndexOutput vectorIndex) thr return offsets; } + /** + * Build the indexes and writes it to the disk. + */ @Override public void flush(int maxDoc, DocMap sortMap) throws IOException { flatVectorsWriter.flush(maxDoc, sortMap); @@ -498,10 +563,23 @@ public void flush(int maxDoc, DocMap sortMap) throws IOException { } } + /** + * Builds the index and writes it to the disk. + * + * @param fieldData + * @throws IOException + */ private void writeField(GPUFieldWriter fieldData) throws IOException { writeFieldInternal(fieldData.fieldInfo(), fieldData.getVectors()); } + /** + * Builds the index and writes it to the disk. + * + * @param fieldData instance of GPUFieldWriter + * @param sortMap instance of the DocMap + * @throws IOException + */ private void writeSortingField(GPUFieldWriter fieldData, Sorter.DocMap sortMap) throws IOException { @@ -517,6 +595,13 @@ private void writeSortingField(GPUFieldWriter fieldData, Sorter.DocMap sortMap) writeFieldInternal(fieldData.fieldInfo(), sortedVectors); } + /** + * Builds and writes a single vector graph. + * + * @param fieldInfo instance of FieldInfo + * @param vectors the list of float vectors + * @throws IOException I/O Exceptions + */ private void writeSingleVectorGraph(FieldInfo fieldInfo, List vectors) throws IOException { // Workaround for CAGRA not supporting single vector indexes @@ -577,6 +662,12 @@ private GPUBuiltHnswGraph createSingleVectorHnswGraph(int size, int dimensions) return new GPUBuiltHnswGraph(size, dimensions, layerNodes, layerAdjacencies); } + /** + * Writes an empty meta information for the field. + * + * @param fieldInfo instance of FieldInfo + * @throws IOException I/O Exceptions + */ private void writeEmpty(FieldInfo fieldInfo) throws IOException { writeMeta(null, hnswMeta, fieldInfo, 0, 0, 0, null, null); } @@ -590,6 +681,13 @@ static int distFuncToOrd(VectorSimilarityFunction func) { throw new IllegalArgumentException("invalid distance function: " + func); } + /** + * Uses the CAGRA API to merge the CAGRA indexes. + * + * @param fieldInfo instance of FieldInfo + * @param mergeState instance of MergeState + * @throws IOException I/O Exceptions + */ private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throws IOException { try { @@ -631,7 +729,7 @@ private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throw /** * Creates List from merged vectors - * */ + */ private List createListFromMergedVectors(FloatVectorValues mergedVectorValues) throws IOException { List vectors = new ArrayList(); @@ -705,6 +803,9 @@ private void writeMergedCagraIndex(FieldInfo fieldInfo, CagraIndex mergedIndex, } } + /** + * Write field for merging. + */ @Override public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { flatVectorsWriter.mergeOneField(fieldInfo, mergeState); @@ -728,6 +829,9 @@ public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOE } } + /** + * Called once at the end before close. + */ @Override public void finish() throws IOException { if (finished) { @@ -750,11 +854,17 @@ public void finish() throws IOException { } } + /** + * Closes the resources. + */ @Override public void close() throws IOException { IOUtils.close(cuvsIndex, hnswMeta, hnswVectorIndex, flatVectorsWriter); } + /** + * Returns the memory usage of this object in bytes. + */ @Override public long ramBytesUsed() { long total = SHALLOW_RAM_BYTES_USED; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java index a44dc03a6d..45d58f12a4 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java @@ -22,10 +22,21 @@ import java.util.List; import java.util.logging.Logger; +/** + * This class provides common static utility methods. + * + * @since 25.10 + */ public class Utils { static final Logger log = Logger.getLogger(Utils.class.getName()); + /** + * A utility method that throws specific types of throwable objects based on types. + * + * @param t the throwable object + * @throws IOException + */ static void handleThrowable(Throwable t) throws IOException { switch (t) { case IOException ioe -> throw ioe; @@ -36,15 +47,15 @@ static void handleThrowable(Throwable t) throws IOException { } /** - * A method to build a {@link CuVSMatrix} from a list of float vectors. + * A method to build a CuVSMatrix from a list of float vectors. * - * Uses {@link CuVSMatrix.Builder} to copy vectors directly to device memory + * Uses CuVSMatrix.Builder to copy vectors directly to device memory * without creating intermediate heap arrays. * * @param data The float vectors * @param dimensions The number float elements in each vector * @param resources The CuVS resources for device matrix creation - * @return an instance of {@link CuVSMatrix} + * @return an instance of CuVSMatrix */ static CuVSMatrix createFloatMatrix(List data, int dimensions, CuVSResources resources) { // Use Builder pattern to avoid intermediate float[][] allocation @@ -64,10 +75,21 @@ static CuVSMatrix createFloatMatrix(List data, int dimensions, CuVSReso return builder.build(); } + /** + * A utility method to convert nanoseconds to milliseconds. + * + * @param nanos + * @return milliseconds + */ static long nanosToMillis(long nanos) { return Duration.ofNanos(nanos).toMillis(); } + /** + * Creates an instance of CuVSResources. + * + * @return an instance of CuVSResources + */ static CuVSResources cuVSResourcesOrNull() { try { System.loadLibrary("cudart"); @@ -87,6 +109,13 @@ static CuVSResources cuVSResourcesOrNull() { return null; } + /** + * A utility method that conditionally ignores certain throwable objects + * + * @param t the throwable object + * @param msg the message to check + * @throws IOException + */ static void handleThrowableWithIgnore(Throwable t, String msg) throws IOException { if (t.getMessage().contains(msg)) { return; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java index 043bfe0818..0271dc114a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java @@ -14,5 +14,9 @@ * limitations under the License. */ -/** CuVS based fast vector search */ +/** + * GPU-based accelerated vector search pluggable library for Apache Lucene search platforms. + * + * @since 25.10 + */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index 1e811e9728..d256955b07 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -23,11 +23,12 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.lucene101.Lucene101Codec; -/** CuVS based codec for GPU based vector search +/** + * cuVS based codec for GPU based vector search + * cuVS serialization formats are in experimental phase and hence backward compatibility cannot be guaranteed. * - * @apiNote cuVS serialization formats are in experimental phase and hence backward compatibility cannot be guaranteed. - * - * */ + * @since 25.10 + */ public class CuVS2510GPUSearchCodec extends FilterCodec { private static final Logger log = Logger.getLogger(CuVS2510GPUSearchCodec.class.getName()); @@ -40,10 +41,19 @@ public class CuVS2510GPUSearchCodec extends FilterCodec { private KnnVectorsFormat format; + /** + * Default constructor for {@link CuVS2510GPUSearchCodec} + */ public CuVS2510GPUSearchCodec() { this(NAME, new Lucene101Codec()); } + /** + * Constructor for the {@link CuVS2510GPUSearchCodec} + * + * @param name the name of the codec + * @param delegate the delegate codec + */ public CuVS2510GPUSearchCodec(String name, Codec delegate) { super(name, delegate); try { @@ -59,11 +69,21 @@ public CuVS2510GPUSearchCodec(String name, Codec delegate) { } } + /** + * Get the configured {@link KnnVectorsFormat} + * + * @return the instance of the {@link KnnVectorsFormat} + */ @Override public KnnVectorsFormat knnVectorsFormat() { return format; } + /** + * Set the {@link KnnVectorsFormat} + * + * @param format the {@link KnnVectorsFormat} to set + */ public void setKnnFormat(KnnVectorsFormat format) { this.format = format; } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java index facd93d8e2..264e462ab4 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java @@ -25,7 +25,11 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.lucene101.Lucene101Codec; -/** CuVS based codec for GPU based vector search */ +/** + * CuVS based codec for GPU based vector search + * + * @since 25.10 + */ public class Lucene101AcceleratedHNSWCodec extends FilterCodec { private static final Logger log = Logger.getLogger(Lucene101AcceleratedHNSWCodec.class.getName()); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java index a2c2e1f2a3..5cbceb7684 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java @@ -47,8 +47,8 @@ import org.apache.lucene.store.FSDirectory; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; -import org.junit.AfterClass; -import org.junit.BeforeClass; +import org.junit.After; +import org.junit.Before; import org.junit.Test; @SuppressSysoutChecks(bugUrl = "") @@ -59,8 +59,8 @@ public class TestCagraToHnswSerializationAndSearch extends LuceneTestCase { private static Random random; private static Path indexDirPath; - @BeforeClass - public static void beforeClass() throws Exception { + @Before + public void beforeTest() throws Exception { assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); // Fixed seed so that we can validate against the same result. random = new Random(222); @@ -199,8 +199,8 @@ public void testSingleVectorIndex() throws Exception { } } - @AfterClass - public static void afterClass() throws Exception { + @After + public void afterTest() throws Exception { File indexDirPathFile = indexDirPath.toFile(); if (indexDirPathFile.exists() && indexDirPathFile.isDirectory()) { FileUtils.deleteDirectory(indexDirPathFile); From f15d934aafd6e20d6b4022a6cba11d3f577da686 Mon Sep 17 00:00:00 2001 From: MithunR Date: Fri, 31 Oct 2025 06:04:34 -0700 Subject: [PATCH 030/105] Fix cuvs-lucene Github artifact publication (NVIDIA/cuvs-lucene#34) This commit fixes the publication of cuvs-lucene artifacts to Github. The Github actions build-logs indicate that while the JAR is built correctly, it doesn't seem to be uploaded properly as a Github artifact. It appears to be the result of a minor typo in the artifact's base dir. This commit fixes the typo, and allows the artifact to be published. Authors: - MithunR (https://github.com/mythrocks) Approvers: - Nate Rock (https://github.com/rockhowse) URL: https://github.com/rapidsai/cuvs-lucene/pull/34 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 5494c906b4..ca5c9da655 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -50,6 +50,6 @@ jobs: date: ${{ inputs.date }} container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/build_java.sh" - file_to_upload: "java/cuvs-java/target/" + file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" sha: ${{ inputs.sha }} diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 95bf5e4706..9a289e0c05 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -75,7 +75,7 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" - file_to_upload: "java/cuvs-java/target/" + file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" telemetry-summarize: # This job must use a self-hosted runner to record telemetry traces. From 2e5dc2b7ce1d6f020438eab25756da541c804856 Mon Sep 17 00:00:00 2001 From: MithunR Date: Thu, 13 Nov 2025 17:24:51 -0600 Subject: [PATCH 031/105] Add support to build the sources jar. (NVIDIA/cuvs-lucene#44) This commit adds support to build the `cuvs-lucene` source jar. Both `sources` and `javadoc` jar artifacts are required for publication to Maven Central. Authors: - MithunR (https://github.com/mythrocks) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/44 --- java/cuvs-lucene/pom.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 38ab629b26..3c9ed1bf9e 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -129,6 +129,19 @@ 0 + + org.apache.maven.plugins + maven-source-plugin + 3.3.1 + + + attach-sources + + jar-no-fork + + + + org.apache.maven.plugins maven-javadoc-plugin From b00e7d073c9861beea8bb4cfb4d9d3586a52b4f2 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Mon, 17 Nov 2025 14:25:47 -0500 Subject: [PATCH 032/105] Use SPDX for all copyright headers (NVIDIA/cuvs-lucene#47) Fixes NVIDIA/cuvs-lucene#46 and NVIDIA/cuvs-lucene#48. This commit brings the copyright headers for `cuvs-lucene` in alignment with the other RAPIDS projects. Issue: https://github.com/rapidsai/build-infra/issues/297 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - MithunR (https://github.com/mythrocks) - Kyle Edwards (https://github.com/KyleFromNVIDIA) - Ben Frederickson (https://github.com/benfred) URL: https://github.com/rapidsai/cuvs-lucene/pull/47 --- java/cuvs-lucene/.gitignore | 3 +++ java/cuvs-lucene/.pre-commit-config.yaml | 11 ++++++++--- java/cuvs-lucene/build.sh | 3 ++- java/cuvs-lucene/ci/build_java.sh | 4 +++- java/cuvs-lucene/ci/check_style.sh | 4 +++- java/cuvs-lucene/ci/release/update-version.sh | 5 ++++- java/cuvs-lucene/ci/test_java.sh | 4 +++- java/cuvs-lucene/dependencies.yaml | 3 +++ java/cuvs-lucene/license-header.txt | 15 ++------------- java/cuvs-lucene/pom.xml | 7 +++++++ .../cuvs/lucene/CuVS2510GPUVectorsFormat.java | 15 ++------------- .../cuvs/lucene/CuVS2510GPUVectorsReader.java | 15 ++------------- .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 15 ++------------- .../nvidia/cuvs/lucene/FilterCuVSProvider.java | 15 ++------------- .../cuvs/lucene/FilterCuVSServiceProvider.java | 15 ++------------- .../com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java | 15 ++------------- .../com/nvidia/cuvs/lucene/GPUFieldWriter.java | 15 ++------------- .../java/com/nvidia/cuvs/lucene/GPUIndex.java | 15 ++------------- .../cuvs/lucene/GPUKnnFloatVectorQuery.java | 15 ++------------- .../cuvs/lucene/GPUPerLeafCuVSKnnCollector.java | 15 ++------------- .../nvidia/cuvs/lucene/IndexInputInputStream.java | 15 ++------------- .../cuvs/lucene/IndexOutputOutputStream.java | 15 ++------------- .../Lucene99AcceleratedHNSWVectorsFormat.java | 15 ++------------- .../Lucene99AcceleratedHNSWVectorsWriter.java | 15 ++------------- .../main/java/com/nvidia/cuvs/lucene/Utils.java | 15 ++------------- .../java/com/nvidia/cuvs/lucene/package-info.java | 15 ++------------- .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 15 ++------------- .../lucene/Lucene101AcceleratedHNSWCodec.java | 15 ++------------- .../TestCagraToHnswSerializationAndSearch.java | 15 ++------------- ...wSerializationAndSearchWithFallbackWriter.java | 15 ++------------- .../cuvs/lucene/TestCuVSDeletedDocuments.java | 15 ++------------- .../java/com/nvidia/cuvs/lucene/TestCuVSGaps.java | 15 ++------------- .../lucene/TestCuVSRandomizedVectorSearch.java | 15 ++------------- .../nvidia/cuvs/lucene/TestCuVSVectorsFormat.java | 15 ++------------- .../cuvs/lucene/TestIndexOutputOutputStream.java | 15 ++------------- .../java/com/nvidia/cuvs/lucene/TestMerge.java | 15 ++------------- .../java/com/nvidia/cuvs/lucene/TestUtils.java | 15 ++------------- 37 files changed, 92 insertions(+), 372 deletions(-) diff --git a/java/cuvs-lucene/.gitignore b/java/cuvs-lucene/.gitignore index 7738e864cb..32d6a5b09e 100644 --- a/java/cuvs-lucene/.gitignore +++ b/java/cuvs-lucene/.gitignore @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + *.jar target **/.DS_Store diff --git a/java/cuvs-lucene/.pre-commit-config.yaml b/java/cuvs-lucene/.pre-commit-config.yaml index 8d8052c38c..03ee2970c1 100644 --- a/java/cuvs-lucene/.pre-commit-config.yaml +++ b/java/cuvs-lucene/.pre-commit-config.yaml @@ -1,4 +1,5 @@ -# Copyright (c) 2022-2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -9,15 +10,19 @@ repos: - id: end-of-file-fixer - id: check-symlinks - repo: https://github.com/rapidsai/pre-commit-hooks - rev: v0.7.0 + rev: v1.2.1 hooks: - id: verify-copyright + name: verify-copyright + args: [--fix, --spdx] files: | (?x) [.](cmake|cpp|cu|cuh|h|hpp|sh|pxd|py|pyx|rs|java)$| CMakeLists[.]txt$| CMakeLists_standalone[.]txt$| - meta[.]yaml$ + meta[.]yaml$| + dependencies[.]yaml$| + ^[.]pre-commit-config[.]yaml$ - repo: https://github.com/rapidsai/dependency-file-generator rev: v1.20.0 hooks: diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index a68169555d..9f65feac36 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -1,6 +1,7 @@ #!/bin/bash -# Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 set -e -u -o pipefail diff --git a/java/cuvs-lucene/ci/build_java.sh b/java/cuvs-lucene/ci/build_java.sh index 43536e23ae..2f61f4b5a7 100755 --- a/java/cuvs-lucene/ci/build_java.sh +++ b/java/cuvs-lucene/ci/build_java.sh @@ -1,5 +1,7 @@ #!/bin/bash -# Copyright (c) 2025, NVIDIA CORPORATION. + +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/java/cuvs-lucene/ci/check_style.sh b/java/cuvs-lucene/ci/check_style.sh index 07126990f9..b50c2da01c 100755 --- a/java/cuvs-lucene/ci/check_style.sh +++ b/java/cuvs-lucene/ci/check_style.sh @@ -1,5 +1,7 @@ #!/bin/bash -# Copyright (c) 2020-2025, NVIDIA CORPORATION. + +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/java/cuvs-lucene/ci/release/update-version.sh b/java/cuvs-lucene/ci/release/update-version.sh index e1e769073b..0404480ee0 100755 --- a/java/cuvs-lucene/ci/release/update-version.sh +++ b/java/cuvs-lucene/ci/release/update-version.sh @@ -1,5 +1,8 @@ #!/bin/bash -# Copyright (c) 2025, NVIDIA CORPORATION. + +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + ######################## # CUVS Version Updater # ######################## diff --git a/java/cuvs-lucene/ci/test_java.sh b/java/cuvs-lucene/ci/test_java.sh index 783b73efda..db17f1b62c 100755 --- a/java/cuvs-lucene/ci/test_java.sh +++ b/java/cuvs-lucene/ci/test_java.sh @@ -1,5 +1,7 @@ #!/bin/bash -# Copyright (c) 2025, NVIDIA CORPORATION. + +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 set -euo pipefail diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index b9eb3fcfd9..322f0be85f 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -1,3 +1,6 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + # Dependency list for https://github.com/rapidsai/dependency-file-generator files: all: diff --git a/java/cuvs-lucene/license-header.txt b/java/cuvs-lucene/license-header.txt index 2001ad3d3a..e49ea20ab7 100644 --- a/java/cuvs-lucene/license-header.txt +++ b/java/cuvs-lucene/license-header.txt @@ -1,15 +1,4 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 3c9ed1bf9e..10091cbc7b 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -1,3 +1,10 @@ + + diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java index d79cb45678..94ccd1b8c6 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java index ae9ef94909..56a9880d3d 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index c55835cab9..bfe25ed8ac 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index 81d248b2bc..01752d10cb 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java index ea0139c4ea..83cebafa1a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java index 2febd27e63..b0b6286d2b 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java index 105fe7c748..92531bb86e 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java index 00ca336d20..76cd916b6d 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java index d69e9fe733..7dc984b286 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java index 64bdcc9676..9d18b5167f 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java index a156614db4..3ab1412295 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java index 982ca6e130..b5fc120b7d 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java index b7ceaa3f49..c3fd4ed94a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index f516a7c209..a0e4437197 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java index 45d58f12a4..301d0837d7 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java index 0271dc114a..01f086761a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ /** diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index d256955b07..3e5085a8c8 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java index 264e462ab4..3dd698dc3c 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java index 5cbceb7684..2a11ff6369 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java index 792aca9654..83758708ef 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java index 1774587f25..696e6bd0e3 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java index bba34bfaf3..38035c1c76 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java index add0fe2043..6cbd646491 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java index cf78b05171..1b3de41d65 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java index 5f6ff3aa57..04b22b3d0f 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java index 27b01e0469..676faff37e 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestUtils.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestUtils.java index 8bd8339ffe..05e37f8480 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestUtils.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestUtils.java @@ -1,17 +1,6 @@ /* - * Copyright (c) 2025, NVIDIA CORPORATION. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; From 2f8977c13c8a32f190bb84d441d620511d8a0264 Mon Sep 17 00:00:00 2001 From: Nate Rock Date: Tue, 18 Nov 2025 09:24:03 -0600 Subject: [PATCH 033/105] updated update-version.sh to handle release branch version changes (NVIDIA/cuvs-lucene#50) Admin merging as we are finalizing updates to rapids-bot as part of NBS. ## Description This PR supports handling the new main branch strategy outlined below: * [RSN 47 - Changes to RAPIDS branching strategy in 25.12](https://docs.rapids.ai/notices/rsn0047/) The `update-version.sh` script should now supports two modes controlled via `CLI` params or `ENV` vars: CLI arguments: `--run-context=main|release` ENV var `RAPIDS_RUN_CONTEXT=main|release` xref: https://github.com/rapidsai/build-planning/issues/224 --- java/cuvs-lucene/VERSION | 2 +- java/cuvs-lucene/ci/release/update-version.sh | 69 ++++++++++++++++++- 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/java/cuvs-lucene/VERSION b/java/cuvs-lucene/VERSION index 296e35288d..7924af6192 100644 --- a/java/cuvs-lucene/VERSION +++ b/java/cuvs-lucene/VERSION @@ -1 +1 @@ -25.10.00 +25.12.00 diff --git a/java/cuvs-lucene/ci/release/update-version.sh b/java/cuvs-lucene/ci/release/update-version.sh index 0404480ee0..61848ed9c8 100755 --- a/java/cuvs-lucene/ci/release/update-version.sh +++ b/java/cuvs-lucene/ci/release/update-version.sh @@ -8,11 +8,69 @@ ######################## ## Usage -# bash update-version.sh +# Primary interface: bash update-version.sh [--run-context=main|release] +# Fallback interface: [RAPIDS_RUN_CONTEXT=main|release] bash update-version.sh +# CLI arguments take precedence over environment variables +# Defaults to main when no run-context is specified +set -euo pipefail + +# Verify we're running from the repository root +if [[ ! -d ".git" ]]; then + echo "Error: This script must be run from the repository root directory." + echo "Expected to find: .git/" + echo "Current directory: $(pwd)" + exit 1 +fi + +# Parse command line arguments +CLI_RUN_CONTEXT="" +VERSION_ARG="" + +for arg in "$@"; do + case $arg in + --run-context=*) + CLI_RUN_CONTEXT="${arg#*=}" + shift + ;; + *) + if [[ -z "$VERSION_ARG" ]]; then + VERSION_ARG="$arg" + fi + ;; + esac +done # Format is YY.MM.PP - no leading 'v' or trailing 'a' -NEXT_FULL_TAG=$1 +NEXT_FULL_TAG="$VERSION_ARG" + +# Determine RUN_CONTEXT with CLI precedence over environment variable, defaulting to main +if [[ -n "$CLI_RUN_CONTEXT" ]]; then + RUN_CONTEXT="$CLI_RUN_CONTEXT" + echo "Using run-context from CLI: $RUN_CONTEXT" +elif [[ -n "${RAPIDS_RUN_CONTEXT}" ]]; then + RUN_CONTEXT="$RAPIDS_RUN_CONTEXT" + echo "Using run-context from environment: $RUN_CONTEXT" +else + RUN_CONTEXT="main" + echo "No run-context provided, defaulting to: $RUN_CONTEXT" +fi + +# Validate RUN_CONTEXT value +if [[ "${RUN_CONTEXT}" != "main" && "${RUN_CONTEXT}" != "release" ]]; then + echo "Error: Invalid run-context value '${RUN_CONTEXT}'" + echo "Valid values: main, release" + exit 1 +fi + +# Validate version argument +if [[ -z "$NEXT_FULL_TAG" ]]; then + echo "Error: Version argument is required" + echo "Usage: $0 [--run-context=]" + echo " or: [RAPIDS_RUN_CONTEXT=] $0 " + echo "Note: Defaults to main when run-context is not specified" + exit 1 +fi # Get current version CURRENT_TAG=$(git tag --merged HEAD | grep -xE '^v.*' | sort --version-sort | tail -n 1 | tr -d 'v') @@ -26,7 +84,12 @@ NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} # Strip leading 0s in versions, so e.g. '25.10.00' becomes '25.10.0' PATCH_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_PATCH}'))") -echo "Preparing release $CURRENT_TAG => $NEXT_FULL_TAG" +# Log update context +if [[ "${RUN_CONTEXT}" == "main" ]]; then + echo "Preparing development branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting main branch)" +elif [[ "${RUN_CONTEXT}" == "release" ]]; then + echo "Preparing release branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting release/${NEXT_SHORT_TAG} branch)" +fi # Inplace sed replace; workaround for Linux and Mac function sed_runner() { From 9a88a3b25a4922e8b8378d6d5b1fa64b9f26c7b0 Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Tue, 18 Nov 2025 09:36:03 -0600 Subject: [PATCH 034/105] Merge pull request NVIDIA/cuvs-lucene#51 from rapidsai/version-update-26.02 Update to 26.02 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 2 +- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/VERSION | 2 +- java/cuvs-lucene/build.sh | 2 +- .../conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- .../conda/environments/all_cuda-130_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-130_arch-x86_64.yaml | 2 +- java/cuvs-lucene/dependencies.yaml | 2 +- java/cuvs-lucene/pom.xml | 4 ++-- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index ca5c9da655..a2f69e8d13 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -48,7 +48,7 @@ jobs: branch: ${{ inputs.branch }} arch: "amd64" date: ${{ inputs.date }} - container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.02-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/build_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 9a289e0c05..e8c1503528 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -73,7 +73,7 @@ jobs: build_type: pull-request node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.02-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index cbecd1f038..65b3a4ad7c 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -39,5 +39,5 @@ jobs: sha: ${{ inputs.sha }} node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.02-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" diff --git a/java/cuvs-lucene/VERSION b/java/cuvs-lucene/VERSION index 7924af6192..5c33046aca 100644 --- a/java/cuvs-lucene/VERSION +++ b/java/cuvs-lucene/VERSION @@ -1 +1 @@ -25.12.00 +26.02.00 diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index 9f65feac36..a6fea7a0bc 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -8,7 +8,7 @@ set -e -u -o pipefail ARGS="$*" NUMARGS=$# -VERSION="25.10.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked +VERSION="26.02.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked GROUP_ID="com.nvidia.cuvs" function hasArg { diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index 05ec85b6ca..2f455069da 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -12,7 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==25.10.* +- libcuvs==26.02.* - maven - openjdk=22.* name: all_cuda-129_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index f0a2538080..72c66a2b99 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -12,7 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==25.10.* +- libcuvs==26.02.* - maven - openjdk=22.* name: all_cuda-129_arch-x86_64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml index 6638808caf..a87f9f2448 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -12,7 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==25.10.* +- libcuvs==26.02.* - maven - openjdk=22.* name: all_cuda-130_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml index bb8a746f4a..6727f793d3 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -12,7 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==25.10.* +- libcuvs==26.02.* - maven - openjdk=22.* name: all_cuda-130_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index 322f0be85f..b846b6d33f 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -70,7 +70,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev - - libcuvs==25.10.* + - libcuvs==26.02.* java: common: - output_types: conda diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 10091cbc7b..75445b9cb8 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -12,7 +12,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 25.10.0 + 26.02.0 cuvs-lucene jar @@ -70,7 +70,7 @@ com.nvidia.cuvs cuvs-java - 25.10.0 + 26.02.0 From 339f5642148d46b5805662d9150f13f286812423 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Tue, 18 Nov 2025 07:36:39 -0800 Subject: [PATCH 035/105] Update for release branch release/25.12 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 2 +- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/build.sh | 2 +- .../conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- .../conda/environments/all_cuda-130_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-130_arch-x86_64.yaml | 2 +- java/cuvs-lucene/dependencies.yaml | 2 +- java/cuvs-lucene/pom.xml | 4 ++-- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index ca5c9da655..a52040f81d 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -48,7 +48,7 @@ jobs: branch: ${{ inputs.branch }} arch: "amd64" date: ${{ inputs.date }} - container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:25.12-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/build_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 9a289e0c05..e81ea84cd0 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -73,7 +73,7 @@ jobs: build_type: pull-request node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:25.12-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index cbecd1f038..1852fa3bb8 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -39,5 +39,5 @@ jobs: sha: ${{ inputs.sha }} node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:25.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:25.12-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index 9f65feac36..1e7ba033c5 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -8,7 +8,7 @@ set -e -u -o pipefail ARGS="$*" NUMARGS=$# -VERSION="25.10.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked +VERSION="25.12.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked GROUP_ID="com.nvidia.cuvs" function hasArg { diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index 05ec85b6ca..816c8fcf11 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -12,7 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==25.10.* +- libcuvs==25.12.* - maven - openjdk=22.* name: all_cuda-129_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index f0a2538080..447b98037a 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -12,7 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==25.10.* +- libcuvs==25.12.* - maven - openjdk=22.* name: all_cuda-129_arch-x86_64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml index 6638808caf..6fb38df428 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -12,7 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==25.10.* +- libcuvs==25.12.* - maven - openjdk=22.* name: all_cuda-130_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml index bb8a746f4a..3be3d797b0 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -12,7 +12,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==25.10.* +- libcuvs==25.12.* - maven - openjdk=22.* name: all_cuda-130_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index 322f0be85f..7066248f89 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -70,7 +70,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev - - libcuvs==25.10.* + - libcuvs==25.12.* java: common: - output_types: conda diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 10091cbc7b..1562081115 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -12,7 +12,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 25.10.0 + 25.12.0 cuvs-lucene jar @@ -70,7 +70,7 @@ com.nvidia.cuvs cuvs-java - 25.10.0 + 25.12.0 From ac8f52576826dbfa5374bf598b95b0fac3f0967b Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 25 Nov 2025 11:40:57 -0500 Subject: [PATCH 036/105] Modify CI pipeline to use latest cuvs-java instead of the one released on Maven Central (NVIDIA/cuvs-lucene#49) Fixes NVIDIA/cuvs-lucene#43 Please note: I had to bump up the `cuda_version` to `13.0.2` because for `26.02` the [image with the expected tag is not available](https://hub.docker.com/r/rapidsai/ci-conda/tags?name=26.02-cuda13.0.0-ubuntu24.04-py3.13) but for `13.0.2` [it does](https://hub.docker.com/r/rapidsai/ci-conda/tags?name=26.02-cuda13.0.2-ubuntu24.04-py3.13). Also, this PR includes updating the `FilterCuVSProvider.java` to adapt to the latest `cuvs-java` changes. Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Bradley Dice (https://github.com/bdice) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/49 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 2 +- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/.gitignore | 1 + java/cuvs-lucene/RAPIDS_BRANCH | 1 + java/cuvs-lucene/build.sh | 18 ++++++++++++ java/cuvs-lucene/ci/build_java.sh | 3 ++ java/cuvs-lucene/ci/release/update-version.sh | 4 +++ .../all_cuda-129_arch-aarch64.yaml | 2 ++ .../all_cuda-129_arch-x86_64.yaml | 2 ++ .../all_cuda-130_arch-aarch64.yaml | 2 ++ .../all_cuda-130_arch-x86_64.yaml | 2 ++ java/cuvs-lucene/dependencies.yaml | 2 ++ java/cuvs-lucene/pom.xml | 2 -- .../cuvs/lucene/FilterCuVSProvider.java | 29 +++++++++++++++++++ 15 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 java/cuvs-lucene/RAPIDS_BRANCH diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index a2f69e8d13..e7b7d90666 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -42,7 +42,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.0.0' + - '13.0.2' with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index e8c1503528..7d7ac7dff2 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -68,7 +68,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.0.0' + - '13.0.2' with: build_type: pull-request node_type: "gpu-l4-latest-1" diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 65b3a4ad7c..c6019f2de9 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -31,7 +31,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.0.0' + - '13.0.2' with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} diff --git a/java/cuvs-lucene/.gitignore b/java/cuvs-lucene/.gitignore index 32d6a5b09e..f5287f1f11 100644 --- a/java/cuvs-lucene/.gitignore +++ b/java/cuvs-lucene/.gitignore @@ -4,3 +4,4 @@ *.jar target **/.DS_Store +cuvs-workdir diff --git a/java/cuvs-lucene/RAPIDS_BRANCH b/java/cuvs-lucene/RAPIDS_BRANCH new file mode 100644 index 0000000000..ba2906d066 --- /dev/null +++ b/java/cuvs-lucene/RAPIDS_BRANCH @@ -0,0 +1 @@ +main diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index a6fea7a0bc..e9b68eb472 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -15,6 +15,24 @@ function hasArg { (( NUMARGS != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") } +if hasArg --build-cuvs-java; then + CUVS_WORKDIR="cuvs-workdir" + CUVS_GIT_REPO="https://github.com/rapidsai/cuvs.git" + if [[ -d "$CUVS_WORKDIR" && -n "$(ls -A "$CUVS_WORKDIR")" ]]; then + echo "Directory '$CUVS_WORKDIR' exists and is not empty." + pushd $CUVS_WORKDIR + git pull + else + BRANCH=$(cat "RAPIDS_BRANCH") + echo "Directory '$CUVS_WORKDIR' does not exist or is empty. Cloning the cuvs's '$BRANCH' branch." + # Correct branch selection is crucial to avoid version mismatch issues when testing. + git clone --branch "$BRANCH" $CUVS_GIT_REPO $CUVS_WORKDIR + pushd $CUVS_WORKDIR + fi + ./build.sh java + popd +fi + MAVEN_VERIFY_ARGS=() if ! hasArg --run-java-tests; then MAVEN_VERIFY_ARGS=("-DskipTests") diff --git a/java/cuvs-lucene/ci/build_java.sh b/java/cuvs-lucene/ci/build_java.sh index 2f61f4b5a7..b586ec9e8b 100755 --- a/java/cuvs-lucene/ci/build_java.sh +++ b/java/cuvs-lucene/ci/build_java.sh @@ -13,6 +13,9 @@ if [[ "${1:-}" == "--run-java-tests" ]]; then EXTRA_BUILD_ARGS+=("--run-java-tests") fi +# Always build cuvs-java when running the pipeline +EXTRA_BUILD_ARGS+=("--build-cuvs-java") + # shellcheck disable=SC1091 . /opt/conda/etc/profile.d/conda.sh diff --git a/java/cuvs-lucene/ci/release/update-version.sh b/java/cuvs-lucene/ci/release/update-version.sh index 61848ed9c8..afe73f640d 100755 --- a/java/cuvs-lucene/ci/release/update-version.sh +++ b/java/cuvs-lucene/ci/release/update-version.sh @@ -86,8 +86,10 @@ PATCH_PEP440=$(python -c "from packaging.version import Version; print(Version(' # Log update context if [[ "${RUN_CONTEXT}" == "main" ]]; then + RAPIDS_BRANCH_NAME="main" echo "Preparing development branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting main branch)" elif [[ "${RUN_CONTEXT}" == "release" ]]; then + RAPIDS_BRANCH_NAME="release/${NEXT_SHORT_TAG}" echo "Preparing release branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting release/${NEXT_SHORT_TAG} branch)" fi @@ -98,6 +100,7 @@ function sed_runner() { # Centralized version file update echo "${NEXT_FULL_TAG}" > VERSION +echo "${RAPIDS_BRANCH_NAME}" > RAPIDS_BRANCH # Update Java version NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}" @@ -111,5 +114,6 @@ for FILE in dependencies.yaml conda/environments/*.yaml; do done for FILE in .github/workflows/*.yaml; do + sed_runner "/shared-workflows/ s|@.*|@${RAPIDS_BRANCH_NAME}|g" "${FILE}" sed_runner "s/:[0-9]*\\.[0-9]*-/:${NEXT_SHORT_TAG}-/g" "${FILE}" done diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index 2f455069da..9f1778f0a5 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -4,6 +4,7 @@ channels: - conda-forge - rapidsai-nightly dependencies: +- cmake>=3.30.4 - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api @@ -14,5 +15,6 @@ dependencies: - libcusparse-dev - libcuvs==26.02.* - maven +- ninja - openjdk=22.* name: all_cuda-129_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index 72c66a2b99..64caceea2d 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -4,6 +4,7 @@ channels: - conda-forge - rapidsai-nightly dependencies: +- cmake>=3.30.4 - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api @@ -14,5 +15,6 @@ dependencies: - libcusparse-dev - libcuvs==26.02.* - maven +- ninja - openjdk=22.* name: all_cuda-129_arch-x86_64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml index a87f9f2448..0edaf41e4c 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -4,6 +4,7 @@ channels: - conda-forge - rapidsai-nightly dependencies: +- cmake>=3.30.4 - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api @@ -14,5 +15,6 @@ dependencies: - libcusparse-dev - libcuvs==26.02.* - maven +- ninja - openjdk=22.* name: all_cuda-130_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml index 6727f793d3..7bc25ec58d 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -4,6 +4,7 @@ channels: - conda-forge - rapidsai-nightly dependencies: +- cmake>=3.30.4 - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api @@ -14,5 +15,6 @@ dependencies: - libcusparse-dev - libcuvs==26.02.* - maven +- ninja - openjdk=22.* name: all_cuda-130_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index b846b6d33f..3443a2f02c 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -75,5 +75,7 @@ dependencies: common: - output_types: conda packages: + - cmake>=3.30.4 - maven + - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 75445b9cb8..160286881a 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -41,8 +41,6 @@ 21 - - org.apache.lucene diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index 01752d10cb..aaf722191d 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -6,14 +6,19 @@ import com.nvidia.cuvs.BruteForceIndex; import com.nvidia.cuvs.CagraIndex; +import com.nvidia.cuvs.CagraIndexParams; +import com.nvidia.cuvs.CagraIndexParams.CuvsDistanceType; +import com.nvidia.cuvs.CagraIndexParams.HnswHeuristicType; import com.nvidia.cuvs.CuVSMatrix; import com.nvidia.cuvs.CuVSMatrix.Builder; import com.nvidia.cuvs.CuVSMatrix.DataType; import com.nvidia.cuvs.CuVSResources; import com.nvidia.cuvs.HnswIndex; +import com.nvidia.cuvs.HnswIndexParams; import com.nvidia.cuvs.spi.CuVSProvider; import java.lang.invoke.MethodHandle; import java.nio.file.Path; +import java.util.logging.Level; /*package-private*/ class FilterCuVSProvider implements CuVSProvider { @@ -114,4 +119,28 @@ public com.nvidia.cuvs.TieredIndex.Builder newTieredIndexBuilder(CuVSResources c throws UnsupportedOperationException { return delegate.newTieredIndexBuilder(cuVSResources); } + + @Override + public CagraIndexParams cagraIndexParamsFromHnswParams( + long arg0, long arg1, int arg2, int arg3, HnswHeuristicType arg4, CuvsDistanceType arg5) { + // TODO Auto-generated method stub + return null; + } + + @Override + public Level getLogLevel() { + // TODO Auto-generated method stub + return null; + } + + @Override + public void setLogLevel(Level arg0) { + // TODO Auto-generated method stub + } + + @Override + public HnswIndex hnswIndexFromCagra(HnswIndexParams arg0, CagraIndex arg1) throws Throwable { + // TODO Auto-generated method stub + return null; + } } From c1837ba1d2c2e2d8aa3b5df3a520642f442057c4 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 25 Nov 2025 13:05:58 -0600 Subject: [PATCH 037/105] Use strict priority in CI conda tests (NVIDIA/cuvs-lucene#53) This PR sets conda to use `strict` priority in CI tests. Mixing channel priority is frequently a cause of unexpected errors. Our CI jobs should always use strict priority in order to enforce that conda packages come from local channels with the artifacts built in CI, not mixing with older nightly artifacts from the `rapidsai-nightly` channel or other sources. xref: https://github.com/rapidsai/build-planning/issues/14 Authors: - Bradley Dice (https://github.com/bdice) - Vivek Narang (https://github.com/narangvivek10) - Gil Forsyth (https://github.com/gforsyth) Approvers: - Gil Forsyth (https://github.com/gforsyth) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/53 --- java/cuvs-lucene/ci/build_java.sh | 3 +++ .../conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- .../conda/environments/all_cuda-130_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-130_arch-x86_64.yaml | 2 +- java/cuvs-lucene/dependencies.yaml | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/java/cuvs-lucene/ci/build_java.sh b/java/cuvs-lucene/ci/build_java.sh index b586ec9e8b..173ca55c28 100755 --- a/java/cuvs-lucene/ci/build_java.sh +++ b/java/cuvs-lucene/ci/build_java.sh @@ -19,6 +19,9 @@ EXTRA_BUILD_ARGS+=("--build-cuvs-java") # shellcheck disable=SC1091 . /opt/conda/etc/profile.d/conda.sh +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + rapids-logger "Generate Java testing dependencies" ENV_YAML_DIR="$(mktemp -d)" diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index 9f1778f0a5..b66afbc36d 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -1,8 +1,8 @@ # This file is generated by `rapids-dependency-file-generator`. # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: -- conda-forge - rapidsai-nightly +- conda-forge dependencies: - cmake>=3.30.4 - cuda-cudart-dev diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index 64caceea2d..ff0c06d71a 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -1,8 +1,8 @@ # This file is generated by `rapids-dependency-file-generator`. # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: -- conda-forge - rapidsai-nightly +- conda-forge dependencies: - cmake>=3.30.4 - cuda-cudart-dev diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml index 0edaf41e4c..489f899a89 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -1,8 +1,8 @@ # This file is generated by `rapids-dependency-file-generator`. # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: -- conda-forge - rapidsai-nightly +- conda-forge dependencies: - cmake>=3.30.4 - cuda-cudart-dev diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml index 7bc25ec58d..71cfa010a2 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -1,8 +1,8 @@ # This file is generated by `rapids-dependency-file-generator`. # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: -- conda-forge - rapidsai-nightly +- conda-forge dependencies: - cmake>=3.30.4 - cuda-cudart-dev diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index 3443a2f02c..f0cb592e08 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -23,8 +23,8 @@ files: - cuda_version - java channels: - - conda-forge - rapidsai-nightly + - conda-forge dependencies: checks: common: From 0025e3dc53e10d6b245204ebe1aec595112e56ce Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 25 Nov 2025 15:34:36 -0500 Subject: [PATCH 038/105] Setup Java Code Coverage and add more tests (NVIDIA/cuvs-lucene#42) Fixes NVIDIA/cuvs-lucene#40 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cuvs-lucene/pull/42 --- java/cuvs-lucene/.gitignore | 1 + java/cuvs-lucene/build.sh | 7 +- java/cuvs-lucene/pom.xml | 19 + .../TestAcceleratedHNSWDeletedDocuments.java | 332 ++++++++++++++++++ ...stCuVSAcceleratedHNSWDeletedDocuments.java | 332 ++++++++++++++++++ .../lucene/TestCuVSAcceleratedHNSWGaps.java | 187 ++++++++++ .../cuvs/lucene/TestCuVSDeletedDocuments.java | 4 +- .../com/nvidia/cuvs/lucene/TestCuVSGaps.java | 2 +- .../TestCuVSRandomizedHNSWVectorSearch.java | 215 ++++++++++++ .../TestCuVSRandomizedVectorSearch.java | 2 +- ...tLucene99AcceleratedHNSWVectorsFormat.java | 155 ++++++++ .../com/nvidia/cuvs/lucene/TestMerge.java | 2 +- 12 files changed, 1252 insertions(+), 6 deletions(-) create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java diff --git a/java/cuvs-lucene/.gitignore b/java/cuvs-lucene/.gitignore index f5287f1f11..571838e7cc 100644 --- a/java/cuvs-lucene/.gitignore +++ b/java/cuvs-lucene/.gitignore @@ -5,3 +5,4 @@ target **/.DS_Store cuvs-workdir +bin diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index e9b68eb472..bf768069d1 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -38,6 +38,11 @@ if ! hasArg --run-java-tests; then MAVEN_VERIFY_ARGS=("-DskipTests") fi -mvn verify "${MAVEN_VERIFY_ARGS[@]}" \ +mvn clean verify "${MAVEN_VERIFY_ARGS[@]}" \ && mvn install:install-file -Dfile=./target/cuvs-lucene-$VERSION.jar -DgroupId=$GROUP_ID -DartifactId=cuvs-lucene -Dversion=$VERSION -Dpackaging=jar \ && cp pom.xml ./target/ + +# Generate JaCoCo code coverage reports available here: target/site/jacoco/index.html +if hasArg --run-java-tests; then + mvn jacoco:report +fi diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 160286881a..4761fec03d 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -165,6 +165,25 @@ + + org.jacoco + jacoco-maven-plugin + 0.8.14 + + + + prepare-agent + + + + report + prepare-package + + report + + + + diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java new file mode 100644 index 0000000000..d46794df1c --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java @@ -0,0 +1,332 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.TestUtils.generateRandomVector; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.analysis.MockTokenizer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestAcceleratedHNSWDeletedDocuments extends LuceneTestCase { + + protected static Logger log = + Logger.getLogger(TestAcceleratedHNSWDeletedDocuments.class.getName()); + + static final Codec codec = + TestUtil.alwaysKnnVectorsFormat(new Lucene99AcceleratedHNSWVectorsFormat()); + private static Random random; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + random = random(); + } + + @Test + public void testVectorSearchWithDeletedDocuments() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200, 1000); // 200-1200 documents + int dimensions = random.nextInt(64, 256); // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float deletionProbability = random.nextFloat() * 0.4f + 0.1f; // 10-50% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + Set deletedDocs = new HashSet<>(); + + // Create index with all documents having vectors + try (RandomIndexWriter writer = createWriter(directory)) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + } + + // Delete documents randomly based on probability + for (int i = 0; i < datasetSize; i++) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + deletedDocs.add(i); + } + } + writer.commit(); + } + + // Search and verify deleted documents are not returned + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + // Use a random vector for query + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify we got results + assertTrue("Should have search results", hits.length > 0); + + // Verify no deleted documents in results + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertFalse( + "Deleted document " + id + " should not appear in results", deletedDocs.contains(id)); + log.info("Found non-deleted document: " + id + ", Score: " + hit.score); + } + + // Verify deleted documents are truly deleted + for (int deletedId : deletedDocs) { + TopDocs result = + searcher.search(new TermQuery(new Term("id", String.valueOf(deletedId))), 1); + assertEquals( + "Deleted document " + deletedId + " should not be found", + 0, + result.totalHits.value()); + } + } + } + } + + @Test + public void testVectorSearchWithMixedDeletedAndMissingVectors() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200) + 50; // 50-250 documents + int dimensions = random.nextInt(256) + 64; // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float vectorProbability = random.nextFloat() * 0.5f + 0.3f; // 30-80% have vectors + float deletionProbability = random.nextFloat() * 0.3f + 0.1f; // 10-40% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + Set docsWithoutVectors = new HashSet<>(); + Set deletedDocs = new HashSet<>(); + + // Create index with mixed documents + try (RandomIndexWriter writer = createWriter(directory)) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + // Randomly assign categories + String category = random.nextBoolean() ? "A" : "B"; + doc.add(new StringField("category", category, Field.Store.YES)); + + // Randomly decide whether to add vectors + if (random.nextFloat() < vectorProbability) { + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + } else { + docsWithoutVectors.add(i); + } + writer.addDocument(doc); + } + + // Delete documents randomly + for (int i = 0; i < datasetSize; i++) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + deletedDocs.add(i); + } + } + writer.commit(); + } + + // Test vector search behavior + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify results + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertFalse("Deleted document should not appear", deletedDocs.contains(id)); + assertFalse("Document without vector should not appear", docsWithoutVectors.contains(id)); + log.info("Found document with vector: " + id + ", Score: " + hit.score); + } + + // Test filtered search with deletions + Query filter = new TermQuery(new Term("category", "A")); + Query filteredQuery = new KnnFloatVectorQuery("vector", queryVector, topK, filter); + ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; + + for (ScoreDoc hit : filteredHits) { + Document doc = reader.storedFields().document(hit.doc); + String category = doc.get("category"); + assertEquals("Should only match category A", "A", category); + int id = Integer.parseInt(doc.get("id")); + assertFalse( + "Deleted document should not appear in filtered results", deletedDocs.contains(id)); + } + } + } + } + + @Test + public void testVectorSearchAfterAllDocumentsDeleted() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(20) + 5; // 5-25 documents for this test + int dimensions = random.nextInt(128) + 32; // 32-160 dimensions + int topK = Math.min(random.nextInt(10) + 5, datasetSize); // 5-15 results + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + + // Create and delete all documents + try (IndexWriter writer = new IndexWriter(directory, createWriterConfig())) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + } + writer.commit(); + + // Delete all documents + for (int i = 0; i < datasetSize; i++) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + } + writer.commit(); + writer.forceMerge(1); // Force merge to apply deletions + } + + // Verify search returns no results + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + TopDocs results = searcher.search(query, topK); + + assertEquals( + "Should return no results when all documents are deleted", + 0, + results.totalHits.value()); + } + } + } + + @Test + public void testVectorSearchWithPartialDeletionAndReindexing() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200) + 50; // 50-250 documents + int dimensions = random.nextInt(256) + 64; // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float deletionProbability = random.nextFloat() * 0.3f + 0.1f; // 10-40% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + List activeDocIds = new ArrayList<>(); + + // Initial indexing + try (IndexWriter writer = new IndexWriter(directory, createWriterConfig())) { + int initialDocs = datasetSize / 2 + random.nextInt(datasetSize / 4); // 50-75% of dataset + for (int i = 0; i < initialDocs; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + activeDocIds.add(i); + } + + // Delete some documents randomly + List candidatesForDeletion = new ArrayList<>(activeDocIds); + for (int docId : candidatesForDeletion) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(docId))); + activeDocIds.remove(Integer.valueOf(docId)); + } + } + + // Add new documents with higher IDs + for (int i = initialDocs; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + activeDocIds.add(i); + } + writer.commit(); + } + + // Verify search behavior after deletions and additions + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + Set resultIds = new HashSet<>(); + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + resultIds.add(id); + assertTrue("Result should be from active documents", activeDocIds.contains(id)); + } + + log.info( + "Search returned " + + hits.length + + " results from " + + activeDocIds.size() + + " active documents"); + } + } + } + + private RandomIndexWriter createWriter(Directory directory) throws IOException { + return new RandomIndexWriter( + random(), + directory, + newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy())); + } + + private IndexWriterConfig createWriterConfig() { + return newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy()); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java new file mode 100644 index 0000000000..49372b6b00 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java @@ -0,0 +1,332 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.TestUtils.generateRandomVector; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.analysis.MockTokenizer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestCuVSAcceleratedHNSWDeletedDocuments extends LuceneTestCase { + + protected static Logger log = + Logger.getLogger(TestCuVSAcceleratedHNSWDeletedDocuments.class.getName()); + + static final Codec codec = + TestUtil.alwaysKnnVectorsFormat(new Lucene99AcceleratedHNSWVectorsFormat()); + private static Random random; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + random = random(); + } + + @Test + public void testVectorSearchWithDeletedDocuments() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200, 1000); // 200-1200 documents + int dimensions = random.nextInt(64, 256); // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float deletionProbability = random.nextFloat() * 0.4f + 0.1f; // 10-50% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + Set deletedDocs = new HashSet<>(); + + // Create index with all documents having vectors + try (RandomIndexWriter writer = createWriter(directory)) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + } + + // Delete documents randomly based on probability + for (int i = 0; i < datasetSize; i++) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + deletedDocs.add(i); + } + } + writer.commit(); + } + + // Search and verify deleted documents are not returned + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + // Use a random vector for query + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify we got results + assertTrue("Should have search results", hits.length > 0); + + // Verify no deleted documents in results + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertFalse( + "Deleted document " + id + " should not appear in results", deletedDocs.contains(id)); + log.info("Found non-deleted document: " + id + ", Score: " + hit.score); + } + + // Verify deleted documents are truly deleted + for (int deletedId : deletedDocs) { + TopDocs result = + searcher.search(new TermQuery(new Term("id", String.valueOf(deletedId))), 1); + assertEquals( + "Deleted document " + deletedId + " should not be found", + 0, + result.totalHits.value()); + } + } + } + } + + @Test + public void testVectorSearchWithMixedDeletedAndMissingVectors() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200) + 50; // 50-250 documents + int dimensions = random.nextInt(256) + 64; // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float vectorProbability = random.nextFloat() * 0.5f + 0.3f; // 30-80% have vectors + float deletionProbability = random.nextFloat() * 0.3f + 0.1f; // 10-40% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + Set docsWithoutVectors = new HashSet<>(); + Set deletedDocs = new HashSet<>(); + + // Create index with mixed documents + try (RandomIndexWriter writer = createWriter(directory)) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + // Randomly assign categories + String category = random.nextBoolean() ? "A" : "B"; + doc.add(new StringField("category", category, Field.Store.YES)); + + // Randomly decide whether to add vectors + if (random.nextFloat() < vectorProbability) { + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + } else { + docsWithoutVectors.add(i); + } + writer.addDocument(doc); + } + + // Delete documents randomly + for (int i = 0; i < datasetSize; i++) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + deletedDocs.add(i); + } + } + writer.commit(); + } + + // Test vector search behavior + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify results + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertFalse("Deleted document should not appear", deletedDocs.contains(id)); + assertFalse("Document without vector should not appear", docsWithoutVectors.contains(id)); + log.info("Found document with vector: " + id + ", Score: " + hit.score); + } + + // Test filtered search with deletions + Query filter = new TermQuery(new Term("category", "A")); + Query filteredQuery = new KnnFloatVectorQuery("vector", queryVector, topK, filter); + ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; + + for (ScoreDoc hit : filteredHits) { + Document doc = reader.storedFields().document(hit.doc); + String category = doc.get("category"); + assertEquals("Should only match category A", "A", category); + int id = Integer.parseInt(doc.get("id")); + assertFalse( + "Deleted document should not appear in filtered results", deletedDocs.contains(id)); + } + } + } + } + + @Test + public void testVectorSearchAfterAllDocumentsDeleted() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(20) + 5; // 5-25 documents for this test + int dimensions = random.nextInt(128) + 32; // 32-160 dimensions + int topK = Math.min(random.nextInt(10) + 5, datasetSize); // 5-15 results + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + + // Create and delete all documents + try (IndexWriter writer = new IndexWriter(directory, createWriterConfig())) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + } + writer.commit(); + + // Delete all documents + for (int i = 0; i < datasetSize; i++) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + } + writer.commit(); + writer.forceMerge(1); // Force merge to apply deletions + } + + // Verify search returns no results + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + TopDocs results = searcher.search(query, topK); + + assertEquals( + "Should return no results when all documents are deleted", + 0, + results.totalHits.value()); + } + } + } + + @Test + public void testVectorSearchWithPartialDeletionAndReindexing() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200) + 50; // 50-250 documents + int dimensions = random.nextInt(256) + 64; // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float deletionProbability = random.nextFloat() * 0.3f + 0.1f; // 10-40% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + List activeDocIds = new ArrayList<>(); + + // Initial indexing + try (IndexWriter writer = new IndexWriter(directory, createWriterConfig())) { + int initialDocs = datasetSize / 2 + random.nextInt(datasetSize / 4); // 50-75% of dataset + for (int i = 0; i < initialDocs; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + activeDocIds.add(i); + } + + // Delete some documents randomly + List candidatesForDeletion = new ArrayList<>(activeDocIds); + for (int docId : candidatesForDeletion) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(docId))); + activeDocIds.remove(Integer.valueOf(docId)); + } + } + + // Add new documents with higher IDs + for (int i = initialDocs; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + activeDocIds.add(i); + } + writer.commit(); + } + + // Verify search behavior after deletions and additions + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + Set resultIds = new HashSet<>(); + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + resultIds.add(id); + assertTrue("Result should be from active documents", activeDocIds.contains(id)); + } + + log.info( + "Search returned " + + hits.length + + " results from " + + activeDocIds.size() + + " active documents"); + } + } + } + + private RandomIndexWriter createWriter(Directory directory) throws IOException { + return new RandomIndexWriter( + random(), + directory, + newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy())); + } + + private IndexWriterConfig createWriterConfig() { + return newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy()); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java new file mode 100644 index 0000000000..58e9acc5c6 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.analysis.MockTokenizer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.English; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestCuVSAcceleratedHNSWGaps extends LuceneTestCase { + + protected static Logger log = Logger.getLogger(TestCuVSAcceleratedHNSWGaps.class.getName()); + + static final Codec codec = + TestUtil.alwaysKnnVectorsFormat(new Lucene99AcceleratedHNSWVectorsFormat()); + static IndexSearcher searcher; + static IndexReader reader; + static Directory directory; + static Random random; + + static int DATASET_SIZE_LIMIT = 1000; + static int DIMENSIONS_LIMIT = 2048; + static int NUM_QUERIES_LIMIT = 10; + static int TOP_K_LIMIT = 64; + + static int datasetSize; + static int dimension; + static float[][] dataset; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + directory = newDirectory(); + random = random(); + + RandomIndexWriter writer = + new RandomIndexWriter( + random(), + directory, + newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy())); + + log.info("Merge Policy: " + writer.w.getConfig().getMergePolicy()); + + datasetSize = random.nextInt(100, DATASET_SIZE_LIMIT); + dimension = random.nextInt(8, DIMENSIONS_LIMIT); + dataset = generateDataset(random, datasetSize, dimension); + + // Create documents where only even-numbered documents have vectors + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(newTextField("field", English.intToEnglish(i), Field.Store.YES)); + + // Only add vectors to even-numbered documents + if (i % 2 == 0) { + doc.add(new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + } + + writer.addDocument(doc); + } + + reader = writer.getReader(); + searcher = newSearcher(reader); + writer.close(); + } + + @AfterClass + public static void afterClass() throws Exception { + if (reader != null) reader.close(); + if (directory != null) directory.close(); + searcher = null; + reader = null; + directory = null; + log.info("Test finished"); + } + + @Test + public void testVectorSearchWithAlternatingDocuments() throws IOException { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + + // Use the first vector (from document 0) as query + float[] queryVector = dataset[0]; + int topK = random.nextInt(5, TOP_K_LIMIT); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify we get exactly TOP_K results + assertEquals("Should return exactly " + topK + " results", topK, hits.length); + + // Verify all returned documents have vectors (even-numbered IDs) + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertEquals("All results should be even-numbered (have vectors)", 0, id % 2); + log.info("Document ID: " + id + ", Score: " + hit.score); + } + + // Verify the results match expected top-k based on Euclidean distance + List expectedIds = calculateExpectedTopK(queryVector, topK, dataset); + for (int i = 0; i < hits.length; i++) { + String docId = reader.storedFields().document(hits[i].doc).get("id"); + int id = Integer.parseInt(docId); + assertTrue("Result " + id + " should be in expected top-k results", expectedIds.contains(id)); + } + + log.info("Alternating document test passed with " + hits.length + " results"); + } + + @Test + public void testVectorSearchWithFilterAndAlternatingDocuments() throws IOException { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + + // Use the first vector (from document 0) as query + float[] queryVector = dataset[0]; + int topK = random.nextInt(5, TOP_K_LIMIT); + + // Create a filter that only matches documents with ID less than 10 + // This should further restrict our results to even numbers 0, 2, 4, 6, 8 + Query filter = new TermQuery(new Term("id", "8")); // Only match document 8 + + Query filteredQuery = new KnnFloatVectorQuery("vector", queryVector, topK, filter); + ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; + + // Should only get document 8 (the only one that matches the filter and has a vector) + assertEquals("Should return exactly 1 result", 1, filteredHits.length); + + String docId = reader.storedFields().document(filteredHits[0].doc).get("id"); + assertEquals("Should only return document 8", "8", docId); + + log.info("Filtered alternating document test passed with " + filteredHits.length + " results"); + } + + public static List calculateExpectedTopK(float[] query, int topK, float[][] dataset) { + Map distances = new TreeMap<>(); + + // Calculate distances only for documents that have vectors (even-numbered) + for (int i = 0; i < dataset.length; i += 2) { + double distance = 0; + for (int j = 0; j < dataset[0].length; j++) { + distance += (query[j] - dataset[i][j]) * (query[j] - dataset[i][j]); + } + distances.put(i, distance); + } + + // Sort by distance and return top-k + return distances.entrySet().stream() + .sorted(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .limit(topK) + .toList(); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java index 696e6bd0e3..179399bba0 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java @@ -40,7 +40,7 @@ import org.junit.BeforeClass; import org.junit.Test; -@SuppressSysoutChecks(bugUrl = "prints info from within cuVS") +@SuppressSysoutChecks(bugUrl = "") public class TestCuVSDeletedDocuments extends LuceneTestCase { protected static Logger log = Logger.getLogger(TestCuVSDeletedDocuments.class.getName()); @@ -50,7 +50,7 @@ public class TestCuVSDeletedDocuments extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { - assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); random = random(); } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java index 38035c1c76..39938d31d1 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java @@ -37,7 +37,7 @@ import org.junit.BeforeClass; import org.junit.Test; -@SuppressSysoutChecks(bugUrl = "prints info from within cuVS") +@SuppressSysoutChecks(bugUrl = "") public class TestCuVSGaps extends LuceneTestCase { protected static Logger log = Logger.getLogger(TestCuVSGaps.class.getName()); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java new file mode 100644 index 0000000000..c8c0485c9e --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java @@ -0,0 +1,215 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.TestUtils.generateQueries; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.analysis.MockTokenizer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.English; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestCuVSRandomizedHNSWVectorSearch extends LuceneTestCase { + + protected static Logger log = + Logger.getLogger(TestCuVSRandomizedHNSWVectorSearch.class.getName()); + + static final Codec codec = + TestUtil.alwaysKnnVectorsFormat(new Lucene99AcceleratedHNSWVectorsFormat()); + static IndexSearcher searcher; + static IndexReader reader; + static Directory directory; + + static int DATASET_SIZE_LIMIT = 1000; + static int DIMENSIONS_LIMIT = 2048; + static int NUM_QUERIES_LIMIT = 10; + static int TOP_K_LIMIT = 64; // TODO This fails beyond 64 + static float[][] dataset; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + directory = newDirectory(); + + RandomIndexWriter writer = + new RandomIndexWriter( + random(), + directory, + newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy())); + + log.info("Merge Policy: " + writer.w.getConfig().getMergePolicy()); + + Random random = random(); + int datasetSize = random.nextInt(DATASET_SIZE_LIMIT) + 1; + int dimensions = random.nextInt(DIMENSIONS_LIMIT) + 1; + dataset = generateDataset(random, datasetSize, dimensions); + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(newTextField("field", English.intToEnglish(i), Field.Store.YES)); + boolean skipVector = + random.nextInt(10) + < 4; // some documents won't have vectors to test deleted/missing vectors + if (!skipVector + || datasetSize < 100) { // about 10th of the documents shouldn't have a single vector + doc.add(new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + doc.add(new KnnFloatVectorField("vector2", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + } + + writer.addDocument(doc); + } + + reader = writer.getReader(); + searcher = newSearcher(reader); + writer.close(); + } + + @AfterClass + public static void afterClass() throws Exception { + if (reader != null) reader.close(); + if (directory != null) directory.close(); + searcher = null; + reader = null; + directory = null; + log.info("Test finished"); + } + + @Test + public void testVectorSearch() throws IOException { + Random random = random(); + int numQueries = random.nextInt(NUM_QUERIES_LIMIT) + 1; + int topK = Math.min(random.nextInt(TOP_K_LIMIT) + 1, dataset.length); + + if (dataset.length < topK) topK = dataset.length; + + float[][] queries = generateQueries(random, dataset[0].length, numQueries); + List> expected = generateExpectedResults(topK, dataset, queries); + + log.info("Dataset size: " + dataset.length + "x" + dataset[0].length); + log.info("Query size: " + numQueries + "x" + queries[0].length); + log.info("TopK: " + topK); + + Query query = new KnnFloatVectorQuery("vector", queries[0], topK); + int correct[] = new int[topK]; + for (int i = 0; i < topK; i++) correct[i] = expected.get(0).get(i); + + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + log.info("RESULTS: " + Arrays.toString(hits)); + log.info("EXPECTD: " + expected.get(0)); + + for (ScoreDoc hit : hits) { + log.info("\t" + reader.storedFields().document(hit.doc).get("id") + ": " + hit.score); + } + + for (ScoreDoc hit : hits) { + int doc = Integer.parseInt(reader.storedFields().document(hit.doc).get("id")); + assertTrue("Result returned was not in topk*2: " + doc, expected.get(0).contains(doc)); + } + } + + private static List> generateExpectedResults( + int topK, float[][] dataset, float[][] queries) { + List> neighborsResult = new ArrayList<>(); + int dimensions = dataset[0].length; + + for (float[] query : queries) { + Map distances = new TreeMap<>(); + for (int j = 0; j < dataset.length; j++) { + double distance = 0; + for (int k = 0; k < dimensions; k++) { + distance += (query[k] - dataset[j][k]) * (query[k] - dataset[j][k]); + } + distances.put(j, (distance)); + } + + Map sorted = new TreeMap(distances); + log.info("EXPECTED: " + sorted); + + // Sort by distance and select the topK nearest neighbors + List neighbors = + distances.entrySet().stream() + .sorted(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .toList(); + neighborsResult.add(neighbors.subList(0, Math.min(topK * 3, dataset.length))); + } + + log.info("Expected results generated successfully."); + return neighborsResult; + } + + @Test + public void testVectorSearchWithFilter() throws IOException { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + + Random random = random(); + int topK = Math.min(random.nextInt(TOP_K_LIMIT) + 1, dataset.length); + + if (dataset.length < topK) topK = dataset.length; + + // Find a document that has a vector by doing a search first + Query unfiltered = new KnnFloatVectorQuery("vector", dataset[0], 1); + ScoreDoc[] unfilteredHits = searcher.search(unfiltered, 1).scoreDocs; + + // Skip test if no vectors found at all + assumeTrue( + "Need at least one document with vector for filtering test", unfilteredHits.length > 0); + + String targetDocId = reader.storedFields().document(unfilteredHits[0].doc).get("id"); + float[] queryVector = dataset[0]; + + // Create a filter that matches only the document we know has a vector + Query filter = new TermQuery(new Term("id", targetDocId)); + + // Test the new constructor with filter + Query filteredQuery = new KnnFloatVectorQuery("vector", queryVector, topK, filter); + + ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; + + // Ensure we got some results + assertTrue("Should have at least one result", filteredHits.length > 0); + + // Verify that all results match the filter + for (ScoreDoc hit : filteredHits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + assertEquals("All results should match the filter", targetDocId, docId); + } + + log.info("Prefiltering test passed with " + filteredHits.length + " results"); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java index 6cbd646491..5e5c2b1b73 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java @@ -40,7 +40,7 @@ import org.junit.BeforeClass; import org.junit.Test; -@SuppressSysoutChecks(bugUrl = "prints info from within cuVS") +@SuppressSysoutChecks(bugUrl = "") public class TestCuVSRandomizedVectorSearch extends LuceneTestCase { protected static Logger log = Logger.getLogger(TestCuVSRandomizedVectorSearch.class.getName()); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java new file mode 100644 index 0000000000..f834d56471 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java @@ -0,0 +1,155 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +import java.util.List; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Ignore; + +@SuppressSysoutChecks(bugUrl = "") +public class TestLucene99AcceleratedHNSWVectorsFormat extends BaseKnnVectorsFormatTestCase { + + @BeforeClass + public static void beforeClass() { + assumeTrue("cuVS is not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + } + + @Override + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat(new Lucene99AcceleratedHNSWVectorsFormat()); + } + + public void testMergeTwoSegsWithASingleDocPerSeg() throws Exception { + float[][] f = new float[][] {randomVector(384), randomVector(384)}; + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc1 = new Document(); + doc1.add(new StringField("id", "0", Field.Store.NO)); + doc1.add(new KnnFloatVectorField("f", f[0], EUCLIDEAN)); + w.addDocument(doc1); + w.commit(); + Document doc2 = new Document(); + doc2.add(new StringField("id", "1", Field.Store.NO)); + doc2.add(new KnnFloatVectorField("f", f[1], EUCLIDEAN)); + w.addDocument(doc2); + w.flush(); + w.commit(); + + // sanity - verify one doc per leaf + try (DirectoryReader reader = DirectoryReader.open(w)) { + List subReaders = reader.leaves(); + assertEquals(2, subReaders.size()); + assertEquals(1, subReaders.get(0).reader().getFloatVectorValues("f").size()); + assertEquals(1, subReaders.get(1).reader().getFloatVectorValues("f").size()); + } + + // now merge to a single segment + w.forceMerge(1); + + // verify merged content + try (DirectoryReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues values = r.getFloatVectorValues("f"); + assertNotNull(values); + assertEquals(2, values.size()); + assertArrayEquals(f[0], values.vectorValue(0), 0.0f); + assertArrayEquals(f[1], values.vectorValue(1), 0.0f); + } + } + } + + // Basic test for multiple vectors fields per document + public void testTwoVectorFieldsPerDoc() throws Exception { + float[][] f1 = new float[][] {randomVector(384), randomVector(384)}; + float[][] f2 = new float[][] {randomVector(384), randomVector(384)}; + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc1 = new Document(); + doc1.add(new StringField("id", "0", Field.Store.NO)); + doc1.add(new KnnFloatVectorField("f1", f1[0], EUCLIDEAN)); + doc1.add(new KnnFloatVectorField("f2", f2[0], EUCLIDEAN)); + w.addDocument(doc1); + Document doc2 = new Document(); + doc2.add(new StringField("id", "1", Field.Store.NO)); + doc2.add(new KnnFloatVectorField("f1", f1[1], EUCLIDEAN)); + doc2.add(new KnnFloatVectorField("f2", f2[1], EUCLIDEAN)); + w.addDocument(doc2); + w.forceMerge(1); + + try (DirectoryReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues values = r.getFloatVectorValues("f1"); + assertNotNull(values); + assertEquals(2, values.size()); + assertArrayEquals(f1[0], values.vectorValue(0), 0.0f); + assertArrayEquals(f1[1], values.vectorValue(1), 0.0f); + + values = r.getFloatVectorValues("f2"); + assertNotNull(values); + assertEquals(2, values.size()); + assertArrayEquals(f2[0], values.vectorValue(0), 0.0f); + assertArrayEquals(f2[1], values.vectorValue(1), 0.0f); + + // opportunistically check boundary condition - search with a 0 topK + var topDocs = r.searchNearestVectors("f1", randomVector(384), 0, null, 10); + assertEquals(0, topDocs.scoreDocs.length); + assertEquals(0, topDocs.totalHits.value()); + } + } + } + + @Override + // Overriding this method from superclass for the tests to only use float vector encoding + protected VectorEncoding randomVectorEncoding() { + return VectorEncoding.FLOAT32; + } + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testByteVectorScorerIteration() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testEmptyByteVectorData() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testMergingWithDifferentByteKnnFields() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testMismatchedFields() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testRandomBytes() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testSortedIndexBytes() {} +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java index 676faff37e..4d76e18360 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java @@ -48,7 +48,7 @@ * CAGRA, and combined index configurations to ensure proper vector handling * and search functionality after segment merging. */ -@SuppressSysoutChecks(bugUrl = "CuVS native library produces verbose logging output") +@SuppressSysoutChecks(bugUrl = "") public class TestMerge extends LuceneTestCase { private static final Logger log = Logger.getLogger(TestMerge.class.getName()); From 65a8b65bfed2516d946a57f3c82c39fefdb9fbc1 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 25 Nov 2025 17:08:23 -0500 Subject: [PATCH 039/105] Lucene version agnostic vector format implementation (NVIDIA/cuvs-lucene#37) Fixes NVIDIA/cuvs-lucene#35 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/37 --- java/cuvs-lucene/.gitignore | 2 + java/cuvs-lucene/pom.xml | 5 + .../cuvs/lucene/CuVS2510GPUVectorsFormat.java | 21 +- .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 17 +- .../Lucene99AcceleratedHNSWVectorsFormat.java | 51 +++-- .../Lucene99AcceleratedHNSWVectorsWriter.java | 24 ++- .../nvidia/cuvs/lucene/LuceneProvider.java | 201 ++++++++++++++++++ .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 20 +- .../lucene/Lucene101AcceleratedHNSWCodec.java | 44 +++- .../nvidia/cuvs/lucene/TestBackCompat.java | 41 ++++ ...TestCagraToHnswSerializationAndSearch.java | 11 +- ...ializationAndSearchWithFallbackWriter.java | 11 +- 12 files changed, 401 insertions(+), 47 deletions(-) create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java diff --git a/java/cuvs-lucene/.gitignore b/java/cuvs-lucene/.gitignore index 571838e7cc..4962a91dae 100644 --- a/java/cuvs-lucene/.gitignore +++ b/java/cuvs-lucene/.gitignore @@ -4,5 +4,7 @@ *.jar target **/.DS_Store +bin +.project cuvs-workdir bin diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 4761fec03d..2f1cedd55c 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -53,6 +53,11 @@ 10.2.0 test + + org.apache.lucene + lucene-backward-codecs + 10.2.0 + org.apache.lucene lucene-test-framework diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java index 94ccd1b8c6..99639b189a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java @@ -15,7 +15,6 @@ import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; @@ -43,10 +42,8 @@ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; static CuVSResources resources = cuVSResourcesOrNull(); - - /** The format for storing, reading, and merging raw vectors on disk. */ - private static final FlatVectorsFormat flatVectorsFormat = - new Lucene99FlatVectorsFormat(DefaultFlatVectorScorer.INSTANCE); + static final LuceneProvider LUCENE_PROVIDER; + static final FlatVectorsFormat FLAT_VECTORS_FORMAT; final int maxDimensions = 4096; final int cuvsWriterThreads; @@ -54,6 +51,16 @@ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { final int graphDegree; final CuVS2510GPUVectorsWriter.IndexType indexType; // the index type to build, when writing + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + FLAT_VECTORS_FORMAT = + LUCENE_PROVIDER.getLuceneFlatVectorsFormatInstance(DefaultFlatVectorScorer.INSTANCE); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + /** * Initializes the {@link CuVS2510GPUVectorsFormat} with default parameter values. * @@ -92,7 +99,7 @@ public CuVS2510GPUVectorsFormat( @Override public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { checkSupported(); - var flatWriter = flatVectorsFormat.fieldsWriter(state); + var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); return new CuVS2510GPUVectorsWriter( state, cuvsWriterThreads, intGraphDegree, graphDegree, indexType, resources, flatWriter); } @@ -103,7 +110,7 @@ public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOE @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { checkSupported(); - return new CuVS2510GPUVectorsReader(state, resources, flatVectorsFormat.fieldsReader(state)); + return new CuVS2510GPUVectorsReader(state, resources, FLAT_VECTORS_FORMAT.fieldsReader(state)); } /** diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index bfe25ed8ac..033bbc14b6 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -9,7 +9,6 @@ import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_META_CODEC_EXT; import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_META_CODEC_NAME; import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.VERSION_CURRENT; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; import static org.apache.lucene.index.VectorEncoding.FLOAT32; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; @@ -68,6 +67,9 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { /** The name of the CUVS component for the info-stream * */ private static final String CUVS_COMPONENT = "CUVS"; + private static final LuceneProvider LUCENE_PROVIDER; + private static final List VECTOR_SIMILARITY_FUNCTIONS; + // The minimum number of vectors in the dataset required before // we attempt to build a Cagra index static final int MIN_CAGRA_INDEX_SIZE = 2; @@ -85,6 +87,15 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { private final InfoStream infoStream; private boolean finished; + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + VECTOR_SIMILARITY_FUNCTIONS = LUCENE_PROVIDER.getSimilarityFunctions(); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + /** * The cuVS index Types. */ @@ -442,8 +453,8 @@ private void writeMeta( } static int distFuncToOrd(VectorSimilarityFunction func) { - for (int i = 0; i < SIMILARITY_FUNCTIONS.size(); i++) { - if (SIMILARITY_FUNCTIONS.get(i).equals(func)) { + for (int i = 0; i < VECTOR_SIMILARITY_FUNCTIONS.size(); i++) { + if (VECTOR_SIMILARITY_FUNCTIONS.get(i).equals(func)) { return (byte) i; } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java index c3fd4ed94a..ffd3ff4bd9 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java @@ -5,9 +5,6 @@ package com.nvidia.cuvs.lucene; import static com.nvidia.cuvs.lucene.Utils.cuVSResourcesOrNull; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; import com.nvidia.cuvs.CuVSResources; import com.nvidia.cuvs.LibraryException; @@ -18,9 +15,6 @@ import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; @@ -43,22 +37,36 @@ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { static final String HNSW_META_CODEC_EXT = "vem"; static final String HNSW_INDEX_CODEC_NAME = "Lucene99HnswVectorsFormatIndex"; static final String HNSW_INDEX_EXT = "vex"; + static final LuceneProvider LUCENE_PROVIDER; private static CuVSResources resources = cuVSResourcesOrNull(); - /** The format for storing, reading, and merging raw vectors on disk. */ - private static final FlatVectorsFormat flatVectorsFormat = - new Lucene99FlatVectorsFormat(DefaultFlatVectorScorer.INSTANCE); + private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; + private static final Integer MAX_CONN; + private static final Integer BEAM_WIDTH; + private static final Integer NUM_MERGE_WORKERS; private final int maxDimensions = 4096; private final int cuvsWriterThreads; private final int intGraphDegree; private final int graphDegree; private final int hnswLayers; - private final int maxConn; private final int beamWidth; + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + MAX_CONN = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_MAX_CONN"); + BEAM_WIDTH = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); + NUM_MERGE_WORKERS = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); + FLAT_VECTORS_FORMAT = + LUCENE_PROVIDER.getLuceneFlatVectorsFormatInstance(DefaultFlatVectorScorer.INSTANCE); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + /** * Initializes {@link Lucene99AcceleratedHNSWVectorsFormat} with default values. * @@ -70,8 +78,8 @@ public Lucene99AcceleratedHNSWVectorsFormat() { DEFAULT_INTERMEDIATE_GRAPH_DEGREE, DEFAULT_GRAPH_DEGREE, DEFAULT_HNSW_GRAPH_LAYERS, - DEFAULT_MAX_CONN, - DEFAULT_BEAM_WIDTH); + MAX_CONN, + BEAM_WIDTH); } /** @@ -105,7 +113,7 @@ public Lucene99AcceleratedHNSWVectorsFormat( */ @Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - var flatWriter = flatVectorsFormat.fieldsWriter(state); + var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); if (supported()) { log.info("cuVS is supported so using the Lucene99AcceleratedHNSWVectorsWriter"); return new Lucene99AcceleratedHNSWVectorsWriter( @@ -114,8 +122,13 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException log.warning( "GPU based indexing not supported, falling back to using the Lucene99HnswVectorsWriter"); // TODO: Make num merge workers configurable. - return new Lucene99HnswVectorsWriter( - state, maxConn, beamWidth, flatWriter, DEFAULT_NUM_MERGE_WORKER, null); + try { + return LUCENE_PROVIDER.getLuceneHnswVectorsWriterInstance( + state, maxConn, beamWidth, flatWriter, NUM_MERGE_WORKERS, null); + } catch (Exception e) { + // maybe there is a better suited option to throwing RuntimeException? Need to explore. + throw new RuntimeException(e.getMessage()); + } } } @@ -124,7 +137,13 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException */ @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { - return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state)); + try { + return LUCENE_PROVIDER.getLuceneHnswVectorsReaderInstance( + state, FLAT_VECTORS_FORMAT.fieldsReader(state)); + } catch (Exception e) { + // maybe there is a better suited option to throwing RuntimeException? Need to explore. + throw new RuntimeException(e.getMessage()); + } } /** diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index a0e4437197..8528ef4566 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -8,7 +8,6 @@ import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_EXT; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_EXT; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_NAME; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; import static org.apache.lucene.index.VectorEncoding.FLOAT32; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; @@ -37,7 +36,6 @@ import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.index.DocsWithFieldSet; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -76,6 +74,10 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { /** The name of the CUVS component for the info-stream * */ private static final String CUVS_COMPONENT = "CUVS"; + private static final LuceneProvider LUCENE_PROVIDER; + private static final Integer VERSION_CURRENT; + private static final List VECTOR_SIMILARITY_FUNCTIONS; + private final int cuvsWriterThreads; private final int intGraphDegree; private final int graphDegree; @@ -90,6 +92,16 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { private String vemFileName; private String vexFileName; + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + VERSION_CURRENT = LUCENE_PROVIDER.getStaticIntParam("VERSION_CURRENT"); + VECTOR_SIMILARITY_FUNCTIONS = LUCENE_PROVIDER.getSimilarityFunctions(); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + /** * Initializes {@link Lucene99AcceleratedHNSWVectorsWriter} * @@ -136,13 +148,13 @@ public Lucene99AcceleratedHNSWVectorsWriter( CodecUtil.writeIndexHeader( hnswMeta, HNSW_META_CODEC_NAME, - Lucene99HnswVectorsFormat.VERSION_CURRENT, + VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); CodecUtil.writeIndexHeader( hnswVectorIndex, HNSW_INDEX_CODEC_NAME, - Lucene99HnswVectorsFormat.VERSION_CURRENT, + VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); @@ -662,8 +674,8 @@ private void writeEmpty(FieldInfo fieldInfo) throws IOException { } static int distFuncToOrd(VectorSimilarityFunction func) { - for (int i = 0; i < SIMILARITY_FUNCTIONS.size(); i++) { - if (SIMILARITY_FUNCTIONS.get(i).equals(func)) { + for (int i = 0; i < VECTOR_SIMILARITY_FUNCTIONS.size(); i++) { + if (VECTOR_SIMILARITY_FUNCTIONS.get(i).equals(func)) { return (byte) i; } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java new file mode 100644 index 0000000000..00855f30e7 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java @@ -0,0 +1,201 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.List; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.TaskExecutor; + +/** + * Dynamically loads Lucene format, reader, and writer classes with a fallback mechanism. + * + * @since 25.12 + */ +public class LuceneProvider { + + static final Logger log = Logger.getLogger(LuceneProvider.class.getName()); + + private static final String BASE = "org.apache.lucene."; + private static String codecs = "codecs.lucene."; + private static String backwardCodecs = "backward_codecs.lucene."; + + private static String luceneflatVectorsFormat = + BASE + codecs + "LuceneFlatVectorsFormat"; + private static String luceneFlatVectorsFormatFallback = + BASE + backwardCodecs + "LuceneFlatVectorsFormat"; + + private static String luceneHnswVectorsFormat = + BASE + codecs + "LuceneHnswVectorsFormat"; + private static String luceneHnswVectorsFormatFallback = + BASE + backwardCodecs + "LuceneHnswVectorsFormat"; + + private static String luceneHnswVectorsReader = + BASE + codecs + "LuceneHnswVectorsReader"; + private static String luceneHnswVectorsReaderFallback = + BASE + backwardCodecs + "LuceneHnswVectorsReader"; + + private static String luceneHnswVectorsWriter = + BASE + codecs + "LuceneHnswVectorsWriter"; + private static String luceneHnswVectorsWriterFallback = + BASE + backwardCodecs + "LuceneHnswVectorsWriter"; + + private static String luceneCodec = BASE + codecs + "LuceneCodec"; + private static String luceneCodecFallback = BASE + backwardCodecs + "LuceneCodec"; + + private static LuceneProvider instance; + + private static MethodHandles.Lookup lookup = MethodHandles.lookup(); + + private Class flatVectorsFormat; + private Class hnswVectorsFormat; + private Class hnswVectorsReader; + private Class hnswVectorsWriter; + + public static LuceneProvider getInstance(String version) throws ClassNotFoundException { + if (instance == null) { + instance = new LuceneProvider(version); + } + return instance; + } + + private LuceneProvider(String version) throws ClassNotFoundException { + flatVectorsFormat = + loadClass( + setVersion(luceneflatVectorsFormat, version), + setVersion(luceneFlatVectorsFormatFallback, version)); + hnswVectorsFormat = + loadClass( + setVersion(luceneHnswVectorsFormat, version), + setVersion(luceneHnswVectorsFormatFallback, version)); + hnswVectorsReader = + loadClass( + setVersion(luceneHnswVectorsReader, version), + setVersion(luceneHnswVectorsReaderFallback, version)); + hnswVectorsWriter = + loadClass( + setVersion(luceneHnswVectorsWriter, version), + setVersion(luceneHnswVectorsWriterFallback, version)); + } + + private static String setVersion(String pkg, String version) { + return pkg.replaceAll("", version); + } + + private static Class loadClass(String defaultClassName, String fallbackClassName) + throws ClassNotFoundException { + try { + return Class.forName(defaultClassName); + } catch (ClassNotFoundException e) { + // Load class from fallback package. + try { + return Class.forName(fallbackClassName); + } catch (ClassNotFoundException e1) { + // Should not reach here. + log.severe("Unable to load class: " + fallbackClassName); + throw e1; + } + } + } + + public static Codec getCodec(String version) + throws ClassNotFoundException, + NoSuchMethodException, + SecurityException, + InstantiationException, + IllegalAccessException, + IllegalArgumentException, + InvocationTargetException { + Class codecClass = + loadClass(setVersion(luceneCodec, version), setVersion(luceneCodecFallback, version)); + Constructor codecClassConstructor = codecClass.getConstructor(); + return (Codec) codecClassConstructor.newInstance(); + } + + public FlatVectorsFormat getLuceneFlatVectorsFormatInstance(FlatVectorsScorer scorer) + throws Exception { + try { + Constructor luceneFlatVectorsFormatConstructor = + flatVectorsFormat.getConstructor(FlatVectorsScorer.class); + return (FlatVectorsFormat) luceneFlatVectorsFormatConstructor.newInstance(scorer); + } catch (Exception e) { + log.severe("Unable to initialize LuceneFlatVectorsFormat: " + e.getMessage()); + throw e; + } + } + + public KnnVectorsReader getLuceneHnswVectorsReaderInstance( + SegmentReadState state, FlatVectorsReader reader) throws Exception { + try { + Constructor luceneHnswVectorsReaderConstructor = + hnswVectorsReader.getConstructor(SegmentReadState.class, FlatVectorsReader.class); + return (KnnVectorsReader) luceneHnswVectorsReaderConstructor.newInstance(state, reader); + } catch (Exception e) { + log.severe("Unable to initialize LuceneHnswVectorsReader: " + e.getMessage()); + throw e; + } + } + + public KnnVectorsWriter getLuceneHnswVectorsWriterInstance( + SegmentWriteState state, + int maxConn, + int beamWidth, + FlatVectorsWriter writer, + int numMergeWorkers, + TaskExecutor executor) + throws Exception { + try { + Constructor luceneHnswVectorsWriterConstructor = + hnswVectorsWriter.getConstructor( + SegmentWriteState.class, + Integer.TYPE, + Integer.TYPE, + FlatVectorsWriter.class, + Integer.TYPE, + TaskExecutor.class); + return (KnnVectorsWriter) + luceneHnswVectorsWriterConstructor.newInstance( + state, maxConn, beamWidth, writer, numMergeWorkers, executor); + } catch (Exception e) { + log.severe("Unable to initialize LuceneHnswVectorsWriter: " + e.getMessage()); + throw e; + } + } + + public int getStaticIntParam(String param) throws ReflectiveOperationException { + try { + VarHandle varHandle = lookup.findStaticVarHandle(hnswVectorsFormat, param, Integer.TYPE); + return (int) varHandle.get(); + } catch (NoSuchFieldException | IllegalAccessException e) { + log.severe("Unable to get " + param + ": " + e.getMessage()); + throw e; + } + } + + public List getSimilarityFunctions() + throws ReflectiveOperationException { + try { + VarHandle varHandle = + lookup.findStaticVarHandle(hnswVectorsReader, "SIMILARITY_FUNCTIONS", List.class); + return (List) varHandle.get(); + } catch (NoSuchFieldException | IllegalAccessException e) { + log.severe("Unable to get SIMILARITY_FUNCTIONS: " + e.getMessage()); + throw e; + } + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index 3e5085a8c8..56c193667a 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -6,11 +6,11 @@ import com.nvidia.cuvs.LibraryException; import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; +import java.lang.reflect.InvocationTargetException; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene101.Lucene101Codec; /** * cuVS based codec for GPU based vector search @@ -32,9 +32,23 @@ public class CuVS2510GPUSearchCodec extends FilterCodec { /** * Default constructor for {@link CuVS2510GPUSearchCodec} + * @throws InvocationTargetException + * @throws IllegalArgumentException + * @throws IllegalAccessException + * @throws InstantiationException + * @throws SecurityException + * @throws NoSuchMethodException + * @throws ClassNotFoundException */ - public CuVS2510GPUSearchCodec() { - this(NAME, new Lucene101Codec()); + public CuVS2510GPUSearchCodec() + throws ClassNotFoundException, + NoSuchMethodException, + SecurityException, + InstantiationException, + IllegalAccessException, + IllegalArgumentException, + InvocationTargetException { + this(NAME, LuceneProvider.getCodec("101")); } /** diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java index 3dd698dc3c..0bc9f6e964 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java @@ -4,15 +4,12 @@ */ package com.nvidia.cuvs.lucene; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; - import com.nvidia.cuvs.LibraryException; +import java.lang.reflect.InvocationTargetException; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene101.Lucene101Codec; /** * CuVS based codec for GPU based vector search @@ -28,11 +25,31 @@ public class Lucene101AcceleratedHNSWCodec extends FilterCodec { private static final int DEFAULT_GRAPH_DEGREE = 64; private static final int DEFAULT_HNSW_LAYERS = 1; private static final String NAME = "Lucene101AcceleratedHNSWCodec"; + private static final LuceneProvider lucene99Provider; + private static final Integer maxConn; + private static final Integer beamWidth; private KnnVectorsFormat format; - public Lucene101AcceleratedHNSWCodec() { - this(NAME, new Lucene101Codec()); + static { + try { + lucene99Provider = LuceneProvider.getInstance("99"); + maxConn = lucene99Provider.getStaticIntParam("DEFAULT_MAX_CONN"); + beamWidth = lucene99Provider.getStaticIntParam("DEFAULT_BEAM_WIDTH"); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + + public Lucene101AcceleratedHNSWCodec() + throws ClassNotFoundException, + NoSuchMethodException, + SecurityException, + InstantiationException, + IllegalAccessException, + IllegalArgumentException, + InvocationTargetException { + this(NAME, LuceneProvider.getCodec("101")); } public Lucene101AcceleratedHNSWCodec(String name, Codec delegate) { @@ -46,8 +63,15 @@ public Lucene101AcceleratedHNSWCodec( int graphDegree, int hnswLayers, int maxConn, - int beamWidth) { - this(NAME, new Lucene101Codec()); + int beamWidth) + throws ClassNotFoundException, + NoSuchMethodException, + SecurityException, + InstantiationException, + IllegalAccessException, + IllegalArgumentException, + InvocationTargetException { + this(NAME, LuceneProvider.getCodec("101")); initializeFormat( cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); } @@ -58,8 +82,8 @@ private void initializeFormatDefaultValues() { DEFAULT_INTERMEDIATE_GRAPH_DEGREE, DEFAULT_GRAPH_DEGREE, DEFAULT_HNSW_LAYERS, - DEFAULT_MAX_CONN, - DEFAULT_BEAM_WIDTH); + maxConn, + beamWidth); } private void initializeFormat( diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java new file mode 100644 index 0000000000..2de6e660fa --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.junit.Test; + +/** + * Tests the backward compatibility mechanism. + * + * @since 25.12 + */ +public class TestBackCompat { + + @Test + public void testFallback() throws Exception { + // Lucene99Codec exists in the org.apache.lucene.backward_codecs.lucene99 + Codec c = LuceneProvider.getCodec("99"); + assertEquals(c.getName(), "Lucene99"); + } + + @Test(expected = ClassNotFoundException.class) + public void testNonexistentCodec() throws Exception { + LuceneProvider.getCodec("0"); + } + + @Test + public void testExistingComponents() throws Exception { + LuceneProvider provider = LuceneProvider.getInstance("99"); + assertTrue(provider.getLuceneFlatVectorsFormatInstance(null) instanceof FlatVectorsFormat); + assertEquals(provider.getStaticIntParam("VERSION_CURRENT"), 0); + assertNotEquals(provider.getSimilarityFunctions().size(), 0); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java index 2a11ff6369..7d35bbd34d 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java @@ -9,6 +9,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -57,7 +58,15 @@ public void beforeTest() throws Exception { } @Test - public void testCagraToHnswSerializationAndSearch() throws IOException { + public void testCagraToHnswSerializationAndSearch() + throws IOException, + ClassNotFoundException, + NoSuchMethodException, + SecurityException, + InstantiationException, + IllegalAccessException, + IllegalArgumentException, + InvocationTargetException { Codec codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java index 83758708ef..2058c29b4d 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java @@ -10,6 +10,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -61,7 +62,15 @@ public static void beforeClass() throws Exception { } @Test - public void testCagraToHnswSerializationAndSearchWithFallbackWriter() throws IOException { + public void testCagraToHnswSerializationAndSearchWithFallbackWriter() + throws IOException, + ClassNotFoundException, + NoSuchMethodException, + SecurityException, + InstantiationException, + IllegalAccessException, + IllegalArgumentException, + InvocationTargetException { Codec codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); From fceaf1a5a2bd5b66661f646f8e1bf2c7aa562be1 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 25 Nov 2025 11:40:57 -0500 Subject: [PATCH 040/105] Modify CI pipeline to use latest cuvs-java instead of the one released on Maven Central (NVIDIA/cuvs-lucene#49) Fixes NVIDIA/cuvs-lucene#43 Please note: I had to bump up the `cuda_version` to `13.0.2` because for `26.02` the [image with the expected tag is not available](https://hub.docker.com/r/rapidsai/ci-conda/tags?name=26.02-cuda13.0.0-ubuntu24.04-py3.13) but for `13.0.2` [it does](https://hub.docker.com/r/rapidsai/ci-conda/tags?name=26.02-cuda13.0.2-ubuntu24.04-py3.13). Also, this PR includes updating the `FilterCuVSProvider.java` to adapt to the latest `cuvs-java` changes. Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Bradley Dice (https://github.com/bdice) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/49 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 2 +- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/.gitignore | 1 + java/cuvs-lucene/RAPIDS_BRANCH | 1 + java/cuvs-lucene/build.sh | 18 ++++++++++++ java/cuvs-lucene/ci/build_java.sh | 3 ++ java/cuvs-lucene/ci/release/update-version.sh | 4 +++ .../all_cuda-129_arch-aarch64.yaml | 2 ++ .../all_cuda-129_arch-x86_64.yaml | 2 ++ .../all_cuda-130_arch-aarch64.yaml | 2 ++ .../all_cuda-130_arch-x86_64.yaml | 2 ++ java/cuvs-lucene/dependencies.yaml | 2 ++ java/cuvs-lucene/pom.xml | 2 -- .../cuvs/lucene/FilterCuVSProvider.java | 29 +++++++++++++++++++ 15 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 java/cuvs-lucene/RAPIDS_BRANCH diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index a52040f81d..cd23671975 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -42,7 +42,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.0.0' + - '13.0.2' with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index e81ea84cd0..59864a4b65 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -68,7 +68,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.0.0' + - '13.0.2' with: build_type: pull-request node_type: "gpu-l4-latest-1" diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 1852fa3bb8..8724ad220a 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -31,7 +31,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.0.0' + - '13.0.2' with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} diff --git a/java/cuvs-lucene/.gitignore b/java/cuvs-lucene/.gitignore index 32d6a5b09e..f5287f1f11 100644 --- a/java/cuvs-lucene/.gitignore +++ b/java/cuvs-lucene/.gitignore @@ -4,3 +4,4 @@ *.jar target **/.DS_Store +cuvs-workdir diff --git a/java/cuvs-lucene/RAPIDS_BRANCH b/java/cuvs-lucene/RAPIDS_BRANCH new file mode 100644 index 0000000000..ba2906d066 --- /dev/null +++ b/java/cuvs-lucene/RAPIDS_BRANCH @@ -0,0 +1 @@ +main diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index 1e7ba033c5..ccff181832 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -15,6 +15,24 @@ function hasArg { (( NUMARGS != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") } +if hasArg --build-cuvs-java; then + CUVS_WORKDIR="cuvs-workdir" + CUVS_GIT_REPO="https://github.com/rapidsai/cuvs.git" + if [[ -d "$CUVS_WORKDIR" && -n "$(ls -A "$CUVS_WORKDIR")" ]]; then + echo "Directory '$CUVS_WORKDIR' exists and is not empty." + pushd $CUVS_WORKDIR + git pull + else + BRANCH=$(cat "RAPIDS_BRANCH") + echo "Directory '$CUVS_WORKDIR' does not exist or is empty. Cloning the cuvs's '$BRANCH' branch." + # Correct branch selection is crucial to avoid version mismatch issues when testing. + git clone --branch "$BRANCH" $CUVS_GIT_REPO $CUVS_WORKDIR + pushd $CUVS_WORKDIR + fi + ./build.sh java + popd +fi + MAVEN_VERIFY_ARGS=() if ! hasArg --run-java-tests; then MAVEN_VERIFY_ARGS=("-DskipTests") diff --git a/java/cuvs-lucene/ci/build_java.sh b/java/cuvs-lucene/ci/build_java.sh index 2f61f4b5a7..b586ec9e8b 100755 --- a/java/cuvs-lucene/ci/build_java.sh +++ b/java/cuvs-lucene/ci/build_java.sh @@ -13,6 +13,9 @@ if [[ "${1:-}" == "--run-java-tests" ]]; then EXTRA_BUILD_ARGS+=("--run-java-tests") fi +# Always build cuvs-java when running the pipeline +EXTRA_BUILD_ARGS+=("--build-cuvs-java") + # shellcheck disable=SC1091 . /opt/conda/etc/profile.d/conda.sh diff --git a/java/cuvs-lucene/ci/release/update-version.sh b/java/cuvs-lucene/ci/release/update-version.sh index 61848ed9c8..afe73f640d 100755 --- a/java/cuvs-lucene/ci/release/update-version.sh +++ b/java/cuvs-lucene/ci/release/update-version.sh @@ -86,8 +86,10 @@ PATCH_PEP440=$(python -c "from packaging.version import Version; print(Version(' # Log update context if [[ "${RUN_CONTEXT}" == "main" ]]; then + RAPIDS_BRANCH_NAME="main" echo "Preparing development branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting main branch)" elif [[ "${RUN_CONTEXT}" == "release" ]]; then + RAPIDS_BRANCH_NAME="release/${NEXT_SHORT_TAG}" echo "Preparing release branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting release/${NEXT_SHORT_TAG} branch)" fi @@ -98,6 +100,7 @@ function sed_runner() { # Centralized version file update echo "${NEXT_FULL_TAG}" > VERSION +echo "${RAPIDS_BRANCH_NAME}" > RAPIDS_BRANCH # Update Java version NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}" @@ -111,5 +114,6 @@ for FILE in dependencies.yaml conda/environments/*.yaml; do done for FILE in .github/workflows/*.yaml; do + sed_runner "/shared-workflows/ s|@.*|@${RAPIDS_BRANCH_NAME}|g" "${FILE}" sed_runner "s/:[0-9]*\\.[0-9]*-/:${NEXT_SHORT_TAG}-/g" "${FILE}" done diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index 816c8fcf11..53faa85e92 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -4,6 +4,7 @@ channels: - conda-forge - rapidsai-nightly dependencies: +- cmake>=3.30.4 - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api @@ -14,5 +15,6 @@ dependencies: - libcusparse-dev - libcuvs==25.12.* - maven +- ninja - openjdk=22.* name: all_cuda-129_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index 447b98037a..753a08ae6f 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -4,6 +4,7 @@ channels: - conda-forge - rapidsai-nightly dependencies: +- cmake>=3.30.4 - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api @@ -14,5 +15,6 @@ dependencies: - libcusparse-dev - libcuvs==25.12.* - maven +- ninja - openjdk=22.* name: all_cuda-129_arch-x86_64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml index 6fb38df428..0a72d8ff2e 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -4,6 +4,7 @@ channels: - conda-forge - rapidsai-nightly dependencies: +- cmake>=3.30.4 - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api @@ -14,5 +15,6 @@ dependencies: - libcusparse-dev - libcuvs==25.12.* - maven +- ninja - openjdk=22.* name: all_cuda-130_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml index 3be3d797b0..00e7006a09 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -4,6 +4,7 @@ channels: - conda-forge - rapidsai-nightly dependencies: +- cmake>=3.30.4 - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api @@ -14,5 +15,6 @@ dependencies: - libcusparse-dev - libcuvs==25.12.* - maven +- ninja - openjdk=22.* name: all_cuda-130_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index 7066248f89..1acefd9549 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -75,5 +75,7 @@ dependencies: common: - output_types: conda packages: + - cmake>=3.30.4 - maven + - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 1562081115..ec31f8478d 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -41,8 +41,6 @@ 21 - - org.apache.lucene diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index 01752d10cb..aaf722191d 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -6,14 +6,19 @@ import com.nvidia.cuvs.BruteForceIndex; import com.nvidia.cuvs.CagraIndex; +import com.nvidia.cuvs.CagraIndexParams; +import com.nvidia.cuvs.CagraIndexParams.CuvsDistanceType; +import com.nvidia.cuvs.CagraIndexParams.HnswHeuristicType; import com.nvidia.cuvs.CuVSMatrix; import com.nvidia.cuvs.CuVSMatrix.Builder; import com.nvidia.cuvs.CuVSMatrix.DataType; import com.nvidia.cuvs.CuVSResources; import com.nvidia.cuvs.HnswIndex; +import com.nvidia.cuvs.HnswIndexParams; import com.nvidia.cuvs.spi.CuVSProvider; import java.lang.invoke.MethodHandle; import java.nio.file.Path; +import java.util.logging.Level; /*package-private*/ class FilterCuVSProvider implements CuVSProvider { @@ -114,4 +119,28 @@ public com.nvidia.cuvs.TieredIndex.Builder newTieredIndexBuilder(CuVSResources c throws UnsupportedOperationException { return delegate.newTieredIndexBuilder(cuVSResources); } + + @Override + public CagraIndexParams cagraIndexParamsFromHnswParams( + long arg0, long arg1, int arg2, int arg3, HnswHeuristicType arg4, CuvsDistanceType arg5) { + // TODO Auto-generated method stub + return null; + } + + @Override + public Level getLogLevel() { + // TODO Auto-generated method stub + return null; + } + + @Override + public void setLogLevel(Level arg0) { + // TODO Auto-generated method stub + } + + @Override + public HnswIndex hnswIndexFromCagra(HnswIndexParams arg0, CagraIndex arg1) throws Throwable { + // TODO Auto-generated method stub + return null; + } } From de0b8e480ddf7aea72ca2b29a2be6ee75133f2bb Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 25 Nov 2025 13:05:58 -0600 Subject: [PATCH 041/105] Use strict priority in CI conda tests (NVIDIA/cuvs-lucene#53) This PR sets conda to use `strict` priority in CI tests. Mixing channel priority is frequently a cause of unexpected errors. Our CI jobs should always use strict priority in order to enforce that conda packages come from local channels with the artifacts built in CI, not mixing with older nightly artifacts from the `rapidsai-nightly` channel or other sources. xref: https://github.com/rapidsai/build-planning/issues/14 Authors: - Bradley Dice (https://github.com/bdice) - Vivek Narang (https://github.com/narangvivek10) - Gil Forsyth (https://github.com/gforsyth) Approvers: - Gil Forsyth (https://github.com/gforsyth) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/53 --- java/cuvs-lucene/ci/build_java.sh | 3 +++ .../conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- .../conda/environments/all_cuda-130_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-130_arch-x86_64.yaml | 2 +- java/cuvs-lucene/dependencies.yaml | 2 +- 6 files changed, 8 insertions(+), 5 deletions(-) diff --git a/java/cuvs-lucene/ci/build_java.sh b/java/cuvs-lucene/ci/build_java.sh index b586ec9e8b..173ca55c28 100755 --- a/java/cuvs-lucene/ci/build_java.sh +++ b/java/cuvs-lucene/ci/build_java.sh @@ -19,6 +19,9 @@ EXTRA_BUILD_ARGS+=("--build-cuvs-java") # shellcheck disable=SC1091 . /opt/conda/etc/profile.d/conda.sh +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + rapids-logger "Generate Java testing dependencies" ENV_YAML_DIR="$(mktemp -d)" diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index 53faa85e92..f26fce99de 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -1,8 +1,8 @@ # This file is generated by `rapids-dependency-file-generator`. # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: -- conda-forge - rapidsai-nightly +- conda-forge dependencies: - cmake>=3.30.4 - cuda-cudart-dev diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index 753a08ae6f..e0d6a1d1eb 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -1,8 +1,8 @@ # This file is generated by `rapids-dependency-file-generator`. # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: -- conda-forge - rapidsai-nightly +- conda-forge dependencies: - cmake>=3.30.4 - cuda-cudart-dev diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml index 0a72d8ff2e..a22ab63dea 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -1,8 +1,8 @@ # This file is generated by `rapids-dependency-file-generator`. # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: -- conda-forge - rapidsai-nightly +- conda-forge dependencies: - cmake>=3.30.4 - cuda-cudart-dev diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml index 00e7006a09..21403cfe3d 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -1,8 +1,8 @@ # This file is generated by `rapids-dependency-file-generator`. # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: -- conda-forge - rapidsai-nightly +- conda-forge dependencies: - cmake>=3.30.4 - cuda-cudart-dev diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index 1acefd9549..f638b76b24 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -23,8 +23,8 @@ files: - cuda_version - java channels: - - conda-forge - rapidsai-nightly + - conda-forge dependencies: checks: common: From ba1ed61a858bee43eafc469db8a7b1889d75a21e Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 25 Nov 2025 15:34:36 -0500 Subject: [PATCH 042/105] Setup Java Code Coverage and add more tests (NVIDIA/cuvs-lucene#42) Fixes NVIDIA/cuvs-lucene#40 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cuvs-lucene/pull/42 --- java/cuvs-lucene/.gitignore | 1 + java/cuvs-lucene/build.sh | 7 +- java/cuvs-lucene/pom.xml | 19 + .../TestAcceleratedHNSWDeletedDocuments.java | 332 ++++++++++++++++++ ...stCuVSAcceleratedHNSWDeletedDocuments.java | 332 ++++++++++++++++++ .../lucene/TestCuVSAcceleratedHNSWGaps.java | 187 ++++++++++ .../cuvs/lucene/TestCuVSDeletedDocuments.java | 4 +- .../com/nvidia/cuvs/lucene/TestCuVSGaps.java | 2 +- .../TestCuVSRandomizedHNSWVectorSearch.java | 215 ++++++++++++ .../TestCuVSRandomizedVectorSearch.java | 2 +- ...tLucene99AcceleratedHNSWVectorsFormat.java | 155 ++++++++ .../com/nvidia/cuvs/lucene/TestMerge.java | 2 +- 12 files changed, 1252 insertions(+), 6 deletions(-) create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java diff --git a/java/cuvs-lucene/.gitignore b/java/cuvs-lucene/.gitignore index f5287f1f11..571838e7cc 100644 --- a/java/cuvs-lucene/.gitignore +++ b/java/cuvs-lucene/.gitignore @@ -5,3 +5,4 @@ target **/.DS_Store cuvs-workdir +bin diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index ccff181832..cd901765d3 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -38,6 +38,11 @@ if ! hasArg --run-java-tests; then MAVEN_VERIFY_ARGS=("-DskipTests") fi -mvn verify "${MAVEN_VERIFY_ARGS[@]}" \ +mvn clean verify "${MAVEN_VERIFY_ARGS[@]}" \ && mvn install:install-file -Dfile=./target/cuvs-lucene-$VERSION.jar -DgroupId=$GROUP_ID -DartifactId=cuvs-lucene -Dversion=$VERSION -Dpackaging=jar \ && cp pom.xml ./target/ + +# Generate JaCoCo code coverage reports available here: target/site/jacoco/index.html +if hasArg --run-java-tests; then + mvn jacoco:report +fi diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index ec31f8478d..ed47e06a60 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -165,6 +165,25 @@ + + org.jacoco + jacoco-maven-plugin + 0.8.14 + + + + prepare-agent + + + + report + prepare-package + + report + + + + diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java new file mode 100644 index 0000000000..d46794df1c --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java @@ -0,0 +1,332 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.TestUtils.generateRandomVector; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.analysis.MockTokenizer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestAcceleratedHNSWDeletedDocuments extends LuceneTestCase { + + protected static Logger log = + Logger.getLogger(TestAcceleratedHNSWDeletedDocuments.class.getName()); + + static final Codec codec = + TestUtil.alwaysKnnVectorsFormat(new Lucene99AcceleratedHNSWVectorsFormat()); + private static Random random; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + random = random(); + } + + @Test + public void testVectorSearchWithDeletedDocuments() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200, 1000); // 200-1200 documents + int dimensions = random.nextInt(64, 256); // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float deletionProbability = random.nextFloat() * 0.4f + 0.1f; // 10-50% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + Set deletedDocs = new HashSet<>(); + + // Create index with all documents having vectors + try (RandomIndexWriter writer = createWriter(directory)) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + } + + // Delete documents randomly based on probability + for (int i = 0; i < datasetSize; i++) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + deletedDocs.add(i); + } + } + writer.commit(); + } + + // Search and verify deleted documents are not returned + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + // Use a random vector for query + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify we got results + assertTrue("Should have search results", hits.length > 0); + + // Verify no deleted documents in results + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertFalse( + "Deleted document " + id + " should not appear in results", deletedDocs.contains(id)); + log.info("Found non-deleted document: " + id + ", Score: " + hit.score); + } + + // Verify deleted documents are truly deleted + for (int deletedId : deletedDocs) { + TopDocs result = + searcher.search(new TermQuery(new Term("id", String.valueOf(deletedId))), 1); + assertEquals( + "Deleted document " + deletedId + " should not be found", + 0, + result.totalHits.value()); + } + } + } + } + + @Test + public void testVectorSearchWithMixedDeletedAndMissingVectors() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200) + 50; // 50-250 documents + int dimensions = random.nextInt(256) + 64; // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float vectorProbability = random.nextFloat() * 0.5f + 0.3f; // 30-80% have vectors + float deletionProbability = random.nextFloat() * 0.3f + 0.1f; // 10-40% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + Set docsWithoutVectors = new HashSet<>(); + Set deletedDocs = new HashSet<>(); + + // Create index with mixed documents + try (RandomIndexWriter writer = createWriter(directory)) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + // Randomly assign categories + String category = random.nextBoolean() ? "A" : "B"; + doc.add(new StringField("category", category, Field.Store.YES)); + + // Randomly decide whether to add vectors + if (random.nextFloat() < vectorProbability) { + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + } else { + docsWithoutVectors.add(i); + } + writer.addDocument(doc); + } + + // Delete documents randomly + for (int i = 0; i < datasetSize; i++) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + deletedDocs.add(i); + } + } + writer.commit(); + } + + // Test vector search behavior + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify results + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertFalse("Deleted document should not appear", deletedDocs.contains(id)); + assertFalse("Document without vector should not appear", docsWithoutVectors.contains(id)); + log.info("Found document with vector: " + id + ", Score: " + hit.score); + } + + // Test filtered search with deletions + Query filter = new TermQuery(new Term("category", "A")); + Query filteredQuery = new KnnFloatVectorQuery("vector", queryVector, topK, filter); + ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; + + for (ScoreDoc hit : filteredHits) { + Document doc = reader.storedFields().document(hit.doc); + String category = doc.get("category"); + assertEquals("Should only match category A", "A", category); + int id = Integer.parseInt(doc.get("id")); + assertFalse( + "Deleted document should not appear in filtered results", deletedDocs.contains(id)); + } + } + } + } + + @Test + public void testVectorSearchAfterAllDocumentsDeleted() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(20) + 5; // 5-25 documents for this test + int dimensions = random.nextInt(128) + 32; // 32-160 dimensions + int topK = Math.min(random.nextInt(10) + 5, datasetSize); // 5-15 results + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + + // Create and delete all documents + try (IndexWriter writer = new IndexWriter(directory, createWriterConfig())) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + } + writer.commit(); + + // Delete all documents + for (int i = 0; i < datasetSize; i++) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + } + writer.commit(); + writer.forceMerge(1); // Force merge to apply deletions + } + + // Verify search returns no results + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + TopDocs results = searcher.search(query, topK); + + assertEquals( + "Should return no results when all documents are deleted", + 0, + results.totalHits.value()); + } + } + } + + @Test + public void testVectorSearchWithPartialDeletionAndReindexing() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200) + 50; // 50-250 documents + int dimensions = random.nextInt(256) + 64; // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float deletionProbability = random.nextFloat() * 0.3f + 0.1f; // 10-40% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + List activeDocIds = new ArrayList<>(); + + // Initial indexing + try (IndexWriter writer = new IndexWriter(directory, createWriterConfig())) { + int initialDocs = datasetSize / 2 + random.nextInt(datasetSize / 4); // 50-75% of dataset + for (int i = 0; i < initialDocs; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + activeDocIds.add(i); + } + + // Delete some documents randomly + List candidatesForDeletion = new ArrayList<>(activeDocIds); + for (int docId : candidatesForDeletion) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(docId))); + activeDocIds.remove(Integer.valueOf(docId)); + } + } + + // Add new documents with higher IDs + for (int i = initialDocs; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + activeDocIds.add(i); + } + writer.commit(); + } + + // Verify search behavior after deletions and additions + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + Set resultIds = new HashSet<>(); + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + resultIds.add(id); + assertTrue("Result should be from active documents", activeDocIds.contains(id)); + } + + log.info( + "Search returned " + + hits.length + + " results from " + + activeDocIds.size() + + " active documents"); + } + } + } + + private RandomIndexWriter createWriter(Directory directory) throws IOException { + return new RandomIndexWriter( + random(), + directory, + newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy())); + } + + private IndexWriterConfig createWriterConfig() { + return newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy()); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java new file mode 100644 index 0000000000..49372b6b00 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java @@ -0,0 +1,332 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.TestUtils.generateRandomVector; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.analysis.MockTokenizer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestCuVSAcceleratedHNSWDeletedDocuments extends LuceneTestCase { + + protected static Logger log = + Logger.getLogger(TestCuVSAcceleratedHNSWDeletedDocuments.class.getName()); + + static final Codec codec = + TestUtil.alwaysKnnVectorsFormat(new Lucene99AcceleratedHNSWVectorsFormat()); + private static Random random; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + random = random(); + } + + @Test + public void testVectorSearchWithDeletedDocuments() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200, 1000); // 200-1200 documents + int dimensions = random.nextInt(64, 256); // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float deletionProbability = random.nextFloat() * 0.4f + 0.1f; // 10-50% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + Set deletedDocs = new HashSet<>(); + + // Create index with all documents having vectors + try (RandomIndexWriter writer = createWriter(directory)) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + } + + // Delete documents randomly based on probability + for (int i = 0; i < datasetSize; i++) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + deletedDocs.add(i); + } + } + writer.commit(); + } + + // Search and verify deleted documents are not returned + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + // Use a random vector for query + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify we got results + assertTrue("Should have search results", hits.length > 0); + + // Verify no deleted documents in results + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertFalse( + "Deleted document " + id + " should not appear in results", deletedDocs.contains(id)); + log.info("Found non-deleted document: " + id + ", Score: " + hit.score); + } + + // Verify deleted documents are truly deleted + for (int deletedId : deletedDocs) { + TopDocs result = + searcher.search(new TermQuery(new Term("id", String.valueOf(deletedId))), 1); + assertEquals( + "Deleted document " + deletedId + " should not be found", + 0, + result.totalHits.value()); + } + } + } + } + + @Test + public void testVectorSearchWithMixedDeletedAndMissingVectors() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200) + 50; // 50-250 documents + int dimensions = random.nextInt(256) + 64; // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float vectorProbability = random.nextFloat() * 0.5f + 0.3f; // 30-80% have vectors + float deletionProbability = random.nextFloat() * 0.3f + 0.1f; // 10-40% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + Set docsWithoutVectors = new HashSet<>(); + Set deletedDocs = new HashSet<>(); + + // Create index with mixed documents + try (RandomIndexWriter writer = createWriter(directory)) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + // Randomly assign categories + String category = random.nextBoolean() ? "A" : "B"; + doc.add(new StringField("category", category, Field.Store.YES)); + + // Randomly decide whether to add vectors + if (random.nextFloat() < vectorProbability) { + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + } else { + docsWithoutVectors.add(i); + } + writer.addDocument(doc); + } + + // Delete documents randomly + for (int i = 0; i < datasetSize; i++) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + deletedDocs.add(i); + } + } + writer.commit(); + } + + // Test vector search behavior + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify results + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertFalse("Deleted document should not appear", deletedDocs.contains(id)); + assertFalse("Document without vector should not appear", docsWithoutVectors.contains(id)); + log.info("Found document with vector: " + id + ", Score: " + hit.score); + } + + // Test filtered search with deletions + Query filter = new TermQuery(new Term("category", "A")); + Query filteredQuery = new KnnFloatVectorQuery("vector", queryVector, topK, filter); + ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; + + for (ScoreDoc hit : filteredHits) { + Document doc = reader.storedFields().document(hit.doc); + String category = doc.get("category"); + assertEquals("Should only match category A", "A", category); + int id = Integer.parseInt(doc.get("id")); + assertFalse( + "Deleted document should not appear in filtered results", deletedDocs.contains(id)); + } + } + } + } + + @Test + public void testVectorSearchAfterAllDocumentsDeleted() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(20) + 5; // 5-25 documents for this test + int dimensions = random.nextInt(128) + 32; // 32-160 dimensions + int topK = Math.min(random.nextInt(10) + 5, datasetSize); // 5-15 results + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + + // Create and delete all documents + try (IndexWriter writer = new IndexWriter(directory, createWriterConfig())) { + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + } + writer.commit(); + + // Delete all documents + for (int i = 0; i < datasetSize; i++) { + writer.deleteDocuments(new Term("id", String.valueOf(i))); + } + writer.commit(); + writer.forceMerge(1); // Force merge to apply deletions + } + + // Verify search returns no results + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + TopDocs results = searcher.search(query, topK); + + assertEquals( + "Should return no results when all documents are deleted", + 0, + results.totalHits.value()); + } + } + } + + @Test + public void testVectorSearchWithPartialDeletionAndReindexing() throws IOException { + + try (Directory directory = newDirectory()) { + int datasetSize = random.nextInt(200) + 50; // 50-250 documents + int dimensions = random.nextInt(256) + 64; // 64-320 dimensions + int topK = Math.min(random.nextInt(20) + 5, datasetSize / 2); // 5-25 results + float deletionProbability = random.nextFloat() * 0.3f + 0.1f; // 10-40% deletion rate + + float[][] dataset = generateDataset(random, datasetSize, dimensions); + List activeDocIds = new ArrayList<>(); + + // Initial indexing + try (IndexWriter writer = new IndexWriter(directory, createWriterConfig())) { + int initialDocs = datasetSize / 2 + random.nextInt(datasetSize / 4); // 50-75% of dataset + for (int i = 0; i < initialDocs; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + activeDocIds.add(i); + } + + // Delete some documents randomly + List candidatesForDeletion = new ArrayList<>(activeDocIds); + for (int docId : candidatesForDeletion) { + if (random.nextFloat() < deletionProbability) { + writer.deleteDocuments(new Term("id", String.valueOf(docId))); + activeDocIds.remove(Integer.valueOf(docId)); + } + } + + // Add new documents with higher IDs + for (int i = initialDocs; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add( + new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + writer.addDocument(doc); + activeDocIds.add(i); + } + writer.commit(); + } + + // Verify search behavior after deletions and additions + try (DirectoryReader reader = DirectoryReader.open(directory)) { + IndexSearcher searcher = newSearcher(reader); + float[] queryVector = generateRandomVector(dimensions, random); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + Set resultIds = new HashSet<>(); + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + resultIds.add(id); + assertTrue("Result should be from active documents", activeDocIds.contains(id)); + } + + log.info( + "Search returned " + + hits.length + + " results from " + + activeDocIds.size() + + " active documents"); + } + } + } + + private RandomIndexWriter createWriter(Directory directory) throws IOException { + return new RandomIndexWriter( + random(), + directory, + newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy())); + } + + private IndexWriterConfig createWriterConfig() { + return newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy()); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java new file mode 100644 index 0000000000..58e9acc5c6 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.analysis.MockTokenizer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.English; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestCuVSAcceleratedHNSWGaps extends LuceneTestCase { + + protected static Logger log = Logger.getLogger(TestCuVSAcceleratedHNSWGaps.class.getName()); + + static final Codec codec = + TestUtil.alwaysKnnVectorsFormat(new Lucene99AcceleratedHNSWVectorsFormat()); + static IndexSearcher searcher; + static IndexReader reader; + static Directory directory; + static Random random; + + static int DATASET_SIZE_LIMIT = 1000; + static int DIMENSIONS_LIMIT = 2048; + static int NUM_QUERIES_LIMIT = 10; + static int TOP_K_LIMIT = 64; + + static int datasetSize; + static int dimension; + static float[][] dataset; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + directory = newDirectory(); + random = random(); + + RandomIndexWriter writer = + new RandomIndexWriter( + random(), + directory, + newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy())); + + log.info("Merge Policy: " + writer.w.getConfig().getMergePolicy()); + + datasetSize = random.nextInt(100, DATASET_SIZE_LIMIT); + dimension = random.nextInt(8, DIMENSIONS_LIMIT); + dataset = generateDataset(random, datasetSize, dimension); + + // Create documents where only even-numbered documents have vectors + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(newTextField("field", English.intToEnglish(i), Field.Store.YES)); + + // Only add vectors to even-numbered documents + if (i % 2 == 0) { + doc.add(new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + } + + writer.addDocument(doc); + } + + reader = writer.getReader(); + searcher = newSearcher(reader); + writer.close(); + } + + @AfterClass + public static void afterClass() throws Exception { + if (reader != null) reader.close(); + if (directory != null) directory.close(); + searcher = null; + reader = null; + directory = null; + log.info("Test finished"); + } + + @Test + public void testVectorSearchWithAlternatingDocuments() throws IOException { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + + // Use the first vector (from document 0) as query + float[] queryVector = dataset[0]; + int topK = random.nextInt(5, TOP_K_LIMIT); + + Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + + // Verify we get exactly TOP_K results + assertEquals("Should return exactly " + topK + " results", topK, hits.length); + + // Verify all returned documents have vectors (even-numbered IDs) + for (ScoreDoc hit : hits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + int id = Integer.parseInt(docId); + assertEquals("All results should be even-numbered (have vectors)", 0, id % 2); + log.info("Document ID: " + id + ", Score: " + hit.score); + } + + // Verify the results match expected top-k based on Euclidean distance + List expectedIds = calculateExpectedTopK(queryVector, topK, dataset); + for (int i = 0; i < hits.length; i++) { + String docId = reader.storedFields().document(hits[i].doc).get("id"); + int id = Integer.parseInt(docId); + assertTrue("Result " + id + " should be in expected top-k results", expectedIds.contains(id)); + } + + log.info("Alternating document test passed with " + hits.length + " results"); + } + + @Test + public void testVectorSearchWithFilterAndAlternatingDocuments() throws IOException { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + + // Use the first vector (from document 0) as query + float[] queryVector = dataset[0]; + int topK = random.nextInt(5, TOP_K_LIMIT); + + // Create a filter that only matches documents with ID less than 10 + // This should further restrict our results to even numbers 0, 2, 4, 6, 8 + Query filter = new TermQuery(new Term("id", "8")); // Only match document 8 + + Query filteredQuery = new KnnFloatVectorQuery("vector", queryVector, topK, filter); + ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; + + // Should only get document 8 (the only one that matches the filter and has a vector) + assertEquals("Should return exactly 1 result", 1, filteredHits.length); + + String docId = reader.storedFields().document(filteredHits[0].doc).get("id"); + assertEquals("Should only return document 8", "8", docId); + + log.info("Filtered alternating document test passed with " + filteredHits.length + " results"); + } + + public static List calculateExpectedTopK(float[] query, int topK, float[][] dataset) { + Map distances = new TreeMap<>(); + + // Calculate distances only for documents that have vectors (even-numbered) + for (int i = 0; i < dataset.length; i += 2) { + double distance = 0; + for (int j = 0; j < dataset[0].length; j++) { + distance += (query[j] - dataset[i][j]) * (query[j] - dataset[i][j]); + } + distances.put(i, distance); + } + + // Sort by distance and return top-k + return distances.entrySet().stream() + .sorted(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .limit(topK) + .toList(); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java index 696e6bd0e3..179399bba0 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java @@ -40,7 +40,7 @@ import org.junit.BeforeClass; import org.junit.Test; -@SuppressSysoutChecks(bugUrl = "prints info from within cuVS") +@SuppressSysoutChecks(bugUrl = "") public class TestCuVSDeletedDocuments extends LuceneTestCase { protected static Logger log = Logger.getLogger(TestCuVSDeletedDocuments.class.getName()); @@ -50,7 +50,7 @@ public class TestCuVSDeletedDocuments extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { - assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); random = random(); } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java index 38035c1c76..39938d31d1 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java @@ -37,7 +37,7 @@ import org.junit.BeforeClass; import org.junit.Test; -@SuppressSysoutChecks(bugUrl = "prints info from within cuVS") +@SuppressSysoutChecks(bugUrl = "") public class TestCuVSGaps extends LuceneTestCase { protected static Logger log = Logger.getLogger(TestCuVSGaps.class.getName()); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java new file mode 100644 index 0000000000..c8c0485c9e --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java @@ -0,0 +1,215 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.TestUtils.generateQueries; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.TreeMap; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.analysis.MockTokenizer; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.apache.lucene.tests.util.English; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestCuVSRandomizedHNSWVectorSearch extends LuceneTestCase { + + protected static Logger log = + Logger.getLogger(TestCuVSRandomizedHNSWVectorSearch.class.getName()); + + static final Codec codec = + TestUtil.alwaysKnnVectorsFormat(new Lucene99AcceleratedHNSWVectorsFormat()); + static IndexSearcher searcher; + static IndexReader reader; + static Directory directory; + + static int DATASET_SIZE_LIMIT = 1000; + static int DIMENSIONS_LIMIT = 2048; + static int NUM_QUERIES_LIMIT = 10; + static int TOP_K_LIMIT = 64; // TODO This fails beyond 64 + static float[][] dataset; + + @BeforeClass + public static void beforeClass() throws Exception { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + directory = newDirectory(); + + RandomIndexWriter writer = + new RandomIndexWriter( + random(), + directory, + newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true)) + .setMaxBufferedDocs(TestUtil.nextInt(random(), 100, 1000)) + .setCodec(codec) + .setMergePolicy(newTieredMergePolicy())); + + log.info("Merge Policy: " + writer.w.getConfig().getMergePolicy()); + + Random random = random(); + int datasetSize = random.nextInt(DATASET_SIZE_LIMIT) + 1; + int dimensions = random.nextInt(DIMENSIONS_LIMIT) + 1; + dataset = generateDataset(random, datasetSize, dimensions); + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(newTextField("field", English.intToEnglish(i), Field.Store.YES)); + boolean skipVector = + random.nextInt(10) + < 4; // some documents won't have vectors to test deleted/missing vectors + if (!skipVector + || datasetSize < 100) { // about 10th of the documents shouldn't have a single vector + doc.add(new KnnFloatVectorField("vector", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + doc.add(new KnnFloatVectorField("vector2", dataset[i], VectorSimilarityFunction.EUCLIDEAN)); + } + + writer.addDocument(doc); + } + + reader = writer.getReader(); + searcher = newSearcher(reader); + writer.close(); + } + + @AfterClass + public static void afterClass() throws Exception { + if (reader != null) reader.close(); + if (directory != null) directory.close(); + searcher = null; + reader = null; + directory = null; + log.info("Test finished"); + } + + @Test + public void testVectorSearch() throws IOException { + Random random = random(); + int numQueries = random.nextInt(NUM_QUERIES_LIMIT) + 1; + int topK = Math.min(random.nextInt(TOP_K_LIMIT) + 1, dataset.length); + + if (dataset.length < topK) topK = dataset.length; + + float[][] queries = generateQueries(random, dataset[0].length, numQueries); + List> expected = generateExpectedResults(topK, dataset, queries); + + log.info("Dataset size: " + dataset.length + "x" + dataset[0].length); + log.info("Query size: " + numQueries + "x" + queries[0].length); + log.info("TopK: " + topK); + + Query query = new KnnFloatVectorQuery("vector", queries[0], topK); + int correct[] = new int[topK]; + for (int i = 0; i < topK; i++) correct[i] = expected.get(0).get(i); + + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + log.info("RESULTS: " + Arrays.toString(hits)); + log.info("EXPECTD: " + expected.get(0)); + + for (ScoreDoc hit : hits) { + log.info("\t" + reader.storedFields().document(hit.doc).get("id") + ": " + hit.score); + } + + for (ScoreDoc hit : hits) { + int doc = Integer.parseInt(reader.storedFields().document(hit.doc).get("id")); + assertTrue("Result returned was not in topk*2: " + doc, expected.get(0).contains(doc)); + } + } + + private static List> generateExpectedResults( + int topK, float[][] dataset, float[][] queries) { + List> neighborsResult = new ArrayList<>(); + int dimensions = dataset[0].length; + + for (float[] query : queries) { + Map distances = new TreeMap<>(); + for (int j = 0; j < dataset.length; j++) { + double distance = 0; + for (int k = 0; k < dimensions; k++) { + distance += (query[k] - dataset[j][k]) * (query[k] - dataset[j][k]); + } + distances.put(j, (distance)); + } + + Map sorted = new TreeMap(distances); + log.info("EXPECTED: " + sorted); + + // Sort by distance and select the topK nearest neighbors + List neighbors = + distances.entrySet().stream() + .sorted(Map.Entry.comparingByValue()) + .map(Map.Entry::getKey) + .toList(); + neighborsResult.add(neighbors.subList(0, Math.min(topK * 3, dataset.length))); + } + + log.info("Expected results generated successfully."); + return neighborsResult; + } + + @Test + public void testVectorSearchWithFilter() throws IOException { + assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + + Random random = random(); + int topK = Math.min(random.nextInt(TOP_K_LIMIT) + 1, dataset.length); + + if (dataset.length < topK) topK = dataset.length; + + // Find a document that has a vector by doing a search first + Query unfiltered = new KnnFloatVectorQuery("vector", dataset[0], 1); + ScoreDoc[] unfilteredHits = searcher.search(unfiltered, 1).scoreDocs; + + // Skip test if no vectors found at all + assumeTrue( + "Need at least one document with vector for filtering test", unfilteredHits.length > 0); + + String targetDocId = reader.storedFields().document(unfilteredHits[0].doc).get("id"); + float[] queryVector = dataset[0]; + + // Create a filter that matches only the document we know has a vector + Query filter = new TermQuery(new Term("id", targetDocId)); + + // Test the new constructor with filter + Query filteredQuery = new KnnFloatVectorQuery("vector", queryVector, topK, filter); + + ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; + + // Ensure we got some results + assertTrue("Should have at least one result", filteredHits.length > 0); + + // Verify that all results match the filter + for (ScoreDoc hit : filteredHits) { + String docId = reader.storedFields().document(hit.doc).get("id"); + assertEquals("All results should match the filter", targetDocId, docId); + } + + log.info("Prefiltering test passed with " + filteredHits.length + " results"); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java index 6cbd646491..5e5c2b1b73 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java @@ -40,7 +40,7 @@ import org.junit.BeforeClass; import org.junit.Test; -@SuppressSysoutChecks(bugUrl = "prints info from within cuVS") +@SuppressSysoutChecks(bugUrl = "") public class TestCuVSRandomizedVectorSearch extends LuceneTestCase { protected static Logger log = Logger.getLogger(TestCuVSRandomizedVectorSearch.class.getName()); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java new file mode 100644 index 0000000000..f834d56471 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java @@ -0,0 +1,155 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +import java.util.List; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Ignore; + +@SuppressSysoutChecks(bugUrl = "") +public class TestLucene99AcceleratedHNSWVectorsFormat extends BaseKnnVectorsFormatTestCase { + + @BeforeClass + public static void beforeClass() { + assumeTrue("cuVS is not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + } + + @Override + protected Codec getCodec() { + return TestUtil.alwaysKnnVectorsFormat(new Lucene99AcceleratedHNSWVectorsFormat()); + } + + public void testMergeTwoSegsWithASingleDocPerSeg() throws Exception { + float[][] f = new float[][] {randomVector(384), randomVector(384)}; + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc1 = new Document(); + doc1.add(new StringField("id", "0", Field.Store.NO)); + doc1.add(new KnnFloatVectorField("f", f[0], EUCLIDEAN)); + w.addDocument(doc1); + w.commit(); + Document doc2 = new Document(); + doc2.add(new StringField("id", "1", Field.Store.NO)); + doc2.add(new KnnFloatVectorField("f", f[1], EUCLIDEAN)); + w.addDocument(doc2); + w.flush(); + w.commit(); + + // sanity - verify one doc per leaf + try (DirectoryReader reader = DirectoryReader.open(w)) { + List subReaders = reader.leaves(); + assertEquals(2, subReaders.size()); + assertEquals(1, subReaders.get(0).reader().getFloatVectorValues("f").size()); + assertEquals(1, subReaders.get(1).reader().getFloatVectorValues("f").size()); + } + + // now merge to a single segment + w.forceMerge(1); + + // verify merged content + try (DirectoryReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues values = r.getFloatVectorValues("f"); + assertNotNull(values); + assertEquals(2, values.size()); + assertArrayEquals(f[0], values.vectorValue(0), 0.0f); + assertArrayEquals(f[1], values.vectorValue(1), 0.0f); + } + } + } + + // Basic test for multiple vectors fields per document + public void testTwoVectorFieldsPerDoc() throws Exception { + float[][] f1 = new float[][] {randomVector(384), randomVector(384)}; + float[][] f2 = new float[][] {randomVector(384), randomVector(384)}; + try (Directory dir = newDirectory(); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + Document doc1 = new Document(); + doc1.add(new StringField("id", "0", Field.Store.NO)); + doc1.add(new KnnFloatVectorField("f1", f1[0], EUCLIDEAN)); + doc1.add(new KnnFloatVectorField("f2", f2[0], EUCLIDEAN)); + w.addDocument(doc1); + Document doc2 = new Document(); + doc2.add(new StringField("id", "1", Field.Store.NO)); + doc2.add(new KnnFloatVectorField("f1", f1[1], EUCLIDEAN)); + doc2.add(new KnnFloatVectorField("f2", f2[1], EUCLIDEAN)); + w.addDocument(doc2); + w.forceMerge(1); + + try (DirectoryReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues values = r.getFloatVectorValues("f1"); + assertNotNull(values); + assertEquals(2, values.size()); + assertArrayEquals(f1[0], values.vectorValue(0), 0.0f); + assertArrayEquals(f1[1], values.vectorValue(1), 0.0f); + + values = r.getFloatVectorValues("f2"); + assertNotNull(values); + assertEquals(2, values.size()); + assertArrayEquals(f2[0], values.vectorValue(0), 0.0f); + assertArrayEquals(f2[1], values.vectorValue(1), 0.0f); + + // opportunistically check boundary condition - search with a 0 topK + var topDocs = r.searchNearestVectors("f1", randomVector(384), 0, null, 10); + assertEquals(0, topDocs.scoreDocs.length); + assertEquals(0, topDocs.totalHits.value()); + } + } + } + + @Override + // Overriding this method from superclass for the tests to only use float vector encoding + protected VectorEncoding randomVectorEncoding() { + return VectorEncoding.FLOAT32; + } + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testByteVectorScorerIteration() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testEmptyByteVectorData() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testMergingWithDifferentByteKnnFields() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testMismatchedFields() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testRandomBytes() {} + + @Ignore + @Override + // Ignoring this test from superclass as we do not support byte vectors + public void testSortedIndexBytes() {} +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java index 676faff37e..4d76e18360 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java @@ -48,7 +48,7 @@ * CAGRA, and combined index configurations to ensure proper vector handling * and search functionality after segment merging. */ -@SuppressSysoutChecks(bugUrl = "CuVS native library produces verbose logging output") +@SuppressSysoutChecks(bugUrl = "") public class TestMerge extends LuceneTestCase { private static final Logger log = Logger.getLogger(TestMerge.class.getName()); From 4446492d9beeeaca8ab806c870b41ad6ee8d747c Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 25 Nov 2025 17:08:23 -0500 Subject: [PATCH 043/105] Lucene version agnostic vector format implementation (NVIDIA/cuvs-lucene#37) Fixes NVIDIA/cuvs-lucene#35 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/37 --- java/cuvs-lucene/.gitignore | 2 + java/cuvs-lucene/pom.xml | 5 + .../cuvs/lucene/CuVS2510GPUVectorsFormat.java | 21 +- .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 17 +- .../Lucene99AcceleratedHNSWVectorsFormat.java | 51 +++-- .../Lucene99AcceleratedHNSWVectorsWriter.java | 24 ++- .../nvidia/cuvs/lucene/LuceneProvider.java | 201 ++++++++++++++++++ .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 20 +- .../lucene/Lucene101AcceleratedHNSWCodec.java | 44 +++- .../nvidia/cuvs/lucene/TestBackCompat.java | 41 ++++ ...TestCagraToHnswSerializationAndSearch.java | 11 +- ...ializationAndSearchWithFallbackWriter.java | 11 +- 12 files changed, 401 insertions(+), 47 deletions(-) create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java diff --git a/java/cuvs-lucene/.gitignore b/java/cuvs-lucene/.gitignore index 571838e7cc..4962a91dae 100644 --- a/java/cuvs-lucene/.gitignore +++ b/java/cuvs-lucene/.gitignore @@ -4,5 +4,7 @@ *.jar target **/.DS_Store +bin +.project cuvs-workdir bin diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index ed47e06a60..69d6f05378 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -53,6 +53,11 @@ 10.2.0 test + + org.apache.lucene + lucene-backward-codecs + 10.2.0 + org.apache.lucene lucene-test-framework diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java index 94ccd1b8c6..99639b189a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java @@ -15,7 +15,6 @@ import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; @@ -43,10 +42,8 @@ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; static CuVSResources resources = cuVSResourcesOrNull(); - - /** The format for storing, reading, and merging raw vectors on disk. */ - private static final FlatVectorsFormat flatVectorsFormat = - new Lucene99FlatVectorsFormat(DefaultFlatVectorScorer.INSTANCE); + static final LuceneProvider LUCENE_PROVIDER; + static final FlatVectorsFormat FLAT_VECTORS_FORMAT; final int maxDimensions = 4096; final int cuvsWriterThreads; @@ -54,6 +51,16 @@ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { final int graphDegree; final CuVS2510GPUVectorsWriter.IndexType indexType; // the index type to build, when writing + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + FLAT_VECTORS_FORMAT = + LUCENE_PROVIDER.getLuceneFlatVectorsFormatInstance(DefaultFlatVectorScorer.INSTANCE); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + /** * Initializes the {@link CuVS2510GPUVectorsFormat} with default parameter values. * @@ -92,7 +99,7 @@ public CuVS2510GPUVectorsFormat( @Override public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { checkSupported(); - var flatWriter = flatVectorsFormat.fieldsWriter(state); + var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); return new CuVS2510GPUVectorsWriter( state, cuvsWriterThreads, intGraphDegree, graphDegree, indexType, resources, flatWriter); } @@ -103,7 +110,7 @@ public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOE @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { checkSupported(); - return new CuVS2510GPUVectorsReader(state, resources, flatVectorsFormat.fieldsReader(state)); + return new CuVS2510GPUVectorsReader(state, resources, FLAT_VECTORS_FORMAT.fieldsReader(state)); } /** diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index bfe25ed8ac..033bbc14b6 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -9,7 +9,6 @@ import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_META_CODEC_EXT; import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_META_CODEC_NAME; import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.VERSION_CURRENT; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; import static org.apache.lucene.index.VectorEncoding.FLOAT32; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; @@ -68,6 +67,9 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { /** The name of the CUVS component for the info-stream * */ private static final String CUVS_COMPONENT = "CUVS"; + private static final LuceneProvider LUCENE_PROVIDER; + private static final List VECTOR_SIMILARITY_FUNCTIONS; + // The minimum number of vectors in the dataset required before // we attempt to build a Cagra index static final int MIN_CAGRA_INDEX_SIZE = 2; @@ -85,6 +87,15 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { private final InfoStream infoStream; private boolean finished; + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + VECTOR_SIMILARITY_FUNCTIONS = LUCENE_PROVIDER.getSimilarityFunctions(); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + /** * The cuVS index Types. */ @@ -442,8 +453,8 @@ private void writeMeta( } static int distFuncToOrd(VectorSimilarityFunction func) { - for (int i = 0; i < SIMILARITY_FUNCTIONS.size(); i++) { - if (SIMILARITY_FUNCTIONS.get(i).equals(func)) { + for (int i = 0; i < VECTOR_SIMILARITY_FUNCTIONS.size(); i++) { + if (VECTOR_SIMILARITY_FUNCTIONS.get(i).equals(func)) { return (byte) i; } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java index c3fd4ed94a..ffd3ff4bd9 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java @@ -5,9 +5,6 @@ package com.nvidia.cuvs.lucene; import static com.nvidia.cuvs.lucene.Utils.cuVSResourcesOrNull; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_NUM_MERGE_WORKER; import com.nvidia.cuvs.CuVSResources; import com.nvidia.cuvs.LibraryException; @@ -18,9 +15,6 @@ import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99FlatVectorsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsWriter; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; @@ -43,22 +37,36 @@ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { static final String HNSW_META_CODEC_EXT = "vem"; static final String HNSW_INDEX_CODEC_NAME = "Lucene99HnswVectorsFormatIndex"; static final String HNSW_INDEX_EXT = "vex"; + static final LuceneProvider LUCENE_PROVIDER; private static CuVSResources resources = cuVSResourcesOrNull(); - /** The format for storing, reading, and merging raw vectors on disk. */ - private static final FlatVectorsFormat flatVectorsFormat = - new Lucene99FlatVectorsFormat(DefaultFlatVectorScorer.INSTANCE); + private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; + private static final Integer MAX_CONN; + private static final Integer BEAM_WIDTH; + private static final Integer NUM_MERGE_WORKERS; private final int maxDimensions = 4096; private final int cuvsWriterThreads; private final int intGraphDegree; private final int graphDegree; private final int hnswLayers; - private final int maxConn; private final int beamWidth; + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + MAX_CONN = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_MAX_CONN"); + BEAM_WIDTH = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); + NUM_MERGE_WORKERS = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); + FLAT_VECTORS_FORMAT = + LUCENE_PROVIDER.getLuceneFlatVectorsFormatInstance(DefaultFlatVectorScorer.INSTANCE); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + /** * Initializes {@link Lucene99AcceleratedHNSWVectorsFormat} with default values. * @@ -70,8 +78,8 @@ public Lucene99AcceleratedHNSWVectorsFormat() { DEFAULT_INTERMEDIATE_GRAPH_DEGREE, DEFAULT_GRAPH_DEGREE, DEFAULT_HNSW_GRAPH_LAYERS, - DEFAULT_MAX_CONN, - DEFAULT_BEAM_WIDTH); + MAX_CONN, + BEAM_WIDTH); } /** @@ -105,7 +113,7 @@ public Lucene99AcceleratedHNSWVectorsFormat( */ @Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - var flatWriter = flatVectorsFormat.fieldsWriter(state); + var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); if (supported()) { log.info("cuVS is supported so using the Lucene99AcceleratedHNSWVectorsWriter"); return new Lucene99AcceleratedHNSWVectorsWriter( @@ -114,8 +122,13 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException log.warning( "GPU based indexing not supported, falling back to using the Lucene99HnswVectorsWriter"); // TODO: Make num merge workers configurable. - return new Lucene99HnswVectorsWriter( - state, maxConn, beamWidth, flatWriter, DEFAULT_NUM_MERGE_WORKER, null); + try { + return LUCENE_PROVIDER.getLuceneHnswVectorsWriterInstance( + state, maxConn, beamWidth, flatWriter, NUM_MERGE_WORKERS, null); + } catch (Exception e) { + // maybe there is a better suited option to throwing RuntimeException? Need to explore. + throw new RuntimeException(e.getMessage()); + } } } @@ -124,7 +137,13 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException */ @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { - return new Lucene99HnswVectorsReader(state, flatVectorsFormat.fieldsReader(state)); + try { + return LUCENE_PROVIDER.getLuceneHnswVectorsReaderInstance( + state, FLAT_VECTORS_FORMAT.fieldsReader(state)); + } catch (Exception e) { + // maybe there is a better suited option to throwing RuntimeException? Need to explore. + throw new RuntimeException(e.getMessage()); + } } /** diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index a0e4437197..8528ef4566 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -8,7 +8,6 @@ import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_EXT; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_EXT; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_NAME; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader.SIMILARITY_FUNCTIONS; import static org.apache.lucene.index.VectorEncoding.FLOAT32; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; @@ -37,7 +36,6 @@ import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.index.DocsWithFieldSet; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; @@ -76,6 +74,10 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { /** The name of the CUVS component for the info-stream * */ private static final String CUVS_COMPONENT = "CUVS"; + private static final LuceneProvider LUCENE_PROVIDER; + private static final Integer VERSION_CURRENT; + private static final List VECTOR_SIMILARITY_FUNCTIONS; + private final int cuvsWriterThreads; private final int intGraphDegree; private final int graphDegree; @@ -90,6 +92,16 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { private String vemFileName; private String vexFileName; + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + VERSION_CURRENT = LUCENE_PROVIDER.getStaticIntParam("VERSION_CURRENT"); + VECTOR_SIMILARITY_FUNCTIONS = LUCENE_PROVIDER.getSimilarityFunctions(); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + /** * Initializes {@link Lucene99AcceleratedHNSWVectorsWriter} * @@ -136,13 +148,13 @@ public Lucene99AcceleratedHNSWVectorsWriter( CodecUtil.writeIndexHeader( hnswMeta, HNSW_META_CODEC_NAME, - Lucene99HnswVectorsFormat.VERSION_CURRENT, + VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); CodecUtil.writeIndexHeader( hnswVectorIndex, HNSW_INDEX_CODEC_NAME, - Lucene99HnswVectorsFormat.VERSION_CURRENT, + VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); @@ -662,8 +674,8 @@ private void writeEmpty(FieldInfo fieldInfo) throws IOException { } static int distFuncToOrd(VectorSimilarityFunction func) { - for (int i = 0; i < SIMILARITY_FUNCTIONS.size(); i++) { - if (SIMILARITY_FUNCTIONS.get(i).equals(func)) { + for (int i = 0; i < VECTOR_SIMILARITY_FUNCTIONS.size(); i++) { + if (VECTOR_SIMILARITY_FUNCTIONS.get(i).equals(func)) { return (byte) i; } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java new file mode 100644 index 0000000000..00855f30e7 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java @@ -0,0 +1,201 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import java.lang.invoke.MethodHandles; +import java.lang.invoke.VarHandle; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.List; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.codecs.hnsw.FlatVectorsReader; +import org.apache.lucene.codecs.hnsw.FlatVectorsScorer; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.search.TaskExecutor; + +/** + * Dynamically loads Lucene format, reader, and writer classes with a fallback mechanism. + * + * @since 25.12 + */ +public class LuceneProvider { + + static final Logger log = Logger.getLogger(LuceneProvider.class.getName()); + + private static final String BASE = "org.apache.lucene."; + private static String codecs = "codecs.lucene."; + private static String backwardCodecs = "backward_codecs.lucene."; + + private static String luceneflatVectorsFormat = + BASE + codecs + "LuceneFlatVectorsFormat"; + private static String luceneFlatVectorsFormatFallback = + BASE + backwardCodecs + "LuceneFlatVectorsFormat"; + + private static String luceneHnswVectorsFormat = + BASE + codecs + "LuceneHnswVectorsFormat"; + private static String luceneHnswVectorsFormatFallback = + BASE + backwardCodecs + "LuceneHnswVectorsFormat"; + + private static String luceneHnswVectorsReader = + BASE + codecs + "LuceneHnswVectorsReader"; + private static String luceneHnswVectorsReaderFallback = + BASE + backwardCodecs + "LuceneHnswVectorsReader"; + + private static String luceneHnswVectorsWriter = + BASE + codecs + "LuceneHnswVectorsWriter"; + private static String luceneHnswVectorsWriterFallback = + BASE + backwardCodecs + "LuceneHnswVectorsWriter"; + + private static String luceneCodec = BASE + codecs + "LuceneCodec"; + private static String luceneCodecFallback = BASE + backwardCodecs + "LuceneCodec"; + + private static LuceneProvider instance; + + private static MethodHandles.Lookup lookup = MethodHandles.lookup(); + + private Class flatVectorsFormat; + private Class hnswVectorsFormat; + private Class hnswVectorsReader; + private Class hnswVectorsWriter; + + public static LuceneProvider getInstance(String version) throws ClassNotFoundException { + if (instance == null) { + instance = new LuceneProvider(version); + } + return instance; + } + + private LuceneProvider(String version) throws ClassNotFoundException { + flatVectorsFormat = + loadClass( + setVersion(luceneflatVectorsFormat, version), + setVersion(luceneFlatVectorsFormatFallback, version)); + hnswVectorsFormat = + loadClass( + setVersion(luceneHnswVectorsFormat, version), + setVersion(luceneHnswVectorsFormatFallback, version)); + hnswVectorsReader = + loadClass( + setVersion(luceneHnswVectorsReader, version), + setVersion(luceneHnswVectorsReaderFallback, version)); + hnswVectorsWriter = + loadClass( + setVersion(luceneHnswVectorsWriter, version), + setVersion(luceneHnswVectorsWriterFallback, version)); + } + + private static String setVersion(String pkg, String version) { + return pkg.replaceAll("", version); + } + + private static Class loadClass(String defaultClassName, String fallbackClassName) + throws ClassNotFoundException { + try { + return Class.forName(defaultClassName); + } catch (ClassNotFoundException e) { + // Load class from fallback package. + try { + return Class.forName(fallbackClassName); + } catch (ClassNotFoundException e1) { + // Should not reach here. + log.severe("Unable to load class: " + fallbackClassName); + throw e1; + } + } + } + + public static Codec getCodec(String version) + throws ClassNotFoundException, + NoSuchMethodException, + SecurityException, + InstantiationException, + IllegalAccessException, + IllegalArgumentException, + InvocationTargetException { + Class codecClass = + loadClass(setVersion(luceneCodec, version), setVersion(luceneCodecFallback, version)); + Constructor codecClassConstructor = codecClass.getConstructor(); + return (Codec) codecClassConstructor.newInstance(); + } + + public FlatVectorsFormat getLuceneFlatVectorsFormatInstance(FlatVectorsScorer scorer) + throws Exception { + try { + Constructor luceneFlatVectorsFormatConstructor = + flatVectorsFormat.getConstructor(FlatVectorsScorer.class); + return (FlatVectorsFormat) luceneFlatVectorsFormatConstructor.newInstance(scorer); + } catch (Exception e) { + log.severe("Unable to initialize LuceneFlatVectorsFormat: " + e.getMessage()); + throw e; + } + } + + public KnnVectorsReader getLuceneHnswVectorsReaderInstance( + SegmentReadState state, FlatVectorsReader reader) throws Exception { + try { + Constructor luceneHnswVectorsReaderConstructor = + hnswVectorsReader.getConstructor(SegmentReadState.class, FlatVectorsReader.class); + return (KnnVectorsReader) luceneHnswVectorsReaderConstructor.newInstance(state, reader); + } catch (Exception e) { + log.severe("Unable to initialize LuceneHnswVectorsReader: " + e.getMessage()); + throw e; + } + } + + public KnnVectorsWriter getLuceneHnswVectorsWriterInstance( + SegmentWriteState state, + int maxConn, + int beamWidth, + FlatVectorsWriter writer, + int numMergeWorkers, + TaskExecutor executor) + throws Exception { + try { + Constructor luceneHnswVectorsWriterConstructor = + hnswVectorsWriter.getConstructor( + SegmentWriteState.class, + Integer.TYPE, + Integer.TYPE, + FlatVectorsWriter.class, + Integer.TYPE, + TaskExecutor.class); + return (KnnVectorsWriter) + luceneHnswVectorsWriterConstructor.newInstance( + state, maxConn, beamWidth, writer, numMergeWorkers, executor); + } catch (Exception e) { + log.severe("Unable to initialize LuceneHnswVectorsWriter: " + e.getMessage()); + throw e; + } + } + + public int getStaticIntParam(String param) throws ReflectiveOperationException { + try { + VarHandle varHandle = lookup.findStaticVarHandle(hnswVectorsFormat, param, Integer.TYPE); + return (int) varHandle.get(); + } catch (NoSuchFieldException | IllegalAccessException e) { + log.severe("Unable to get " + param + ": " + e.getMessage()); + throw e; + } + } + + public List getSimilarityFunctions() + throws ReflectiveOperationException { + try { + VarHandle varHandle = + lookup.findStaticVarHandle(hnswVectorsReader, "SIMILARITY_FUNCTIONS", List.class); + return (List) varHandle.get(); + } catch (NoSuchFieldException | IllegalAccessException e) { + log.severe("Unable to get SIMILARITY_FUNCTIONS: " + e.getMessage()); + throw e; + } + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index 3e5085a8c8..56c193667a 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -6,11 +6,11 @@ import com.nvidia.cuvs.LibraryException; import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; +import java.lang.reflect.InvocationTargetException; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene101.Lucene101Codec; /** * cuVS based codec for GPU based vector search @@ -32,9 +32,23 @@ public class CuVS2510GPUSearchCodec extends FilterCodec { /** * Default constructor for {@link CuVS2510GPUSearchCodec} + * @throws InvocationTargetException + * @throws IllegalArgumentException + * @throws IllegalAccessException + * @throws InstantiationException + * @throws SecurityException + * @throws NoSuchMethodException + * @throws ClassNotFoundException */ - public CuVS2510GPUSearchCodec() { - this(NAME, new Lucene101Codec()); + public CuVS2510GPUSearchCodec() + throws ClassNotFoundException, + NoSuchMethodException, + SecurityException, + InstantiationException, + IllegalAccessException, + IllegalArgumentException, + InvocationTargetException { + this(NAME, LuceneProvider.getCodec("101")); } /** diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java index 3dd698dc3c..0bc9f6e964 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java @@ -4,15 +4,12 @@ */ package com.nvidia.cuvs.lucene; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; -import static org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; - import com.nvidia.cuvs.LibraryException; +import java.lang.reflect.InvocationTargetException; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.KnnVectorsFormat; -import org.apache.lucene.codecs.lucene101.Lucene101Codec; /** * CuVS based codec for GPU based vector search @@ -28,11 +25,31 @@ public class Lucene101AcceleratedHNSWCodec extends FilterCodec { private static final int DEFAULT_GRAPH_DEGREE = 64; private static final int DEFAULT_HNSW_LAYERS = 1; private static final String NAME = "Lucene101AcceleratedHNSWCodec"; + private static final LuceneProvider lucene99Provider; + private static final Integer maxConn; + private static final Integer beamWidth; private KnnVectorsFormat format; - public Lucene101AcceleratedHNSWCodec() { - this(NAME, new Lucene101Codec()); + static { + try { + lucene99Provider = LuceneProvider.getInstance("99"); + maxConn = lucene99Provider.getStaticIntParam("DEFAULT_MAX_CONN"); + beamWidth = lucene99Provider.getStaticIntParam("DEFAULT_BEAM_WIDTH"); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + + public Lucene101AcceleratedHNSWCodec() + throws ClassNotFoundException, + NoSuchMethodException, + SecurityException, + InstantiationException, + IllegalAccessException, + IllegalArgumentException, + InvocationTargetException { + this(NAME, LuceneProvider.getCodec("101")); } public Lucene101AcceleratedHNSWCodec(String name, Codec delegate) { @@ -46,8 +63,15 @@ public Lucene101AcceleratedHNSWCodec( int graphDegree, int hnswLayers, int maxConn, - int beamWidth) { - this(NAME, new Lucene101Codec()); + int beamWidth) + throws ClassNotFoundException, + NoSuchMethodException, + SecurityException, + InstantiationException, + IllegalAccessException, + IllegalArgumentException, + InvocationTargetException { + this(NAME, LuceneProvider.getCodec("101")); initializeFormat( cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); } @@ -58,8 +82,8 @@ private void initializeFormatDefaultValues() { DEFAULT_INTERMEDIATE_GRAPH_DEGREE, DEFAULT_GRAPH_DEGREE, DEFAULT_HNSW_LAYERS, - DEFAULT_MAX_CONN, - DEFAULT_BEAM_WIDTH); + maxConn, + beamWidth); } private void initializeFormat( diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java new file mode 100644 index 0000000000..2de6e660fa --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertTrue; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.junit.Test; + +/** + * Tests the backward compatibility mechanism. + * + * @since 25.12 + */ +public class TestBackCompat { + + @Test + public void testFallback() throws Exception { + // Lucene99Codec exists in the org.apache.lucene.backward_codecs.lucene99 + Codec c = LuceneProvider.getCodec("99"); + assertEquals(c.getName(), "Lucene99"); + } + + @Test(expected = ClassNotFoundException.class) + public void testNonexistentCodec() throws Exception { + LuceneProvider.getCodec("0"); + } + + @Test + public void testExistingComponents() throws Exception { + LuceneProvider provider = LuceneProvider.getInstance("99"); + assertTrue(provider.getLuceneFlatVectorsFormatInstance(null) instanceof FlatVectorsFormat); + assertEquals(provider.getStaticIntParam("VERSION_CURRENT"), 0); + assertNotEquals(provider.getSimilarityFunctions().size(), 0); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java index 2a11ff6369..7d35bbd34d 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java @@ -9,6 +9,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -57,7 +58,15 @@ public void beforeTest() throws Exception { } @Test - public void testCagraToHnswSerializationAndSearch() throws IOException { + public void testCagraToHnswSerializationAndSearch() + throws IOException, + ClassNotFoundException, + NoSuchMethodException, + SecurityException, + InstantiationException, + IllegalAccessException, + IllegalArgumentException, + InvocationTargetException { Codec codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java index 83758708ef..2058c29b4d 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java @@ -10,6 +10,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -61,7 +62,15 @@ public static void beforeClass() throws Exception { } @Test - public void testCagraToHnswSerializationAndSearchWithFallbackWriter() throws IOException { + public void testCagraToHnswSerializationAndSearchWithFallbackWriter() + throws IOException, + ClassNotFoundException, + NoSuchMethodException, + SecurityException, + InstantiationException, + IllegalAccessException, + IllegalArgumentException, + InvocationTargetException { Codec codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); From 38d4355067ccfa4083c1d41fb298f2b0e58ce445 Mon Sep 17 00:00:00 2001 From: Vivek Narang Date: Wed, 26 Nov 2025 11:41:59 -0500 Subject: [PATCH 044/105] Make RAPIDS_BRANCH have release/25.12 --- java/cuvs-lucene/RAPIDS_BRANCH | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/cuvs-lucene/RAPIDS_BRANCH b/java/cuvs-lucene/RAPIDS_BRANCH index ba2906d066..26b84372d3 100644 --- a/java/cuvs-lucene/RAPIDS_BRANCH +++ b/java/cuvs-lucene/RAPIDS_BRANCH @@ -1 +1 @@ -main +release/25.12 From ef5c64c540f8a907d0763309b84d4c10d49e573e Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 2 Dec 2025 15:16:58 -0600 Subject: [PATCH 045/105] Use strict priority in CI conda tests (NVIDIA/cuvs-lucene#60) This PR sets conda to use `strict` priority in CI tests. Mixing channel priority is frequently a cause of unexpected errors. Our CI jobs should always use strict priority in order to enforce that conda packages come from local channels with the artifacts built in CI, not mixing with older nightly artifacts from the `rapidsai-nightly` channel or other sources. xref: https://github.com/rapidsai/build-planning/issues/14 Authors: - Bradley Dice (https://github.com/bdice) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cuvs-lucene/pull/60 --- java/cuvs-lucene/ci/check_style.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/java/cuvs-lucene/ci/check_style.sh b/java/cuvs-lucene/ci/check_style.sh index b50c2da01c..9b6dc337b7 100755 --- a/java/cuvs-lucene/ci/check_style.sh +++ b/java/cuvs-lucene/ci/check_style.sh @@ -9,6 +9,9 @@ rapids-logger "Create checks conda environment" # shellcheck disable=SC1091 . /opt/conda/etc/profile.d/conda.sh +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + rapids-dependency-file-generator \ --output conda \ --file-key checks \ From 54f89cdf736e0d07c58aa41b0a856f234630fcdd Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Thu, 4 Dec 2025 13:57:51 -0500 Subject: [PATCH 046/105] Updates to the logging setup and adapt with latest cuvs-java changes (NVIDIA/cuvs-lucene#59) Code changes that: - makes updates to the logging setup. - Adapts with a few cuvs-java changes. Please note: This PR is on top of NVIDIA/cuvs-lucene#56 and once that gets merged, the changes in PR will be relatively very few. Thanks! --- java/cuvs-lucene/.gitignore | 1 - java/cuvs-lucene/pom.xml | 5 + .../cuvs/lucene/FilterCuVSProvider.java | 26 ++- .../Lucene99AcceleratedHNSWVectorsFormat.java | 6 +- .../nvidia/cuvs/lucene/LuceneProvider.java | 13 +- .../java/com/nvidia/cuvs/lucene/Utils.java | 9 +- .../cuvs-lucene/src/main/resources/log4j2.xml | 14 -- .../src/main/resources/logging.properties | 9 + .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 5 +- .../lucene/Lucene101AcceleratedHNSWCodec.java | 5 +- .../TestAcceleratedHNSWDeletedDocuments.java | 8 +- ...TestCagraToHnswSerializationAndSearch.java | 15 +- ...ializationAndSearchWithFallbackWriter.java | 15 +- ...stCuVSAcceleratedHNSWDeletedDocuments.java | 8 +- .../lucene/TestCuVSAcceleratedHNSWGaps.java | 13 +- .../cuvs/lucene/TestCuVSDeletedDocuments.java | 8 +- .../com/nvidia/cuvs/lucene/TestCuVSGaps.java | 13 +- .../TestCuVSRandomizedHNSWVectorSearch.java | 24 +-- .../TestCuVSRandomizedVectorSearch.java | 24 +-- .../com/nvidia/cuvs/lucene/TestMerge.java | 173 +++++++++++------- 20 files changed, 240 insertions(+), 154 deletions(-) delete mode 100644 java/cuvs-lucene/src/main/resources/log4j2.xml create mode 100644 java/cuvs-lucene/src/main/resources/logging.properties diff --git a/java/cuvs-lucene/.gitignore b/java/cuvs-lucene/.gitignore index 4962a91dae..f7ee48d382 100644 --- a/java/cuvs-lucene/.gitignore +++ b/java/cuvs-lucene/.gitignore @@ -7,4 +7,3 @@ target bin .project cuvs-workdir -bin diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 69d6f05378..50d6911471 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -39,6 +39,8 @@ 21 21 + UTF-8 + UTF-8 @@ -137,6 +139,9 @@ false false 0 + + ${project.basedir}/src/main/resources/logging.properties + diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index aaf722191d..b6cd3227f2 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -123,24 +123,36 @@ public com.nvidia.cuvs.TieredIndex.Builder newTieredIndexBuilder(CuVSResources c @Override public CagraIndexParams cagraIndexParamsFromHnswParams( long arg0, long arg1, int arg2, int arg3, HnswHeuristicType arg4, CuvsDistanceType arg5) { - // TODO Auto-generated method stub - return null; + return delegate.cagraIndexParamsFromHnswParams(arg0, arg1, arg2, arg3, arg4, arg5); } @Override public Level getLogLevel() { - // TODO Auto-generated method stub - return null; + return delegate.getLogLevel(); } @Override public void setLogLevel(Level arg0) { - // TODO Auto-generated method stub + delegate.setLogLevel(arg0); } @Override public HnswIndex hnswIndexFromCagra(HnswIndexParams arg0, CagraIndex arg1) throws Throwable { - // TODO Auto-generated method stub - return null; + return delegate.hnswIndexFromCagra(arg0, arg1); + } + + @Override + public void enableRMMManagedPooledMemory(int arg0, int arg1) { + delegate.enableRMMManagedPooledMemory(arg0, arg1); + } + + @Override + public void enableRMMPooledMemory(int arg0, int arg1) { + delegate.enableRMMManagedPooledMemory(arg0, arg1); + } + + @Override + public void resetRMMPooledMemory() { + delegate.resetRMMPooledMemory(); } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java index ffd3ff4bd9..5128ce7ff2 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java @@ -9,6 +9,7 @@ import com.nvidia.cuvs.CuVSResources; import com.nvidia.cuvs.LibraryException; import java.io.IOException; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; @@ -115,11 +116,12 @@ public Lucene99AcceleratedHNSWVectorsFormat( public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); if (supported()) { - log.info("cuVS is supported so using the Lucene99AcceleratedHNSWVectorsWriter"); + log.log(Level.FINE, "cuVS is supported so using the Lucene99AcceleratedHNSWVectorsWriter"); return new Lucene99AcceleratedHNSWVectorsWriter( state, cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, resources, flatWriter); } else { - log.warning( + log.log( + Level.WARNING, "GPU based indexing not supported, falling back to using the Lucene99HnswVectorsWriter"); // TODO: Make num merge workers configurable. try { diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java index 00855f30e7..f7f90abe92 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java @@ -9,6 +9,7 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.KnnVectorsReader; @@ -107,7 +108,7 @@ private static Class loadClass(String defaultClassName, String fallbackClassN return Class.forName(fallbackClassName); } catch (ClassNotFoundException e1) { // Should not reach here. - log.severe("Unable to load class: " + fallbackClassName); + log.log(Level.SEVERE, "Unable to load class: " + fallbackClassName); throw e1; } } @@ -134,7 +135,7 @@ public FlatVectorsFormat getLuceneFlatVectorsFormatInstance(FlatVectorsScorer sc flatVectorsFormat.getConstructor(FlatVectorsScorer.class); return (FlatVectorsFormat) luceneFlatVectorsFormatConstructor.newInstance(scorer); } catch (Exception e) { - log.severe("Unable to initialize LuceneFlatVectorsFormat: " + e.getMessage()); + log.log(Level.SEVERE, "Unable to initialize LuceneFlatVectorsFormat: " + e.getMessage()); throw e; } } @@ -146,7 +147,7 @@ public KnnVectorsReader getLuceneHnswVectorsReaderInstance( hnswVectorsReader.getConstructor(SegmentReadState.class, FlatVectorsReader.class); return (KnnVectorsReader) luceneHnswVectorsReaderConstructor.newInstance(state, reader); } catch (Exception e) { - log.severe("Unable to initialize LuceneHnswVectorsReader: " + e.getMessage()); + log.log(Level.SEVERE, "Unable to initialize LuceneHnswVectorsReader: " + e.getMessage()); throw e; } } @@ -172,7 +173,7 @@ public KnnVectorsWriter getLuceneHnswVectorsWriterInstance( luceneHnswVectorsWriterConstructor.newInstance( state, maxConn, beamWidth, writer, numMergeWorkers, executor); } catch (Exception e) { - log.severe("Unable to initialize LuceneHnswVectorsWriter: " + e.getMessage()); + log.log(Level.SEVERE, "Unable to initialize LuceneHnswVectorsWriter: " + e.getMessage()); throw e; } } @@ -182,7 +183,7 @@ public int getStaticIntParam(String param) throws ReflectiveOperationException { VarHandle varHandle = lookup.findStaticVarHandle(hnswVectorsFormat, param, Integer.TYPE); return (int) varHandle.get(); } catch (NoSuchFieldException | IllegalAccessException e) { - log.severe("Unable to get " + param + ": " + e.getMessage()); + log.log(Level.SEVERE, "Unable to get " + param + ": " + e.getMessage()); throw e; } } @@ -194,7 +195,7 @@ public List getSimilarityFunctions() lookup.findStaticVarHandle(hnswVectorsReader, "SIMILARITY_FUNCTIONS", List.class); return (List) varHandle.get(); } catch (NoSuchFieldException | IllegalAccessException e) { - log.severe("Unable to get SIMILARITY_FUNCTIONS: " + e.getMessage()); + log.log(Level.SEVERE, "Unable to get SIMILARITY_FUNCTIONS: " + e.getMessage()); throw e; } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java index 301d0837d7..74d2cbc558 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java @@ -9,6 +9,7 @@ import java.io.IOException; import java.time.Duration; import java.util.List; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -83,17 +84,19 @@ static CuVSResources cuVSResourcesOrNull() { try { System.loadLibrary("cudart"); } catch (UnsatisfiedLinkError e) { - log.warning("Could not load CUDA runtime library: " + e.getMessage()); + log.log(Level.WARNING, "Could not load CUDA runtime library: " + e.getMessage()); } try { return CuVSResources.create(); } catch (UnsupportedOperationException uoe) { - log.warning("cuVS is not supported on this platform or java version: " + uoe.getMessage()); + log.log( + Level.WARNING, + "cuVS is not supported on this platform or java version: " + uoe.getMessage()); } catch (Throwable t) { if (t instanceof ExceptionInInitializerError ex) { t = ex.getCause(); } - log.warning("Exception occurred during creation of cuVS resources. " + t); + log.log(Level.WARNING, "Exception occurred during creation of cuVS resources. " + t); } return null; } diff --git a/java/cuvs-lucene/src/main/resources/log4j2.xml b/java/cuvs-lucene/src/main/resources/log4j2.xml deleted file mode 100644 index a810b6b71a..0000000000 --- a/java/cuvs-lucene/src/main/resources/log4j2.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - diff --git a/java/cuvs-lucene/src/main/resources/logging.properties b/java/cuvs-lucene/src/main/resources/logging.properties new file mode 100644 index 0000000000..337c59000e --- /dev/null +++ b/java/cuvs-lucene/src/main/resources/logging.properties @@ -0,0 +1,9 @@ +# Set root level +.level=WARNING + +# Define handlers +handlers=java.util.logging.ConsoleHandler + +java.util.logging.ConsoleHandler.level=WARNING +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=[%1$tF %1$tT] %4$s %2$s %5$s%6$s%n diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index 56c193667a..57b1e3c9cf 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -7,6 +7,7 @@ import com.nvidia.cuvs.LibraryException; import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; import java.lang.reflect.InvocationTargetException; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; @@ -68,7 +69,9 @@ public CuVS2510GPUSearchCodec(String name, Codec delegate) { DEFAULT_INDEX_TYPE); setKnnFormat(format); } catch (LibraryException ex) { - log.severe("Couldn't load native library, possible classloader issue. " + ex.getMessage()); + log.log( + Level.SEVERE, + "Couldn't load native library, possible classloader issue. " + ex.getMessage()); } } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java index 0bc9f6e964..9b0db6930b 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java @@ -6,6 +6,7 @@ import com.nvidia.cuvs.LibraryException; import java.lang.reflect.InvocationTargetException; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; @@ -99,7 +100,9 @@ private void initializeFormat( cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); setKnnFormat(format); } catch (LibraryException ex) { - log.severe("Couldn't load native library, possible classloader issue. " + ex.getMessage()); + log.log( + Level.SEVERE, + "Couldn't load native library, possible classloader issue. " + ex.getMessage()); } } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java index d46794df1c..c1814faeed 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; @@ -106,7 +107,7 @@ public void testVectorSearchWithDeletedDocuments() throws IOException { int id = Integer.parseInt(docId); assertFalse( "Deleted document " + id + " should not appear in results", deletedDocs.contains(id)); - log.info("Found non-deleted document: " + id + ", Score: " + hit.score); + log.log(Level.FINE, "Found non-deleted document: " + id + ", Score: " + hit.score); } // Verify deleted documents are truly deleted @@ -179,7 +180,7 @@ public void testVectorSearchWithMixedDeletedAndMissingVectors() throws IOExcepti int id = Integer.parseInt(docId); assertFalse("Deleted document should not appear", deletedDocs.contains(id)); assertFalse("Document without vector should not appear", docsWithoutVectors.contains(id)); - log.info("Found document with vector: " + id + ", Score: " + hit.score); + log.log(Level.FINE, "Found document with vector: " + id + ", Score: " + hit.score); } // Test filtered search with deletions @@ -305,7 +306,8 @@ public void testVectorSearchWithPartialDeletionAndReindexing() throws IOExceptio assertTrue("Result should be from active documents", activeDocIds.contains(id)); } - log.info( + log.log( + Level.FINE, "Search returned " + hits.length + " results from " diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java index 7d35bbd34d..335163409b 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java @@ -16,6 +16,7 @@ import java.util.HashSet; import java.util.Random; import java.util.UUID; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.io.FileUtils; import org.apache.lucene.codecs.Codec; @@ -99,14 +100,15 @@ public void testCagraToHnswSerializationAndSearch() // Searching try (Directory indexDirectory = FSDirectory.open(indexDirPath); DirectoryReader reader = DirectoryReader.open(indexDirectory)) { - log.info("Successfully opened index"); + log.log(Level.FINE, "Successfully opened index"); int vectorCount = 0; for (LeafReaderContext leafReaderContext : reader.leaves()) { LeafReader leafReader = leafReaderContext.reader(); FloatVectorValues knnValues = leafReader.getFloatVectorValues(VECTOR_FIELD); assertNotNull(knnValues); - log.info( + log.log( + Level.FINE, VECTOR_FIELD + " field: " + knnValues.size() @@ -118,16 +120,16 @@ public void testCagraToHnswSerializationAndSearch() } assertTrue("Dataset size mismatch", vectorCount == numDocs); - log.info("Testing vector search queries..."); + log.log(Level.FINE, "Testing vector search queries..."); IndexSearcher searcher = new IndexSearcher(reader); float[] queryVector = generateDataset(random, 1, dimension)[0]; - log.info("Query vector: " + Arrays.toString(queryVector)); + log.log(Level.FINE, "Query vector: " + Arrays.toString(queryVector)); KnnFloatVectorQuery query = new KnnFloatVectorQuery(VECTOR_FIELD, queryVector, topK); TopDocs results = searcher.search(query, topK); - log.info("Search results (" + results.totalHits + " total hits):"); + log.log(Level.FINE, "Search results (" + results.totalHits + " total hits):"); Integer[] expected = new Integer[] {1869, 1803, 1302, 59, 1497, 108, 1411, 351, 1982}; HashSet expectedIds = new HashSet(Arrays.asList(expected)); @@ -135,7 +137,8 @@ public void testCagraToHnswSerializationAndSearch() ScoreDoc scoreDoc = results.scoreDocs[i]; Document doc = searcher.storedFields().document(scoreDoc.doc); String id = doc.get(ID_FIELD); - log.info( + log.log( + Level.FINE, " Rank " + (i + 1) + ": doc " diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java index 2058c29b4d..746437d698 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java @@ -17,6 +17,7 @@ import java.util.HashSet; import java.util.Random; import java.util.UUID; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.commons.io.FileUtils; import org.apache.lucene.codecs.Codec; @@ -103,14 +104,15 @@ public void testCagraToHnswSerializationAndSearchWithFallbackWriter() // Searching try (Directory indexDirectory = FSDirectory.open(indexDirPath)) { try (DirectoryReader reader = DirectoryReader.open(indexDirectory)) { - log.info("Successfully opened index"); + log.log(Level.FINE, "Successfully opened index"); int vectorCount = 0; for (LeafReaderContext leafReaderContext : reader.leaves()) { LeafReader leafReader = leafReaderContext.reader(); FloatVectorValues knnValues = leafReader.getFloatVectorValues(VECTOR_FIELD); assertNotNull(knnValues); - log.info( + log.log( + Level.FINE, VECTOR_FIELD + " field: " + knnValues.size() @@ -122,16 +124,16 @@ public void testCagraToHnswSerializationAndSearchWithFallbackWriter() } assertTrue("Dataset size mismatch", vectorCount == numDocs); - log.info("Testing vector search queries..."); + log.log(Level.FINE, "Testing vector search queries..."); IndexSearcher searcher = new IndexSearcher(reader); float[] queryVector = generateDataset(random, 1, dimension)[0]; - log.info("Query vector: " + Arrays.toString(queryVector)); + log.log(Level.FINE, "Query vector: " + Arrays.toString(queryVector)); KnnFloatVectorQuery query = new KnnFloatVectorQuery(VECTOR_FIELD, queryVector, topK); TopDocs results = searcher.search(query, topK); - log.info("Search results (" + results.totalHits + " total hits):"); + log.log(Level.FINE, "Search results (" + results.totalHits + " total hits):"); Integer[] expected = new Integer[] {1869, 1411, 1497, 351, 554}; HashSet expectedIds = new HashSet(Arrays.asList(expected)); @@ -139,7 +141,8 @@ public void testCagraToHnswSerializationAndSearchWithFallbackWriter() ScoreDoc scoreDoc = results.scoreDocs[i]; Document doc = searcher.storedFields().document(scoreDoc.doc); String id = doc.get(ID_FIELD); - log.info( + log.log( + Level.FINE, " Rank " + (i + 1) + ": doc " diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java index 49372b6b00..ba53c8b000 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; @@ -106,7 +107,7 @@ public void testVectorSearchWithDeletedDocuments() throws IOException { int id = Integer.parseInt(docId); assertFalse( "Deleted document " + id + " should not appear in results", deletedDocs.contains(id)); - log.info("Found non-deleted document: " + id + ", Score: " + hit.score); + log.log(Level.FINE, "Found non-deleted document: " + id + ", Score: " + hit.score); } // Verify deleted documents are truly deleted @@ -179,7 +180,7 @@ public void testVectorSearchWithMixedDeletedAndMissingVectors() throws IOExcepti int id = Integer.parseInt(docId); assertFalse("Deleted document should not appear", deletedDocs.contains(id)); assertFalse("Document without vector should not appear", docsWithoutVectors.contains(id)); - log.info("Found document with vector: " + id + ", Score: " + hit.score); + log.log(Level.FINE, "Found document with vector: " + id + ", Score: " + hit.score); } // Test filtered search with deletions @@ -305,7 +306,8 @@ public void testVectorSearchWithPartialDeletionAndReindexing() throws IOExceptio assertTrue("Result should be from active documents", activeDocIds.contains(id)); } - log.info( + log.log( + Level.FINE, "Search returned " + hits.length + " results from " diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java index 58e9acc5c6..6cac680426 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Random; import java.util.TreeMap; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; @@ -73,7 +74,7 @@ public static void beforeClass() throws Exception { .setCodec(codec) .setMergePolicy(newTieredMergePolicy())); - log.info("Merge Policy: " + writer.w.getConfig().getMergePolicy()); + log.log(Level.FINE, "Merge Policy: " + writer.w.getConfig().getMergePolicy()); datasetSize = random.nextInt(100, DATASET_SIZE_LIMIT); dimension = random.nextInt(8, DIMENSIONS_LIMIT); @@ -105,7 +106,7 @@ public static void afterClass() throws Exception { searcher = null; reader = null; directory = null; - log.info("Test finished"); + log.log(Level.FINE, "Test finished"); } @Test @@ -127,7 +128,7 @@ public void testVectorSearchWithAlternatingDocuments() throws IOException { String docId = reader.storedFields().document(hit.doc).get("id"); int id = Integer.parseInt(docId); assertEquals("All results should be even-numbered (have vectors)", 0, id % 2); - log.info("Document ID: " + id + ", Score: " + hit.score); + log.log(Level.FINE, "Document ID: " + id + ", Score: " + hit.score); } // Verify the results match expected top-k based on Euclidean distance @@ -138,7 +139,7 @@ public void testVectorSearchWithAlternatingDocuments() throws IOException { assertTrue("Result " + id + " should be in expected top-k results", expectedIds.contains(id)); } - log.info("Alternating document test passed with " + hits.length + " results"); + log.log(Level.FINE, "Alternating document test passed with " + hits.length + " results"); } @Test @@ -162,7 +163,9 @@ public void testVectorSearchWithFilterAndAlternatingDocuments() throws IOExcepti String docId = reader.storedFields().document(filteredHits[0].doc).get("id"); assertEquals("Should only return document 8", "8", docId); - log.info("Filtered alternating document test passed with " + filteredHits.length + " results"); + log.log( + Level.FINE, + "Filtered alternating document test passed with " + filteredHits.length + " results"); } public static List calculateExpectedTopK(float[] query, int topK, float[][] dataset) { diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java index 179399bba0..04c807a054 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java @@ -13,6 +13,7 @@ import java.util.List; import java.util.Random; import java.util.Set; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; @@ -104,7 +105,7 @@ public void testVectorSearchWithDeletedDocuments() throws IOException { int id = Integer.parseInt(docId); assertFalse( "Deleted document " + id + " should not appear in results", deletedDocs.contains(id)); - log.info("Found non-deleted document: " + id + ", Score: " + hit.score); + log.log(Level.FINE, "Found non-deleted document: " + id + ", Score: " + hit.score); } // Verify deleted documents are truly deleted @@ -177,7 +178,7 @@ public void testVectorSearchWithMixedDeletedAndMissingVectors() throws IOExcepti int id = Integer.parseInt(docId); assertFalse("Deleted document should not appear", deletedDocs.contains(id)); assertFalse("Document without vector should not appear", docsWithoutVectors.contains(id)); - log.info("Found document with vector: " + id + ", Score: " + hit.score); + log.log(Level.FINE, "Found document with vector: " + id + ", Score: " + hit.score); } // Test filtered search with deletions @@ -304,7 +305,8 @@ public void testVectorSearchWithPartialDeletionAndReindexing() throws IOExceptio assertTrue("Result should be from active documents", activeDocIds.contains(id)); } - log.info( + log.log( + Level.FINE, "Search returned " + hits.length + " results from " diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java index 39938d31d1..a48b8f6c8e 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.Random; import java.util.TreeMap; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; @@ -72,7 +73,7 @@ public static void beforeClass() throws Exception { .setCodec(codec) .setMergePolicy(newTieredMergePolicy())); - log.info("Merge Policy: " + writer.w.getConfig().getMergePolicy()); + log.log(Level.FINE, "Merge Policy: " + writer.w.getConfig().getMergePolicy()); datasetSize = random.nextInt(100, DATASET_SIZE_LIMIT); dimension = random.nextInt(8, DIMENSIONS_LIMIT); @@ -104,7 +105,7 @@ public static void afterClass() throws Exception { searcher = null; reader = null; directory = null; - log.info("Test finished"); + log.log(Level.FINE, "Test finished"); } @Test @@ -126,7 +127,7 @@ public void testVectorSearchWithAlternatingDocuments() throws IOException { String docId = reader.storedFields().document(hit.doc).get("id"); int id = Integer.parseInt(docId); assertEquals("All results should be even-numbered (have vectors)", 0, id % 2); - log.info("Document ID: " + id + ", Score: " + hit.score); + log.log(Level.FINE, "Document ID: " + id + ", Score: " + hit.score); } // Verify the results match expected top-k based on Euclidean distance @@ -137,7 +138,7 @@ public void testVectorSearchWithAlternatingDocuments() throws IOException { assertTrue("Result " + id + " should be in expected top-k results", expectedIds.contains(id)); } - log.info("Alternating document test passed with " + hits.length + " results"); + log.log(Level.FINE, "Alternating document test passed with " + hits.length + " results"); } @Test @@ -161,7 +162,9 @@ public void testVectorSearchWithFilterAndAlternatingDocuments() throws IOExcepti String docId = reader.storedFields().document(filteredHits[0].doc).get("id"); assertEquals("Should only return document 8", "8", docId); - log.info("Filtered alternating document test passed with " + filteredHits.length + " results"); + log.log( + Level.FINE, + "Filtered alternating document test passed with " + filteredHits.length + " results"); } public static List calculateExpectedTopK(float[] query, int topK, float[][] dataset) { diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java index c8c0485c9e..385ca8f581 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java @@ -14,6 +14,7 @@ import java.util.Map; import java.util.Random; import java.util.TreeMap; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; @@ -72,7 +73,7 @@ public static void beforeClass() throws Exception { .setCodec(codec) .setMergePolicy(newTieredMergePolicy())); - log.info("Merge Policy: " + writer.w.getConfig().getMergePolicy()); + log.log(Level.FINE, "Merge Policy: " + writer.w.getConfig().getMergePolicy()); Random random = random(); int datasetSize = random.nextInt(DATASET_SIZE_LIMIT) + 1; @@ -106,7 +107,7 @@ public static void afterClass() throws Exception { searcher = null; reader = null; directory = null; - log.info("Test finished"); + log.log(Level.FINE, "Test finished"); } @Test @@ -120,20 +121,21 @@ public void testVectorSearch() throws IOException { float[][] queries = generateQueries(random, dataset[0].length, numQueries); List> expected = generateExpectedResults(topK, dataset, queries); - log.info("Dataset size: " + dataset.length + "x" + dataset[0].length); - log.info("Query size: " + numQueries + "x" + queries[0].length); - log.info("TopK: " + topK); + log.log(Level.FINE, "Dataset size: " + dataset.length + "x" + dataset[0].length); + log.log(Level.FINE, "Query size: " + numQueries + "x" + queries[0].length); + log.log(Level.FINE, "TopK: " + topK); Query query = new KnnFloatVectorQuery("vector", queries[0], topK); int correct[] = new int[topK]; for (int i = 0; i < topK; i++) correct[i] = expected.get(0).get(i); ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; - log.info("RESULTS: " + Arrays.toString(hits)); - log.info("EXPECTD: " + expected.get(0)); + log.log(Level.FINE, "RESULTS: " + Arrays.toString(hits)); + log.log(Level.FINE, "EXPECTD: " + expected.get(0)); for (ScoreDoc hit : hits) { - log.info("\t" + reader.storedFields().document(hit.doc).get("id") + ": " + hit.score); + log.log( + Level.FINE, "\t" + reader.storedFields().document(hit.doc).get("id") + ": " + hit.score); } for (ScoreDoc hit : hits) { @@ -158,7 +160,7 @@ private static List> generateExpectedResults( } Map sorted = new TreeMap(distances); - log.info("EXPECTED: " + sorted); + log.log(Level.FINE, "EXPECTED: " + sorted); // Sort by distance and select the topK nearest neighbors List neighbors = @@ -169,7 +171,7 @@ private static List> generateExpectedResults( neighborsResult.add(neighbors.subList(0, Math.min(topK * 3, dataset.length))); } - log.info("Expected results generated successfully."); + log.log(Level.FINE, "Expected results generated successfully."); return neighborsResult; } @@ -210,6 +212,6 @@ public void testVectorSearchWithFilter() throws IOException { assertEquals("All results should match the filter", targetDocId, docId); } - log.info("Prefiltering test passed with " + filteredHits.length + " results"); + log.log(Level.FINE, "Prefiltering test passed with " + filteredHits.length + " results"); } } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java index 5e5c2b1b73..abcd6b67b3 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java @@ -14,6 +14,7 @@ import java.util.Map; import java.util.Random; import java.util.TreeMap; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; @@ -70,7 +71,7 @@ public static void beforeClass() throws Exception { .setCodec(codec) .setMergePolicy(newTieredMergePolicy())); - log.info("Merge Policy: " + writer.w.getConfig().getMergePolicy()); + log.log(Level.FINE, "Merge Policy: " + writer.w.getConfig().getMergePolicy()); Random random = random(); int datasetSize = random.nextInt(DATASET_SIZE_LIMIT) + 1; @@ -104,7 +105,7 @@ public static void afterClass() throws Exception { searcher = null; reader = null; directory = null; - log.info("Test finished"); + log.log(Level.FINE, "Test finished"); } @Test @@ -118,20 +119,21 @@ public void testVectorSearch() throws IOException { float[][] queries = generateQueries(random, dataset[0].length, numQueries); List> expected = generateExpectedResults(topK, dataset, queries); - log.info("Dataset size: " + dataset.length + "x" + dataset[0].length); - log.info("Query size: " + numQueries + "x" + queries[0].length); - log.info("TopK: " + topK); + log.log(Level.FINE, "Dataset size: " + dataset.length + "x" + dataset[0].length); + log.log(Level.FINE, "Query size: " + numQueries + "x" + queries[0].length); + log.log(Level.FINE, "TopK: " + topK); Query query = new KnnFloatVectorQuery("vector", queries[0], topK); int correct[] = new int[topK]; for (int i = 0; i < topK; i++) correct[i] = expected.get(0).get(i); ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; - log.info("RESULTS: " + Arrays.toString(hits)); - log.info("EXPECTD: " + expected.get(0)); + log.log(Level.FINE, "RESULTS: " + Arrays.toString(hits)); + log.log(Level.FINE, "EXPECTD: " + expected.get(0)); for (ScoreDoc hit : hits) { - log.info("\t" + reader.storedFields().document(hit.doc).get("id") + ": " + hit.score); + log.log( + Level.FINE, "\t" + reader.storedFields().document(hit.doc).get("id") + ": " + hit.score); } for (ScoreDoc hit : hits) { @@ -156,7 +158,7 @@ private static List> generateExpectedResults( } Map sorted = new TreeMap(distances); - log.info("EXPECTED: " + sorted); + log.log(Level.FINE, "EXPECTED: " + sorted); // Sort by distance and select the topK nearest neighbors List neighbors = @@ -167,7 +169,7 @@ private static List> generateExpectedResults( neighborsResult.add(neighbors.subList(0, Math.min(topK * 3, dataset.length))); } - log.info("Expected results generated successfully."); + log.log(Level.FINE, "Expected results generated successfully."); return neighborsResult; } @@ -208,6 +210,6 @@ public void testVectorSearchWithFilter() throws IOException { assertEquals("All results should match the filter", targetDocId, docId); } - log.info("Prefiltering test passed with " + filteredHits.length + " results"); + log.log(Level.FINE, "Prefiltering test passed with " + filteredHits.length + " results"); } } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java index 4d76e18360..9baea0e629 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java @@ -11,6 +11,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; +import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -76,7 +77,7 @@ public void setUp() throws Exception { // Ensure dimension is multiple of 4 for better performance vectorDimension = (vectorDimension / 4) * 4; - log.info("Using randomized vector dimension: " + vectorDimension); + log.log(Level.FINE, "Using randomized vector dimension: " + vectorDimension); } @After @@ -92,7 +93,7 @@ public void tearDown() throws Exception { **/ @Test public void testMergeManyDocumentsMultipleSegments() throws IOException { - log.info("Starting testMergeManyDocumentsMultipleSegments"); + log.log(Level.FINE, "Starting testMergeManyDocumentsMultipleSegments"); // Randomize configuration parameters int maxBufferedDocs = 5 + random().nextInt(16); // 5-20 docs per buffer @@ -103,7 +104,8 @@ public void testMergeManyDocumentsMultipleSegments() throws IOException { // Randomize vector presence probability (60-85%) double vectorProbability = 0.6 + (random().nextDouble() * 0.25); - log.info( + log.log( + Level.FINE, "Randomized parameters: maxBufferedDocs=" + maxBufferedDocs + ", totalBatches=" @@ -149,13 +151,14 @@ public void testMergeManyDocumentsMultipleSegments() throws IOException { } int documentsWithoutVectors = totalDocuments - documentsWithVectors; - log.info("Created " + totalDocuments + " documents in " + totalBatches + " segments"); - log.info("Documents with vectors: " + documentsWithVectors); - log.info("Documents without vectors: " + documentsWithoutVectors); + log.log( + Level.FINE, "Created " + totalDocuments + " documents in " + totalBatches + " segments"); + log.log(Level.FINE, "Documents with vectors: " + documentsWithVectors); + log.log(Level.FINE, "Documents without vectors: " + documentsWithoutVectors); // Force merge to trigger merge logic writer.forceMerge(1); - log.info("Forced merge to single segment completed"); + log.log(Level.FINE, "Forced merge to single segment completed"); } // Verify the merged index @@ -182,7 +185,8 @@ public void testMergeManyDocumentsMultipleSegments() throws IOException { "Should find reasonable number of results", results.scoreDocs.length <= documentsWithVectors); - log.info( + log.log( + Level.FINE, "Vector search returned " + results.scoreDocs.length + " results out of " @@ -195,10 +199,10 @@ public void testMergeManyDocumentsMultipleSegments() throws IOException { assertTrue("Document ID should be valid", docId >= 0 && docId < totalDocuments); } } else { - log.info("No documents with vectors - skipping vector search verification"); + log.log(Level.FINE, "No documents with vectors - skipping vector search verification"); } - log.info("Merge verification completed successfully"); + log.log(Level.FINE, "Merge verification completed successfully"); } } @@ -207,7 +211,7 @@ public void testMergeManyDocumentsMultipleSegments() throws IOException { **/ @Test public void testMergeWithIndexSorting() throws IOException { - log.info("Starting testMergeWithIndexSorting with text-based sorting"); + log.log(Level.FINE, "Starting testMergeWithIndexSorting with text-based sorting"); // Randomize sort field type SortField.Type sortType = random().nextBoolean() ? SortField.Type.STRING : SortField.Type.LONG; @@ -227,9 +231,11 @@ public void testMergeWithIndexSorting() throws IOException { int segmentSize = 15 + random().nextInt(11); // 15-25 docs per segment double vectorProbability = 0.65 + (random().nextDouble() * 0.25); // 65-90% have vectors - log.info( + log.log( + Level.FINE, "Randomized sorting parameters: sortType=" + sortType + ", sortFieldName=" + sortFieldName); - log.info( + log.log( + Level.FINE, "Randomized config: maxBufferedDocs=" + maxBufferedDocs + ", totalDocuments=" @@ -288,7 +294,8 @@ public void testMergeWithIndexSorting() throws IOException { // Commit based on randomized segment size if ((i + 1) % segmentSize == 0) { writer.commit(); - log.info( + log.log( + Level.FINE, "Committed segment " + ((i + 1) / segmentSize) + " with " @@ -297,11 +304,11 @@ public void testMergeWithIndexSorting() throws IOException { } } - log.info("Created " + totalDocuments + " documents with text-based index sorting"); + log.log(Level.FINE, "Created " + totalDocuments + " documents with text-based index sorting"); // Force merge with sorting - this will use the sorting merge policy writer.forceMerge(1); - log.info("Forced merge with text-based sorting completed"); + log.log(Level.FINE, "Forced merge with text-based sorting completed"); } // Verify the merged and sorted index @@ -312,7 +319,8 @@ public void testMergeWithIndexSorting() throws IOException { assertEquals("Total documents should match", totalDocuments, leafReader.maxDoc()); // Verify documents are sorted correctly by the randomized sort field - log.info( + log.log( + Level.FINE, "Verifying document sorting order using sortType: " + sortType + ", field: " @@ -344,7 +352,8 @@ public void testMergeWithIndexSorting() throws IOException { if (docId < 10) { IndexSearcher searcher = new IndexSearcher(reader); String originalOrder = searcher.storedFields().document(docId).get("original_order"); - log.info( + log.log( + Level.FINE, "DocId: " + docId + ", OriginalOrder: " @@ -379,7 +388,8 @@ public void testMergeWithIndexSorting() throws IOException { if (docId < 10) { IndexSearcher searcher = new IndexSearcher(reader); String originalOrder = searcher.storedFields().document(docId).get("original_order"); - log.info( + log.log( + Level.FINE, "DocId: " + docId + ", OriginalOrder: " @@ -394,7 +404,9 @@ public void testMergeWithIndexSorting() throws IOException { var vectorValues = leafReader.getFloatVectorValues("vector"); int documentsWithVectors = vectorValues != null ? vectorValues.size() : 0; - log.info("Found " + documentsWithVectors + " documents with vectors after sorted merge"); + log.log( + Level.FINE, + "Found " + documentsWithVectors + " documents with vectors after sorted merge"); // Test vector search on sorted index if (documentsWithVectors > 0) { @@ -406,17 +418,20 @@ public void testMergeWithIndexSorting() throws IOException { TopDocs results = searcher.search(query, 10); assertTrue("Should find results in sorted index", results.scoreDocs.length > 0); - log.info("Vector search on sorted index returned " + results.scoreDocs.length + " results"); + log.log( + Level.FINE, + "Vector search on sorted index returned " + results.scoreDocs.length + " results"); // Verify that returned documents maintain sort order if we check their sort keys - log.info("Verifying vector search results maintain sorting consistency..."); + log.log(Level.FINE, "Verifying vector search results maintain sorting consistency..."); for (int i = 0; i < Math.min(3, results.scoreDocs.length); i++) { ScoreDoc scoreDoc = results.scoreDocs[i]; String originalOrder = searcher.storedFields().document(scoreDoc.doc).get("original_order"); String sortKey = searcher.storedFields().document(scoreDoc.doc).get(sortFieldName + "_stored"); - log.info( + log.log( + Level.FINE, "Result " + i + ": DocId=" @@ -430,7 +445,7 @@ public void testMergeWithIndexSorting() throws IOException { } } - log.info("Text-based index sorting verification completed successfully"); + log.log(Level.FINE, "Text-based index sorting verification completed successfully"); } } @@ -439,7 +454,7 @@ public void testMergeWithIndexSorting() throws IOException { **/ @Test public void testMergeWithMissingVectors() throws IOException { - log.info("Starting testMergeWithMissingVectors"); + log.log(Level.FINE, "Starting testMergeWithMissingVectors"); // Randomize configuration int maxBufferedDocs = 10 + random().nextInt(11); // 10-20 docs per buffer @@ -451,7 +466,8 @@ public void testMergeWithMissingVectors() throws IOException { .setMaxBufferedDocs(maxBufferedDocs) .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); - log.info( + log.log( + Level.FINE, "Randomized parameters: maxBufferedDocs=" + maxBufferedDocs + ", numSegments=" @@ -490,7 +506,8 @@ public void testMergeWithMissingVectors() throws IOException { totalDocuments += docsInSegment; totalExpectedVectors += segmentVectorCount; - log.info( + log.log( + Level.FINE, "Created segment " + seg + ": " @@ -504,7 +521,7 @@ public void testMergeWithMissingVectors() throws IOException { // Force merge all segments writer.forceMerge(1); - log.info("Forced merge of " + numSegments + " segments completed"); + log.log(Level.FINE, "Forced merge of " + numSegments + " segments completed"); } // Verify the merged index handles missing vectors correctly @@ -518,7 +535,8 @@ public void testMergeWithMissingVectors() throws IOException { var vectorValues = leafReader.getFloatVectorValues("vector"); int actualVectorCount = vectorValues != null ? vectorValues.size() : 0; - log.info( + log.log( + Level.FINE, "Total documents: " + totalDocuments + ", Expected vectors: " @@ -544,17 +562,18 @@ public void testMergeWithMissingVectors() throws IOException { "Should not find more vectors than exist", vectorResults.scoreDocs.length <= actualVectorCount); - log.info( + log.log( + Level.FINE, "Found " + vectorResults.scoreDocs.length + " vector results out of " + actualVectorCount + " available"); } else { - log.info("No vectors in merged index - skipping vector search"); + log.log(Level.FINE, "No vectors in merged index - skipping vector search"); } - log.info("Missing vectors test completed successfully"); + log.log(Level.FINE, "Missing vectors test completed successfully"); } } @@ -563,7 +582,7 @@ public void testMergeWithMissingVectors() throws IOException { **/ @Test public void testMergeWithDeletions() throws IOException { - log.info("Starting testMergeWithDeletions"); + log.log(Level.FINE, "Starting testMergeWithDeletions"); // Randomize configuration parameters int maxBufferedDocs = 15 + random().nextInt(11); // 15-25 docs per buffer @@ -572,7 +591,8 @@ public void testMergeWithDeletions() throws IOException { double vectorProbability = 0.7 + (random().nextDouble() * 0.25); // 70-95% have vectors double deletionProbability = 0.2 + (random().nextDouble() * 0.3); // 20-50% deletion rate - log.info( + log.log( + Level.FINE, "Randomized parameters: maxBufferedDocs=" + maxBufferedDocs + ", numSegments=" @@ -616,7 +636,8 @@ public void testMergeWithDeletions() throws IOException { writer.commit(); } - log.info( + log.log( + Level.FINE, "Created " + numSegments + " segments with " @@ -637,7 +658,8 @@ public void testMergeWithDeletions() throws IOException { } } - log.info( + log.log( + Level.FINE, "Deleted " + deletedCount + " documents (" @@ -649,7 +671,7 @@ public void testMergeWithDeletions() throws IOException { // Force merge to apply deletions writer.forceMerge(1); - log.info("Forced merge with deletions completed"); + log.log(Level.FINE, "Forced merge with deletions completed"); } // Verify the merged index correctly handles deletions @@ -690,8 +712,10 @@ public void testMergeWithDeletions() throws IOException { assertTrue( "Should find some vector results after deletions", vectorResults.scoreDocs.length > 0); - log.info("Found " + vectorResults.scoreDocs.length + " vector results after deletions"); - log.info("Deletion merge verification completed successfully"); + log.log( + Level.FINE, + "Found " + vectorResults.scoreDocs.length + " vector results after deletions"); + log.log(Level.FINE, "Deletion merge verification completed successfully"); } } @@ -700,7 +724,7 @@ public void testMergeWithDeletions() throws IOException { * */ @Test public void testMergeBruteForceIndex() throws IOException { - log.info("Starting testMergeBruteForceIndex"); + log.log(Level.FINE, "Starting testMergeBruteForceIndex"); // Randomize configuration parameters int maxBufferedDocs = 8 + random().nextInt(8); // 8-15 docs per buffer @@ -708,7 +732,8 @@ public void testMergeBruteForceIndex() throws IOException { int docsPerSegment = 12 + random().nextInt(9); // 12-20 docs per segment double vectorProbability = 0.8 + (random().nextDouble() * 0.2); // 80-100% have vectors - log.info( + log.log( + Level.FINE, "Randomized parameters: maxBufferedDocs=" + maxBufferedDocs + ", numSegments=" @@ -761,7 +786,8 @@ public void testMergeBruteForceIndex() throws IOException { writer.commit(); totalExpectedVectors += segmentVectorCount; - log.info( + log.log( + Level.FINE, "Created brute force segment " + seg + ": " @@ -771,7 +797,8 @@ public void testMergeBruteForceIndex() throws IOException { + " with vectors"); } - log.info( + log.log( + Level.FINE, "Created " + numSegments + " brute force segments with " @@ -782,7 +809,7 @@ public void testMergeBruteForceIndex() throws IOException { // Force merge all brute force segments writer.forceMerge(1); - log.info("Forced merge of brute force segments completed"); + log.log(Level.FINE, "Forced merge of brute force segments completed"); } // Verify the merged brute force index @@ -796,7 +823,8 @@ public void testMergeBruteForceIndex() throws IOException { var vectorValues = leafReader.getFloatVectorValues("vector"); int actualVectorCount = vectorValues != null ? vectorValues.size() : 0; - log.info( + log.log( + Level.FINE, "Brute force merge results: Total documents: " + totalDocuments + ", Expected vectors: " @@ -824,7 +852,8 @@ public void testMergeBruteForceIndex() throws IOException { "Should not find more vectors than exist", vectorResults.scoreDocs.length <= actualVectorCount); - log.info( + log.log( + Level.FINE, "Brute force search found " + vectorResults.scoreDocs.length + " results out of " @@ -838,10 +867,10 @@ public void testMergeBruteForceIndex() throws IOException { assertTrue("Score should be positive", scoreDoc.score > 0); } } else { - log.info("No vectors in brute force merged index - skipping vector search"); + log.log(Level.FINE, "No vectors in brute force merged index - skipping vector search"); } - log.info("Brute force merge verification completed successfully"); + log.log(Level.FINE, "Brute force merge verification completed successfully"); } } @@ -850,7 +879,7 @@ public void testMergeBruteForceIndex() throws IOException { * */ @Test public void testMergeCagraAndBruteForceIndex() throws IOException { - log.info("Starting testMergeCagraAndBruteForceIndex"); + log.log(Level.FINE, "Starting testMergeCagraAndBruteForceIndex"); // Use moderate dataset size int maxBufferedDocs = 15 + random().nextInt(10); // 15-24 docs per buffer @@ -859,7 +888,8 @@ public void testMergeCagraAndBruteForceIndex() throws IOException { int docsPerSegment = 20 + random().nextInt(11); // 20-30 docs per segment double vectorProbability = 0.9 + (random().nextDouble() * 0.1); // 90-100% have vectors - log.info( + log.log( + Level.FINE, "Randomized parameters: maxBufferedDocs=" + maxBufferedDocs + ", numSegments=" @@ -913,7 +943,8 @@ public void testMergeCagraAndBruteForceIndex() throws IOException { writer.commit(); totalExpectedVectors += segmentVectorCount; - log.info( + log.log( + Level.FINE, "Created CAGRA+brute force segment " + seg + ": " @@ -923,7 +954,8 @@ public void testMergeCagraAndBruteForceIndex() throws IOException { + " with vectors"); } - log.info( + log.log( + Level.FINE, "Created " + numSegments + " CAGRA+brute force segments with " @@ -934,7 +966,7 @@ public void testMergeCagraAndBruteForceIndex() throws IOException { // Force merge all CAGRA+brute force segments writer.forceMerge(1); - log.info("Forced merge of CAGRA+brute force segments completed"); + log.log(Level.FINE, "Forced merge of CAGRA+brute force segments completed"); } // Verify the merged CAGRA+brute force index @@ -948,7 +980,8 @@ public void testMergeCagraAndBruteForceIndex() throws IOException { var vectorValues = leafReader.getFloatVectorValues("vector"); int actualVectorCount = vectorValues != null ? vectorValues.size() : 0; - log.info( + log.log( + Level.FINE, "CAGRA+brute force merge results: Total documents: " + totalDocuments + ", Expected vectors: " @@ -976,7 +1009,8 @@ public void testMergeCagraAndBruteForceIndex() throws IOException { "Should not find more vectors than exist", vectorResults.scoreDocs.length <= actualVectorCount); - log.info( + log.log( + Level.FINE, "CAGRA+brute force index search found " + vectorResults.scoreDocs.length + " results out of " @@ -1006,13 +1040,16 @@ public void testMergeCagraAndBruteForceIndex() throws IOException { TopDocs trialResults = searcher.search(trialQuery, Math.min(5, actualVectorCount)); assertTrue("Trial " + trial + " should find results", trialResults.scoreDocs.length > 0); - log.info("Trial " + trial + " found " + trialResults.scoreDocs.length + " results"); + log.log( + Level.FINE, + "Trial " + trial + " found " + trialResults.scoreDocs.length + " results"); } } else { - log.info("No vectors in CAGRA+brute force merged index - skipping vector search"); + log.log( + Level.FINE, "No vectors in CAGRA+brute force merged index - skipping vector search"); } - log.info("CAGRA+brute force merge verification completed successfully"); + log.log(Level.FINE, "CAGRA+brute force merge verification completed successfully"); } } @@ -1025,7 +1062,7 @@ public void testLargeScaleMerge() throws IOException { "testLargeScaleMerge requires -DlargeScale=true", Boolean.parseBoolean(System.getProperty("largeScale", "false"))); - log.info("Starting testLargeScaleMerge"); + log.log(Level.FINE, "Starting testLargeScaleMerge"); // Randomize large scale parameters int maxBufferedDocs = 40 + random().nextInt(21); // 40-60 docs per buffer @@ -1033,7 +1070,8 @@ public void testLargeScaleMerge() throws IOException { int docsPerSegment = 30 + random().nextInt(21); // 30-50 docs per segment int totalDocuments = segmentCount * docsPerSegment; - log.info( + log.log( + Level.FINE, "Randomized large scale parameters: maxBufferedDocs=" + maxBufferedDocs + ", segmentCount=" @@ -1051,7 +1089,7 @@ public void testLargeScaleMerge() throws IOException { try (IndexWriter writer = new IndexWriter(directory, config)) { for (int seg = 0; seg < segmentCount; seg++) { - log.info("Creating segment " + (seg + 1) + "/" + segmentCount); + log.log(Level.FINE, "Creating segment " + (seg + 1) + "/" + segmentCount); // Randomize vector probability per segment double vectorProbability = @@ -1075,14 +1113,16 @@ public void testLargeScaleMerge() throws IOException { writer.commit(); } - log.info("Created " + segmentCount + " segments with " + totalDocuments + " total documents"); + log.log( + Level.FINE, + "Created " + segmentCount + " segments with " + totalDocuments + " total documents"); // Force merge all segments long startTime = System.currentTimeMillis(); writer.forceMerge(1); long mergeTime = System.currentTimeMillis() - startTime; - log.info("Large scale merge completed in " + mergeTime + "ms"); + log.log(Level.FINE, "Large scale merge completed in " + mergeTime + "ms"); } // Verify the large merged index @@ -1110,7 +1150,8 @@ public void testLargeScaleMerge() throws IOException { long searchTime = System.currentTimeMillis() - searchStart; assertTrue("Should find vector results in large index", vectorResults.scoreDocs.length > 0); - log.info( + log.log( + Level.FINE, "Vector search in large index returned " + vectorResults.scoreDocs.length + " results out of " @@ -1119,10 +1160,10 @@ public void testLargeScaleMerge() throws IOException { + searchTime + "ms"); } else { - log.info("No vectors in large merged index - skipping vector search"); + log.log(Level.FINE, "No vectors in large merged index - skipping vector search"); } - log.info("Large scale merge verification completed successfully"); + log.log(Level.FINE, "Large scale merge verification completed successfully"); } } From 14531890fdbbf122ffe4f9220f32eb69a08c5b89 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 9 Dec 2025 11:08:46 -0500 Subject: [PATCH 047/105] Enable merge barriers (NVIDIA/cuvs-lucene#68) Enable the `merge_barriers` setting in `.github/ops-bot.yaml` to enable the new merge barriers plugin. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cuvs-lucene/pull/68 --- java/cuvs-lucene/.github/ops-bot.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/java/cuvs-lucene/.github/ops-bot.yaml b/java/cuvs-lucene/.github/ops-bot.yaml index 2ed5231aef..2d5999296a 100644 --- a/java/cuvs-lucene/.github/ops-bot.yaml +++ b/java/cuvs-lucene/.github/ops-bot.yaml @@ -7,3 +7,4 @@ label_checker: true release_drafter: true recently_updated: true forward_merger: true +merge_barriers: true From 43b4986d1822f6095a3c8c7ce8fc3b0acd476621 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Mon, 5 Jan 2026 09:54:41 -0500 Subject: [PATCH 048/105] Fix for issue - pipeline marked successful even when the build failed (NVIDIA/cuvs-lucene#74) Fixes issue NVIDIA/cuvs-lucene#70 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Gil Forsyth (https://github.com/gforsyth) URL: https://github.com/rapidsai/cuvs-lucene/pull/74 --- java/cuvs-lucene/build.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index bf768069d1..b67cb56fec 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -39,10 +39,6 @@ if ! hasArg --run-java-tests; then fi mvn clean verify "${MAVEN_VERIFY_ARGS[@]}" \ + && mvn jacoco:report \ && mvn install:install-file -Dfile=./target/cuvs-lucene-$VERSION.jar -DgroupId=$GROUP_ID -DartifactId=cuvs-lucene -Dversion=$VERSION -Dpackaging=jar \ && cp pom.xml ./target/ - -# Generate JaCoCo code coverage reports available here: target/site/jacoco/index.html -if hasArg --run-java-tests; then - mvn jacoco:report -fi From 432400f236823f37c5536928a0058cb8feb470e9 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Mon, 5 Jan 2026 12:10:27 -0500 Subject: [PATCH 049/105] Update spotless config - remove license header enforcement (NVIDIA/cuvs-lucene#83) Fixes NVIDIA/cuvs-lucene#82 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/83 --- java/cuvs-lucene/license-header.txt | 4 ---- java/cuvs-lucene/pom.xml | 3 --- 2 files changed, 7 deletions(-) delete mode 100644 java/cuvs-lucene/license-header.txt diff --git a/java/cuvs-lucene/license-header.txt b/java/cuvs-lucene/license-header.txt deleted file mode 100644 index e49ea20ab7..0000000000 --- a/java/cuvs-lucene/license-header.txt +++ /dev/null @@ -1,4 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. - * SPDX-License-Identifier: Apache-2.0 - */ diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 1ba4d7d45a..743b1df910 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -104,9 +104,6 @@ true false - - ${project.basedir}/license-header.txt - From 46ce1d5abf23a7f080fa5f0034cce6e1b8f2bb83 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Mon, 5 Jan 2026 12:55:14 -0500 Subject: [PATCH 050/105] Fix groupId in build.sh and remove warnings (NVIDIA/cuvs-lucene#69) PR that: - Fixes the `groupId` in the build script. - Fixes the warnings observed when building `cuvs-lucene` - Fix `enableRMMPooledMemory` method in `FilterCuVSProvider` Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - MithunR (https://github.com/mythrocks) - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/cuvs-lucene/pull/69 --- java/cuvs-lucene/build.sh | 4 ++-- java/cuvs-lucene/pom.xml | 9 ++++----- .../java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java | 4 ++-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index b67cb56fec..fc3c492278 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -e -u -o pipefail @@ -9,7 +9,7 @@ ARGS="$*" NUMARGS=$# VERSION="26.02.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked -GROUP_ID="com.nvidia.cuvs" +GROUP_ID="com.nvidia.cuvs.lucene" function hasArg { (( NUMARGS != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 743b1df910..010e9b67b3 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -37,8 +37,8 @@ - 21 - 21 + 22 + 22 UTF-8 UTF-8 @@ -165,9 +165,8 @@ jar - com.nvidia.cuvs.lucene - all - ${project.build.directory}/javadocs + all,-missing + ${project.build.directory}/javadocs diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index b6cd3227f2..6cdb656b5c 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; @@ -148,7 +148,7 @@ public void enableRMMManagedPooledMemory(int arg0, int arg1) { @Override public void enableRMMPooledMemory(int arg0, int arg1) { - delegate.enableRMMManagedPooledMemory(arg0, arg1); + delegate.enableRMMPooledMemory(arg0, arg1); } @Override From 2e191ff1f9672aa350a1c0e4a8bc8bb437a0c311 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 6 Jan 2026 08:47:23 -0500 Subject: [PATCH 051/105] Setup examples (NVIDIA/cuvs-lucene#72) Fixes NVIDIA/cuvs-lucene#67 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - James Lamb (https://github.com/jameslamb) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/72 --- java/cuvs-lucene/ci/release/update-version.sh | 10 +- java/cuvs-lucene/examples/README.md | 35 ++++ java/cuvs-lucene/examples/pom.xml | 159 ++++++++++++++++++ .../assembly/jar-with-merged-services.xml | 22 +++ .../examples/AcceleratedHnswExample.java | 130 ++++++++++++++ .../examples/IndexAndSearchonGPUExample.java | 132 +++++++++++++++ .../nvidia/cuvs/lucene/examples/Utils.java | 39 +++++ .../src/main/resources/logging.properties | 9 + .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 0 .../lucene/Lucene101AcceleratedHNSWCodec.java | 0 .../services/org.apache.lucene.codecs.Codec | 5 + .../org.apache.lucene.codecs.KnnVectorsFormat | 16 +- 12 files changed, 541 insertions(+), 16 deletions(-) create mode 100644 java/cuvs-lucene/examples/README.md create mode 100644 java/cuvs-lucene/examples/pom.xml create mode 100644 java/cuvs-lucene/examples/src/main/assembly/jar-with-merged-services.xml create mode 100644 java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java create mode 100644 java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java create mode 100644 java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/Utils.java create mode 100644 java/cuvs-lucene/examples/src/main/resources/logging.properties rename java/cuvs-lucene/src/{test => main}/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java (100%) rename java/cuvs-lucene/src/{test => main}/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java (100%) create mode 100644 java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec diff --git a/java/cuvs-lucene/ci/release/update-version.sh b/java/cuvs-lucene/ci/release/update-version.sh index afe73f640d..404010440e 100755 --- a/java/cuvs-lucene/ci/release/update-version.sh +++ b/java/cuvs-lucene/ci/release/update-version.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 ######################## @@ -105,7 +105,9 @@ echo "${RAPIDS_BRANCH_NAME}" > RAPIDS_BRANCH # Update Java version NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}" sed_runner "s/VERSION=\".*\"/VERSION=\"${NEXT_FULL_JAVA_TAG}\"/g" build.sh -sed_runner "/.*/s//${NEXT_FULL_JAVA_TAG}<\/version>/g" pom.xml +for FILE in pom.xml examples/pom.xml; do + sed_runner "/.*/s//${NEXT_FULL_JAVA_TAG}<\/version>/g" "${FILE}" +done sed_runner "s| CuVS [[:digit:]]\{2\}\.[[:digit:]]\{2\} | CuVS ${NEXT_SHORT_TAG} |g" README.md @@ -117,3 +119,7 @@ for FILE in .github/workflows/*.yaml; do sed_runner "/shared-workflows/ s|@.*|@${RAPIDS_BRANCH_NAME}|g" "${FILE}" sed_runner "s/:[0-9]*\\.[0-9]*-/:${NEXT_SHORT_TAG}-/g" "${FILE}" done + +# Update versions in the examples readme file +sed_runner "s/rapidsai\/ci-conda:[\.0-9]*-cuda/rapidsai\/ci-conda:${NEXT_SHORT_TAG}-cuda/g" "examples/README.md" +sed_runner "s/target\/examples-[\.0-9]*-jar/target\/examples-${NEXT_FULL_JAVA_TAG}-jar/g" "examples/README.md" diff --git a/java/cuvs-lucene/examples/README.md b/java/cuvs-lucene/examples/README.md new file mode 100644 index 0000000000..acebe9e252 --- /dev/null +++ b/java/cuvs-lucene/examples/README.md @@ -0,0 +1,35 @@ +# Examples + +This maven project contains basic examples that showcase how `cuvs-lucene` can be used. + +## Prerequisites +- [Docker](https://www.docker.com/) +- [Nvidia Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) +- A machine with an Nvidia GPU + +## Steps + +If you are currently in this directory (and to be in the `cuvs-lucene's` root directory) do: +```sh +cd .. +``` + +Then do: +```sh +docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.02-cuda13.0.2-ubuntu24.04-py3.13 +``` + +Inside the docker container (and in the `cuvs-lucene's` root directory) do: +```sh +./ci/build_java.sh && conda activate java && cd examples +``` + +To run Accelerated HNSW example do: +```sh +mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.02.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.AcceleratedHnswExample +``` + +To run the Index and Search on GPU example do: +```sh +mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.02.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.IndexAndSearchonGPUExample +``` diff --git a/java/cuvs-lucene/examples/pom.xml b/java/cuvs-lucene/examples/pom.xml new file mode 100644 index 0000000000..ecdb5b4fc5 --- /dev/null +++ b/java/cuvs-lucene/examples/pom.xml @@ -0,0 +1,159 @@ + + + + 4.0.0 + + com.nvidia.cuvs.lucene.examples + examples + 26.02.0 + examples + + + 22 + 22 + UTF-8 + UTF-8 + + + + + + + + + + org.apache.lucene + lucene-core + 10.2.0 + + + org.apache.lucene + lucene-codecs + 10.2.0 + test + + + org.apache.lucene + lucene-backward-codecs + 10.2.0 + + + commons-io + commons-io + 2.18.0 + + + com.nvidia.cuvs.lucene + cuvs-lucene + 26.02.0 + + + + + + + com.diffplug.spotless + spotless-maven-plugin + 2.44.5 + + + + apply + + validate + + + + + + src/**/*.java + + + 1.27.0 + + true + false + + + ../license-header.txt + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.6.0 + + + src/main/assembly/jar-with-merged-services.xml + + + + true + + + true + true + + + + + + make-assembly + package + + single + + + + + + + + + maven-clean-plugin + 3.4.0 + + + maven-resources-plugin + 3.3.1 + + + maven-compiler-plugin + 3.13.0 + + + maven-surefire-plugin + 3.3.0 + + + maven-jar-plugin + 3.4.2 + + + maven-install-plugin + 3.1.2 + + + maven-deploy-plugin + 3.1.2 + + + maven-site-plugin + 3.12.1 + + + maven-project-info-reports-plugin + 3.6.1 + + + + + diff --git a/java/cuvs-lucene/examples/src/main/assembly/jar-with-merged-services.xml b/java/cuvs-lucene/examples/src/main/assembly/jar-with-merged-services.xml new file mode 100644 index 0000000000..dc6706188d --- /dev/null +++ b/java/cuvs-lucene/examples/src/main/assembly/jar-with-merged-services.xml @@ -0,0 +1,22 @@ + + jar-with-merged-services + + jar + + false + + + / + true + true + runtime + + + + + metaInf-services + + + diff --git a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java new file mode 100644 index 0000000000..8ac19071a8 --- /dev/null +++ b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java @@ -0,0 +1,130 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene.examples; + +import static com.nvidia.cuvs.lucene.examples.Utils.generateDataset; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +import com.nvidia.cuvs.lucene.Lucene101AcceleratedHNSWCodec; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Random; +import java.util.UUID; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.commons.io.FileUtils; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; + +public class AcceleratedHnswExample { + + private static Logger log = Logger.getLogger(AcceleratedHnswExample.class.getName()); + + private static Random random; + private static Path indexDirPath; + + public static void main(String[] args) throws Exception { + + Codec codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); + IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); + + random = new Random(222); + indexDirPath = Paths.get(UUID.randomUUID().toString()); + + final int COMMIT_FREQ = 2000; + final String ID_FIELD = "id"; + final String VECTOR_FIELD = "vector_field"; + + int numDocs = 2000; + int dimension = 32; + int topK = 5; + int count = COMMIT_FREQ; + float[][] dataset = generateDataset(random, numDocs, dimension); + + // Indexing + try (Directory indexDirectory = FSDirectory.open(indexDirPath); + IndexWriter indexWriter = new IndexWriter(indexDirectory, config)) { + for (int i = 0; i < numDocs; i++) { + Document document = new Document(); + document.add(new StringField(ID_FIELD, Integer.toString(i), Field.Store.YES)); + document.add(new KnnFloatVectorField(VECTOR_FIELD, dataset[i], EUCLIDEAN)); + indexWriter.addDocument(document); + count -= 1; + if (count == 0) { + indexWriter.commit(); + count = COMMIT_FREQ; + } + } + } + + // Searching + try (Directory indexDirectory = FSDirectory.open(indexDirPath); + DirectoryReader reader = DirectoryReader.open(indexDirectory)) { + log.log(Level.FINE, "Successfully opened index"); + + for (LeafReaderContext leafReaderContext : reader.leaves()) { + LeafReader leafReader = leafReaderContext.reader(); + FloatVectorValues knnValues = leafReader.getFloatVectorValues(VECTOR_FIELD); + log.log( + Level.FINE, + VECTOR_FIELD + + " field: " + + knnValues.size() + + " vectors, " + + knnValues.dimension() + + " dimensions"); + } + + log.log(Level.FINE, "Testing vector search queries..."); + IndexSearcher searcher = new IndexSearcher(reader); + + float[] queryVector = generateDataset(random, 1, dimension)[0]; + log.log(Level.FINE, "Query vector: " + Arrays.toString(queryVector)); + + KnnFloatVectorQuery query = new KnnFloatVectorQuery(VECTOR_FIELD, queryVector, topK); + TopDocs results = searcher.search(query, topK); + + log.log(Level.FINE, "Search results (" + results.totalHits + " total hits):"); + + for (int i = 0; i < results.scoreDocs.length; i++) { + ScoreDoc scoreDoc = results.scoreDocs[i]; + Document doc = searcher.storedFields().document(scoreDoc.doc); + String id = doc.get(ID_FIELD); + log.log( + Level.FINE, + " Rank " + + (i + 1) + + ": doc " + + scoreDoc.doc + + " (id=" + + id + + "), score=" + + scoreDoc.score); + } + } + + File indexDirPathFile = indexDirPath.toFile(); + if (indexDirPathFile.exists() && indexDirPathFile.isDirectory()) { + FileUtils.deleteDirectory(indexDirPathFile); + } + } +} diff --git a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java new file mode 100644 index 0000000000..8c889620c5 --- /dev/null +++ b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java @@ -0,0 +1,132 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene.examples; + +import static com.nvidia.cuvs.lucene.examples.Utils.generateDataset; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +import com.nvidia.cuvs.lucene.CuVS2510GPUSearchCodec; +import com.nvidia.cuvs.lucene.GPUKnnFloatVectorQuery; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Random; +import java.util.UUID; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.commons.io.FileUtils; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; + +public class IndexAndSearchonGPUExample { + + private static Logger log = Logger.getLogger(IndexAndSearchonGPUExample.class.getName()); + + private static Random random; + private static Path indexDirPath; + + public static void main(String[] args) throws Exception { + + Codec codec = new CuVS2510GPUSearchCodec(); + IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); + + random = new Random(222); + indexDirPath = Paths.get(UUID.randomUUID().toString()); + + final int COMMIT_FREQ = 2000; + final String ID_FIELD = "id"; + final String VECTOR_FIELD = "vector_field"; + + int numDocs = 2000; + int dimension = 32; + int topK = 5; + int count = COMMIT_FREQ; + float[][] dataset = generateDataset(random, numDocs, dimension); + + // Indexing + try (Directory indexDirectory = FSDirectory.open(indexDirPath); + IndexWriter indexWriter = new IndexWriter(indexDirectory, config)) { + for (int i = 0; i < numDocs; i++) { + Document document = new Document(); + document.add(new StringField(ID_FIELD, Integer.toString(i), Field.Store.YES)); + document.add(new KnnFloatVectorField(VECTOR_FIELD, dataset[i], EUCLIDEAN)); + indexWriter.addDocument(document); + count -= 1; + if (count == 0) { + indexWriter.commit(); + count = COMMIT_FREQ; + } + } + } + + // Searching + try (Directory indexDirectory = FSDirectory.open(indexDirPath); + DirectoryReader reader = DirectoryReader.open(indexDirectory)) { + log.log(Level.FINE, "Successfully opened index"); + + for (LeafReaderContext leafReaderContext : reader.leaves()) { + LeafReader leafReader = leafReaderContext.reader(); + FloatVectorValues knnValues = leafReader.getFloatVectorValues(VECTOR_FIELD); + log.log( + Level.FINE, + VECTOR_FIELD + + " field: " + + knnValues.size() + + " vectors, " + + knnValues.dimension() + + " dimensions"); + } + + log.log(Level.FINE, "Testing vector search queries..."); + IndexSearcher searcher = new IndexSearcher(reader); + + float[] queryVector = generateDataset(random, 1, dimension)[0]; + log.log(Level.FINE, "Query vector: " + Arrays.toString(queryVector)); + + KnnFloatVectorQuery query = + new GPUKnnFloatVectorQuery(VECTOR_FIELD, queryVector, topK, null, topK, 1); + TopDocs results = searcher.search(query, topK); + + log.log(Level.FINE, "Search results (" + results.totalHits + " total hits):"); + + for (int i = 0; i < results.scoreDocs.length; i++) { + ScoreDoc scoreDoc = results.scoreDocs[i]; + Document doc = searcher.storedFields().document(scoreDoc.doc); + String id = doc.get(ID_FIELD); + log.log( + Level.FINE, + " Rank " + + (i + 1) + + ": doc " + + scoreDoc.doc + + " (id=" + + id + + "), score=" + + scoreDoc.score); + } + } + + File indexDirPathFile = indexDirPath.toFile(); + if (indexDirPathFile.exists() && indexDirPathFile.isDirectory()) { + FileUtils.deleteDirectory(indexDirPathFile); + } + } +} diff --git a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/Utils.java b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/Utils.java new file mode 100644 index 0000000000..a7a14e22c4 --- /dev/null +++ b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/Utils.java @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene.examples; + +import java.util.Random; + +public class Utils { + + public static float[][] generateDataset(Random random, int size, int dimensions) { + float[][] dataset = new float[size][dimensions]; + for (int i = 0; i < size; i++) { + for (int j = 0; j < dimensions; j++) { + dataset[i][j] = random.nextFloat() * 100; + } + } + return dataset; + } + + public static float[] generateRandomVector(int dimensions, Random random) { + float[] vector = new float[dimensions]; + for (int i = 0; i < dimensions; i++) { + vector[i] = random.nextFloat() * 100; + } + return vector; + } + + public static float[][] generateQueries(Random random, int dimensions, int numQueries) { + // Generate random query vectors + float[][] queries = new float[numQueries][dimensions]; + for (int i = 0; i < numQueries; i++) { + for (int j = 0; j < dimensions; j++) { + queries[i][j] = random.nextFloat() * 100; + } + } + return queries; + } +} diff --git a/java/cuvs-lucene/examples/src/main/resources/logging.properties b/java/cuvs-lucene/examples/src/main/resources/logging.properties new file mode 100644 index 0000000000..53f2c42727 --- /dev/null +++ b/java/cuvs-lucene/examples/src/main/resources/logging.properties @@ -0,0 +1,9 @@ +# Set root level +.level=FINEST + +# Define handlers +handlers=java.util.logging.ConsoleHandler + +java.util.logging.ConsoleHandler.level=FINEST +java.util.logging.ConsoleHandler.formatter=java.util.logging.SimpleFormatter +java.util.logging.SimpleFormatter.format=[%1$tF %1$tT] %4$s %2$s %5$s%6$s%n diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java similarity index 100% rename from java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java similarity index 100% rename from java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java diff --git a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec new file mode 100644 index 0000000000..f01d2e8da9 --- /dev/null +++ b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 + +com.nvidia.cuvs.lucene.Lucene101AcceleratedHNSWCodec +com.nvidia.cuvs.lucene.CuVS2510GPUSearchCodec diff --git a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index 747a157595..4e20fcb599 100644 --- a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -1,17 +1,5 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-License-Identifier: Apache-2.0 org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat From 43225c72e4a56d03e818fc572aaf2d5144d07550 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 6 Jan 2026 11:26:44 -0500 Subject: [PATCH 052/105] Refactor GPUPerLeafCuVSKnnCollector, plug in iTopK and search width values (NVIDIA/cuvs-lucene#79) Fixes NVIDIA/cuvs-lucene#78 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/79 --- .../cuvs/lucene/CuVS2510GPUVectorsReader.java | 22 +++-- .../cuvs/lucene/GPUKnnFloatVectorQuery.java | 6 +- .../lucene/GPUPerLeafCuVSKnnCollector.java | 82 +++---------------- .../cuvs/lucene/TestCuVSDeletedDocuments.java | 17 ++-- .../com/nvidia/cuvs/lucene/TestCuVSGaps.java | 9 +- .../TestCuVSRandomizedVectorSearch.java | 12 +-- 6 files changed, 53 insertions(+), 95 deletions(-) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java index 56a9880d3d..a016feebd4 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; @@ -173,7 +173,7 @@ private void validateFieldEntry(FieldInfo info, FieldEntry fieldEntry) { /** * Reads the fieldInfo for each index field and loads FieldEntry in a map. * - * @param meta intance of the ChecksumIndexInput + * @param meta instance of the ChecksumIndexInput * @throws IOException */ private void readFields(ChecksumIndexInput meta) throws IOException { @@ -428,11 +428,19 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits Map result; if (knnCollector.k() <= 1024 && cuvsIndex.getCagraIndex() != null) { - CagraSearchParams searchParams = - new CagraSearchParams.Builder() - .withItopkSize(topK) // TODO: params - .withSearchWidth(1) - .build(); + + CagraSearchParams searchParams; + if (knnCollector instanceof GPUPerLeafCuVSKnnCollector) { + GPUPerLeafCuVSKnnCollector collector = (GPUPerLeafCuVSKnnCollector) knnCollector; + searchParams = + new CagraSearchParams.Builder() + .withItopkSize(Math.max(collector.getiTopK(), topK)) + .withSearchWidth(collector.getSearchWidth()) + .build(); + } else { + // Setting itopK as topK because in any case iTopK should be ATLEAST equal to topK + searchParams = new CagraSearchParams.Builder().withItopkSize(topK).build(); + } var query = new CagraQuery.Builder(resources) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java index 7dc984b286..6a9daae772 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; @@ -7,7 +7,6 @@ import java.io.IOException; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.search.KnnCollector; import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TopDocs; @@ -49,7 +48,8 @@ protected TopDocs approximateSearch( KnnCollectorManager knnCollectorManager) throws IOException { - KnnCollector results = new GPUPerLeafCuVSKnnCollector(k, iTopK, searchWidth); + GPUPerLeafCuVSKnnCollector results = + new GPUPerLeafCuVSKnnCollector(k, visitedLimit, iTopK, searchWidth); LeafReader reader = context.reader(); reader.searchNearestVectors(field, this.getTargetCopy(), results, acceptDocs); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java index 9d18b5167f..421b2baa7c 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java @@ -1,95 +1,39 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; -import java.util.ArrayList; -import java.util.List; -import org.apache.lucene.search.KnnCollector; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.search.TotalHits; -import org.apache.lucene.search.knn.KnnSearchStrategy; +import org.apache.lucene.search.TopKnnCollector; /** * KnnCollector for cuVS used for search on the GPU. * * @since 25.10 */ -class GPUPerLeafCuVSKnnCollector implements KnnCollector { +class GPUPerLeafCuVSKnnCollector extends TopKnnCollector { - public List scoreDocs; - public int topK = 0; - public int iTopK = topK; // TODO getter, no setter - public int searchWidth = 1; // TODO getter, no setter - public int results = 0; + private int iTopK; + private int searchWidth; /** * Initializes {@link GPUPerLeafCuVSKnnCollector} * - * @param topK the topk value + * @param topK the topK value * @param iTopK the iTopK value * @param searchWidth the search width */ - public GPUPerLeafCuVSKnnCollector(int topK, int iTopK, int searchWidth) { - super(); - this.topK = topK; - this.iTopK = iTopK; + public GPUPerLeafCuVSKnnCollector(int topK, int visitLimit, int iTopK, int searchWidth) { + super(topK, visitLimit); + this.iTopK = iTopK > topK ? iTopK : topK; this.searchWidth = searchWidth; - scoreDocs = new ArrayList(); } - @Override - public boolean earlyTerminated() { - // TODO: may need implementation - return false; + public int getiTopK() { + return iTopK; } - @Override - public void incVisitedCount(int count) { - // TODO: may need implementation - } - - @Override - public long visitedCount() { - // TODO: may need implementation - return 0; - } - - @Override - public long visitLimit() { - // TODO: may need implementation - return 0; - } - - @Override - public int k() { - return topK; - } - - @Override - @SuppressWarnings("cast") - public boolean collect(int docId, float similarity) { - scoreDocs.add(new ScoreDoc(docId, similarity)); - return true; - } - - @Override - public float minCompetitiveSimilarity() { - // TODO: may need implementation - return 0; - } - - @Override - public TopDocs topDocs() { - return new TopDocs( - new TotalHits(scoreDocs.size(), TotalHits.Relation.EQUAL_TO), - scoreDocs.toArray(new ScoreDoc[scoreDocs.size()])); - } - - @Override - public KnnSearchStrategy getSearchStrategy() { - return KnnSearchStrategy.Patience.DEFAULT; + public int getSearchWidth() { + return searchWidth; } } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java index 04c807a054..3363a61478 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; @@ -26,7 +26,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -93,7 +92,8 @@ public void testVectorSearchWithDeletedDocuments() throws IOException { // Use a random vector for query float[] queryVector = generateRandomVector(dimensions, random); - Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + GPUKnnFloatVectorQuery query = + new GPUKnnFloatVectorQuery("vector", queryVector, topK, null, topK, 1); ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; // Verify we got results @@ -169,7 +169,8 @@ public void testVectorSearchWithMixedDeletedAndMissingVectors() throws IOExcepti IndexSearcher searcher = newSearcher(reader); float[] queryVector = generateRandomVector(dimensions, random); - Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + GPUKnnFloatVectorQuery query = + new GPUKnnFloatVectorQuery("vector", queryVector, topK, null, topK, 1); ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; // Verify results @@ -183,7 +184,7 @@ public void testVectorSearchWithMixedDeletedAndMissingVectors() throws IOExcepti // Test filtered search with deletions Query filter = new TermQuery(new Term("category", "A")); - Query filteredQuery = + GPUKnnFloatVectorQuery filteredQuery = new GPUKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; @@ -233,7 +234,8 @@ public void testVectorSearchAfterAllDocumentsDeleted() throws IOException { IndexSearcher searcher = newSearcher(reader); float[] queryVector = generateRandomVector(dimensions, random); - Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + GPUKnnFloatVectorQuery query = + new GPUKnnFloatVectorQuery("vector", queryVector, topK, null, topK, 1); TopDocs results = searcher.search(query, topK); assertEquals( @@ -294,7 +296,8 @@ public void testVectorSearchWithPartialDeletionAndReindexing() throws IOExceptio IndexSearcher searcher = newSearcher(reader); float[] queryVector = generateRandomVector(dimensions, random); - Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + GPUKnnFloatVectorQuery query = + new GPUKnnFloatVectorQuery("vector", queryVector, topK, null, topK, 1); ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; Set resultIds = new HashSet<>(); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java index a48b8f6c8e..42d3464c06 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; @@ -22,7 +22,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -116,7 +115,8 @@ public void testVectorSearchWithAlternatingDocuments() throws IOException { float[] queryVector = dataset[0]; int topK = random.nextInt(5, TOP_K_LIMIT); - Query query = new KnnFloatVectorQuery("vector", queryVector, topK); + GPUKnnFloatVectorQuery query = + new GPUKnnFloatVectorQuery("vector", queryVector, topK, null, topK, 1); ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; // Verify we get exactly TOP_K results @@ -153,7 +153,8 @@ public void testVectorSearchWithFilterAndAlternatingDocuments() throws IOExcepti // This should further restrict our results to even numbers 0, 2, 4, 6, 8 Query filter = new TermQuery(new Term("id", "8")); // Only match document 8 - Query filteredQuery = new GPUKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); + GPUKnnFloatVectorQuery filteredQuery = + new GPUKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; // Should only get document 8 (the only one that matches the filter and has a vector) diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java index abcd6b67b3..9e42528327 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; @@ -25,7 +25,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.KnnFloatVectorQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -123,7 +122,8 @@ public void testVectorSearch() throws IOException { log.log(Level.FINE, "Query size: " + numQueries + "x" + queries[0].length); log.log(Level.FINE, "TopK: " + topK); - Query query = new KnnFloatVectorQuery("vector", queries[0], topK); + GPUKnnFloatVectorQuery query = + new GPUKnnFloatVectorQuery("vector", queries[0], topK, null, topK, 1); int correct[] = new int[topK]; for (int i = 0; i < topK; i++) correct[i] = expected.get(0).get(i); @@ -183,7 +183,8 @@ public void testVectorSearchWithFilter() throws IOException { if (dataset.length < topK) topK = dataset.length; // Find a document that has a vector by doing a search first - Query unfiltered = new KnnFloatVectorQuery("vector", dataset[0], 1); + GPUKnnFloatVectorQuery unfiltered = + new GPUKnnFloatVectorQuery("vector", dataset[0], 1, null, 1, 1); ScoreDoc[] unfilteredHits = searcher.search(unfiltered, 1).scoreDocs; // Skip test if no vectors found at all @@ -197,7 +198,8 @@ public void testVectorSearchWithFilter() throws IOException { Query filter = new TermQuery(new Term("id", targetDocId)); // Test the new constructor with filter - Query filteredQuery = new GPUKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); + GPUKnnFloatVectorQuery filteredQuery = + new GPUKnnFloatVectorQuery("vector", queryVector, topK, filter, topK, 1); ScoreDoc[] filteredHits = searcher.search(filteredQuery, topK).scoreDocs; From 2413df0bf0a0ef0ecb2aec08ce549ad1ee3edaca Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 6 Jan 2026 11:36:42 -0500 Subject: [PATCH 053/105] Setup JMH (NVIDIA/cuvs-lucene#71) Fixes NVIDIA/cuvs-lucene#41 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - James Lamb (https://github.com/jameslamb) - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/71 --- java/cuvs-lucene/bench/README.md | 9 + java/cuvs-lucene/bench/pom.xml | 180 ++++++++++++++++++ .../AcceleratedHnswIndexingBenchmarks.java | 78 ++++++++ .../AcceleratedHnswSearchBenchmarks.java | 86 +++++++++ .../benchmarks/CagraIndexingBenchmarks.java | 78 ++++++++ .../benchmarks/CagraSearchBenchmarks.java | 88 +++++++++ .../nvidia/cuvs/lucene/benchmarks/Utils.java | 102 ++++++++++ java/cuvs-lucene/ci/release/update-version.sh | 3 +- .../services/org.apache.lucene.codecs.Codec | 2 - 9 files changed, 623 insertions(+), 3 deletions(-) create mode 100644 java/cuvs-lucene/bench/README.md create mode 100644 java/cuvs-lucene/bench/pom.xml create mode 100644 java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswIndexingBenchmarks.java create mode 100644 java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswSearchBenchmarks.java create mode 100644 java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraIndexingBenchmarks.java create mode 100644 java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraSearchBenchmarks.java create mode 100644 java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/Utils.java delete mode 100644 java/cuvs-lucene/src/test/resources/META-INF/services/org.apache.lucene.codecs.Codec diff --git a/java/cuvs-lucene/bench/README.md b/java/cuvs-lucene/bench/README.md new file mode 100644 index 0000000000..c11db0b630 --- /dev/null +++ b/java/cuvs-lucene/bench/README.md @@ -0,0 +1,9 @@ +# Microbenchmarks + +This maven project provides microbenchmarks for `cuvs-lucene` using JMH. + +To run do (if on current directory): + +```sh +cd .. && mvn clean install -DskipTests && cd benchmarks && mvn clean install && java -jar target/benchmarks.jar +``` diff --git a/java/cuvs-lucene/bench/pom.xml b/java/cuvs-lucene/bench/pom.xml new file mode 100644 index 0000000000..e864793c52 --- /dev/null +++ b/java/cuvs-lucene/bench/pom.xml @@ -0,0 +1,180 @@ + + + + 4.0.0 + + com.nvidia.cuvs.lucene.benchmarks + cuvs-lucene-benchmarks + 26.02.0 + jar + + cuvs-lucene-benchmarks + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + com.nvidia.cuvs.lucene + cuvs-lucene + 26.02.0 + + + commons-io + commons-io + 2.18.0 + + + + + UTF-8 + 1.37 + 22 + 22 + 22 + benchmarks + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.4.2 + + + + true + + + true + true + + + + + + com.diffplug.spotless + spotless-maven-plugin + 2.44.5 + + + + apply + + validate + + + + + + src/**/*.java + + + 1.27.0 + + true + false + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + ${javac.target} + ${javac.target} + ${javac.target} + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + + package + + shade + + + ${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + maven-clean-plugin + 2.5 + + + maven-deploy-plugin + 2.8.1 + + + maven-install-plugin + 2.5.1 + + + maven-jar-plugin + 3.4.2 + + + maven-javadoc-plugin + 2.9.1 + + + maven-resources-plugin + 2.6 + + + maven-site-plugin + 3.3 + + + maven-source-plugin + 2.2.1 + + + maven-surefire-plugin + 2.17 + + + + + diff --git a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswIndexingBenchmarks.java b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswIndexingBenchmarks.java new file mode 100644 index 0000000000..e91e816ce6 --- /dev/null +++ b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswIndexingBenchmarks.java @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene.benchmarks; + +import static com.nvidia.cuvs.lucene.benchmarks.Utils.cleanup; +import static com.nvidia.cuvs.lucene.benchmarks.Utils.generateDataset; +import static com.nvidia.cuvs.lucene.benchmarks.Utils.index; + +import com.nvidia.cuvs.lucene.Lucene101AcceleratedHNSWCodec; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.index.IndexWriterConfig; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +@BenchmarkMode(Mode.All) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Fork(value = 1) +@Warmup(iterations = 0, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +public class AcceleratedHnswIndexingBenchmarks { + + @SuppressWarnings("unused") + private static Logger log = Logger.getLogger(AcceleratedHnswIndexingBenchmarks.class.getName()); + + private static Random random; + private static Path indexDirPath; + private static Codec codec; + + private final int COMMIT_FREQ = 100; + private final String ID_FIELD = "id"; + private final String VECTOR_FIELD = "vector_field"; + + private int numDocs; + private int dimension; + private float[][] dataset; + + @Setup(Level.Trial) + public void setup() throws Exception { + random = new Random(222); + indexDirPath = Paths.get(UUID.randomUUID().toString()); + codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); + numDocs = 1000; + dimension = 128; + dataset = generateDataset(random, numDocs, dimension); + } + + @Benchmark + public void benchmarkAcceleratedHnswIndexing(Blackhole blackhole) throws Exception { + IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); + index(indexDirPath, config, dataset, numDocs, VECTOR_FIELD, ID_FIELD, COMMIT_FREQ); + } + + @TearDown(Level.Trial) + public void tearDown() throws IOException { + cleanup(indexDirPath); + } +} diff --git a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswSearchBenchmarks.java b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswSearchBenchmarks.java new file mode 100644 index 0000000000..161c6a3934 --- /dev/null +++ b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswSearchBenchmarks.java @@ -0,0 +1,86 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene.benchmarks; + +import static com.nvidia.cuvs.lucene.benchmarks.Utils.cleanup; +import static com.nvidia.cuvs.lucene.benchmarks.Utils.generateDataset; +import static com.nvidia.cuvs.lucene.benchmarks.Utils.index; +import static com.nvidia.cuvs.lucene.benchmarks.Utils.search; + +import com.nvidia.cuvs.lucene.Lucene101AcceleratedHNSWCodec; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +@BenchmarkMode(Mode.All) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Fork(value = 1) +@Warmup(iterations = 0, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +public class AcceleratedHnswSearchBenchmarks { + + @SuppressWarnings("unused") + private static Logger log = Logger.getLogger(AcceleratedHnswSearchBenchmarks.class.getName()); + + private static Random random; + private static Path indexDirPath; + private static Codec codec; + + private final int COMMIT_FREQ = 100; + private final String ID_FIELD = "id"; + private final String VECTOR_FIELD = "vector_field"; + + private int numDocs; + private int dimension; + private int topK; + private float[][] dataset; + private float[] queryVector; + + @Setup(Level.Trial) + public void setup() throws Exception { + random = new Random(222); + indexDirPath = Paths.get(UUID.randomUUID().toString()); + codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); + numDocs = 1000; + dimension = 128; + topK = 5; + dataset = generateDataset(random, numDocs, dimension); + queryVector = generateDataset(random, 1, dimension)[0]; + IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); + index(indexDirPath, config, dataset, numDocs, VECTOR_FIELD, ID_FIELD, COMMIT_FREQ); + } + + @Benchmark + public void benchmarkAcceleratedHnswSearch(Blackhole blackhole) throws Exception { + KnnFloatVectorQuery query = new KnnFloatVectorQuery(VECTOR_FIELD, queryVector, topK); + search(indexDirPath, VECTOR_FIELD, queryVector, topK, query); + } + + @TearDown(Level.Trial) + public void tearDown() throws IOException { + cleanup(indexDirPath); + } +} diff --git a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraIndexingBenchmarks.java b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraIndexingBenchmarks.java new file mode 100644 index 0000000000..119f95a175 --- /dev/null +++ b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraIndexingBenchmarks.java @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene.benchmarks; + +import static com.nvidia.cuvs.lucene.benchmarks.Utils.cleanup; +import static com.nvidia.cuvs.lucene.benchmarks.Utils.generateDataset; +import static com.nvidia.cuvs.lucene.benchmarks.Utils.index; + +import com.nvidia.cuvs.lucene.CuVS2510GPUSearchCodec; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.index.IndexWriterConfig; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +@BenchmarkMode(Mode.All) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Fork(value = 1) +@Warmup(iterations = 0, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +public class CagraIndexingBenchmarks { + + @SuppressWarnings("unused") + private static Logger log = Logger.getLogger(CagraIndexingBenchmarks.class.getName()); + + private static Random random; + private static Path indexDirPath; + private static Codec codec; + + private final int COMMIT_FREQ = 100; + private final String ID_FIELD = "id"; + private final String VECTOR_FIELD = "vector_field"; + + private int numDocs; + private int dimension; + private float[][] dataset; + + @Setup(Level.Trial) + public void setup() throws Exception { + random = new Random(222); + indexDirPath = Paths.get(UUID.randomUUID().toString()); + codec = new CuVS2510GPUSearchCodec(); + numDocs = 1000; + dimension = 128; + dataset = generateDataset(random, numDocs, dimension); + } + + @Benchmark + public void benchmarkAcceleratedHnswIndexing(Blackhole blackhole) throws Exception { + IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); + index(indexDirPath, config, dataset, numDocs, VECTOR_FIELD, ID_FIELD, COMMIT_FREQ); + } + + @TearDown(Level.Trial) + public void tearDown() throws IOException { + cleanup(indexDirPath); + } +} diff --git a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraSearchBenchmarks.java b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraSearchBenchmarks.java new file mode 100644 index 0000000000..45f35caff7 --- /dev/null +++ b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraSearchBenchmarks.java @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene.benchmarks; + +import static com.nvidia.cuvs.lucene.benchmarks.Utils.cleanup; +import static com.nvidia.cuvs.lucene.benchmarks.Utils.generateDataset; +import static com.nvidia.cuvs.lucene.benchmarks.Utils.index; +import static com.nvidia.cuvs.lucene.benchmarks.Utils.search; + +import com.nvidia.cuvs.lucene.CuVS2510GPUSearchCodec; +import com.nvidia.cuvs.lucene.GPUKnnFloatVectorQuery; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +@BenchmarkMode(Mode.All) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Fork(value = 1) +@Warmup(iterations = 0, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 1, timeUnit = TimeUnit.SECONDS) +public class CagraSearchBenchmarks { + + @SuppressWarnings("unused") + private static Logger log = Logger.getLogger(CagraSearchBenchmarks.class.getName()); + + private static Random random; + private static Path indexDirPath; + private static Codec codec; + + private final int COMMIT_FREQ = 100; + private final String ID_FIELD = "id"; + private final String VECTOR_FIELD = "vector_field"; + + private int numDocs; + private int dimension; + private int topK; + private float[][] dataset; + private float[] queryVector; + + @Setup(Level.Trial) + public void setup() throws Exception { + random = new Random(222); + indexDirPath = Paths.get(UUID.randomUUID().toString()); + codec = new CuVS2510GPUSearchCodec(); + numDocs = 1000; + dimension = 128; + topK = 5; + dataset = generateDataset(random, numDocs, dimension); + queryVector = generateDataset(random, 1, dimension)[0]; + IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); + index(indexDirPath, config, dataset, numDocs, VECTOR_FIELD, ID_FIELD, COMMIT_FREQ); + } + + @Benchmark + public void benchmarkAcceleratedHnswSearch(Blackhole blackhole) throws Exception { + KnnFloatVectorQuery query = + new GPUKnnFloatVectorQuery(VECTOR_FIELD, queryVector, topK, null, topK, 1); + search(indexDirPath, VECTOR_FIELD, queryVector, topK, query); + } + + @TearDown(Level.Trial) + public void tearDown() throws IOException { + cleanup(indexDirPath); + } +} diff --git a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/Utils.java b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/Utils.java new file mode 100644 index 0000000000..f311c82b49 --- /dev/null +++ b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/Utils.java @@ -0,0 +1,102 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene.benchmarks; + +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Random; +import org.apache.commons.io.FileUtils; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; + +public class Utils { + + public static void index( + Path indexDirPath, + IndexWriterConfig config, + float[][] dataset, + int numDocs, + String idField, + String vectorField, + int commitFreq) + throws Exception { + int count = commitFreq; + try (Directory indexDirectory = FSDirectory.open(indexDirPath); + IndexWriter indexWriter = new IndexWriter(indexDirectory, config)) { + for (int i = 0; i < numDocs; i++) { + Document document = new Document(); + document.add(new StringField(idField, Integer.toString(i), Field.Store.YES)); + document.add(new KnnFloatVectorField(vectorField, dataset[i], EUCLIDEAN)); + indexWriter.addDocument(document); + count -= 1; + if (count == 0) { + indexWriter.commit(); + count = commitFreq; + } + } + } + } + + public static void search( + Path indexDirPath, + String vectorField, + float[] queryVector, + int topK, + KnnFloatVectorQuery query) + throws IOException { + try (Directory indexDirectory = FSDirectory.open(indexDirPath); + DirectoryReader reader = DirectoryReader.open(indexDirectory)) { + IndexSearcher searcher = new IndexSearcher(reader); + searcher.search(query, topK); + } + } + + public static void cleanup(Path indexDirPath) throws IOException { + File indexDirPathFile = indexDirPath.toFile(); + if (indexDirPathFile.exists() && indexDirPathFile.isDirectory()) { + FileUtils.deleteDirectory(indexDirPathFile); + } + } + + public static float[][] generateDataset(Random random, int size, int dimensions) { + float[][] dataset = new float[size][dimensions]; + for (int i = 0; i < size; i++) { + for (int j = 0; j < dimensions; j++) { + dataset[i][j] = random.nextFloat() * 100; + } + } + return dataset; + } + + public static float[] generateRandomVector(int dimensions, Random random) { + float[] vector = new float[dimensions]; + for (int i = 0; i < dimensions; i++) { + vector[i] = random.nextFloat() * 100; + } + return vector; + } + + public static float[][] generateQueries(Random random, int dimensions, int numQueries) { + float[][] queries = new float[numQueries][dimensions]; + for (int i = 0; i < numQueries; i++) { + for (int j = 0; j < dimensions; j++) { + queries[i][j] = random.nextFloat() * 100; + } + } + return queries; + } +} diff --git a/java/cuvs-lucene/ci/release/update-version.sh b/java/cuvs-lucene/ci/release/update-version.sh index 404010440e..714d92b84f 100755 --- a/java/cuvs-lucene/ci/release/update-version.sh +++ b/java/cuvs-lucene/ci/release/update-version.sh @@ -105,7 +105,8 @@ echo "${RAPIDS_BRANCH_NAME}" > RAPIDS_BRANCH # Update Java version NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}" sed_runner "s/VERSION=\".*\"/VERSION=\"${NEXT_FULL_JAVA_TAG}\"/g" build.sh -for FILE in pom.xml examples/pom.xml; do + +for FILE in pom.xml examples/pom.xml bench/pom.xml; do sed_runner "/.*/s//${NEXT_FULL_JAVA_TAG}<\/version>/g" "${FILE}" done diff --git a/java/cuvs-lucene/src/test/resources/META-INF/services/org.apache.lucene.codecs.Codec b/java/cuvs-lucene/src/test/resources/META-INF/services/org.apache.lucene.codecs.Codec deleted file mode 100644 index 9865a33d13..0000000000 --- a/java/cuvs-lucene/src/test/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ /dev/null @@ -1,2 +0,0 @@ -com.nvidia.cuvs.lucene.Lucene101AcceleratedHNSWCodec -com.nvidia.cuvs.lucene.CuVS2510GPUSearchCodec From 9a6567ba50b488dd9bfa41dacd912534f3b7e8c4 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 6 Jan 2026 12:03:11 -0500 Subject: [PATCH 054/105] Cleanup FilterCuVSProvider (NVIDIA/cuvs-lucene#77) Fixes NVIDIA/cuvs-lucene#76 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/77 --- .../nvidia/cuvs/lucene/FilterCuVSProvider.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index 6cdb656b5c..d6b8a0d5c0 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -9,18 +9,22 @@ import com.nvidia.cuvs.CagraIndexParams; import com.nvidia.cuvs.CagraIndexParams.CuvsDistanceType; import com.nvidia.cuvs.CagraIndexParams.HnswHeuristicType; +import com.nvidia.cuvs.CuVSDeviceMatrix; +import com.nvidia.cuvs.CuVSHostMatrix; import com.nvidia.cuvs.CuVSMatrix; import com.nvidia.cuvs.CuVSMatrix.Builder; import com.nvidia.cuvs.CuVSMatrix.DataType; import com.nvidia.cuvs.CuVSResources; +import com.nvidia.cuvs.GPUInfoProvider; import com.nvidia.cuvs.HnswIndex; import com.nvidia.cuvs.HnswIndexParams; +import com.nvidia.cuvs.TieredIndex; import com.nvidia.cuvs.spi.CuVSProvider; import java.lang.invoke.MethodHandle; import java.nio.file.Path; import java.util.logging.Level; -/*package-private*/ class FilterCuVSProvider implements CuVSProvider { +class FilterCuVSProvider implements CuVSProvider { private final CuVSProvider delegate; @@ -62,29 +66,29 @@ public CagraIndex mergeCagraIndexes(CagraIndex[] arg0) throws Throwable { } @Override - public com.nvidia.cuvs.GPUInfoProvider gpuInfoProvider() { + public GPUInfoProvider gpuInfoProvider() { return delegate.gpuInfoProvider(); } @Override - public Builder newHostMatrixBuilder(long rows, long cols, DataType dataType) { + public Builder newHostMatrixBuilder(long rows, long cols, DataType dataType) { return delegate.newHostMatrixBuilder(rows, cols, dataType); } @Override - public Builder newHostMatrixBuilder( + public Builder newHostMatrixBuilder( long rows, long cols, int maxRows, int maxCols, DataType dataType) { return delegate.newHostMatrixBuilder(rows, cols, maxRows, maxCols, dataType); } @Override - public Builder newDeviceMatrixBuilder( + public Builder newDeviceMatrixBuilder( CuVSResources resources, long rows, long cols, DataType dataType) { return delegate.newDeviceMatrixBuilder(resources, rows, cols, dataType); } @Override - public Builder newDeviceMatrixBuilder( + public Builder newDeviceMatrixBuilder( CuVSResources resources, long rows, long cols, int maxRows, int maxCols, DataType dataType) { return delegate.newDeviceMatrixBuilder(resources, rows, cols, maxRows, maxCols, dataType); } @@ -115,7 +119,7 @@ public CuVSMatrix newMatrixFromArray(byte[][] vectors) { } @Override - public com.nvidia.cuvs.TieredIndex.Builder newTieredIndexBuilder(CuVSResources cuVSResources) + public TieredIndex.Builder newTieredIndexBuilder(CuVSResources cuVSResources) throws UnsupportedOperationException { return delegate.newTieredIndexBuilder(cuVSResources); } From fb17889f148e4ca70dc6b6b3e805df939146b4fd Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 6 Jan 2026 18:08:53 -0500 Subject: [PATCH 055/105] Fix spotless plugin configuration for examples (NVIDIA/cuvs-lucene#88) Fixes the spotless plugin configuration - remove license header enforcement, part of NVIDIA/cuvs-lucene#82, for the examples maven sub-project. Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/88 --- java/cuvs-lucene/examples/pom.xml | 3 --- 1 file changed, 3 deletions(-) diff --git a/java/cuvs-lucene/examples/pom.xml b/java/cuvs-lucene/examples/pom.xml index ecdb5b4fc5..f92332cebb 100644 --- a/java/cuvs-lucene/examples/pom.xml +++ b/java/cuvs-lucene/examples/pom.xml @@ -80,9 +80,6 @@ true false - - ../license-header.txt - From 8bb8171933b53e54e19c90612d31bbdb959bbdc1 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 8 Jan 2026 13:07:46 -0600 Subject: [PATCH 056/105] build and test against CUDA 13.1.0 (NVIDIA/cuvs-lucene#89) Contributes to https://github.com/rapidsai/build-planning/issues/236 Tests that CI here will work with the changes from https://github.com/rapidsai/shared-workflows/pull/483, switches CUDA 13 builds to CUDA 13.1.0 and adds some CUDA 13.1.0 test jobs. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cuvs-lucene/pull/89 --- java/cuvs-lucene/.github/workflows/build.yaml | 4 ++-- java/cuvs-lucene/.github/workflows/pr.yaml | 10 +++++----- java/cuvs-lucene/.github/workflows/test.yaml | 4 ++-- ...rch-aarch64.yaml => all_cuda-131_arch-aarch64.yaml} | 4 ++-- ..._arch-x86_64.yaml => all_cuda-131_arch-x86_64.yaml} | 4 ++-- java/cuvs-lucene/dependencies.yaml | 8 ++++++-- java/cuvs-lucene/examples/README.md | 2 +- 7 files changed, 20 insertions(+), 16 deletions(-) rename java/cuvs-lucene/conda/environments/{all_cuda-130_arch-aarch64.yaml => all_cuda-131_arch-aarch64.yaml} (88%) rename java/cuvs-lucene/conda/environments/{all_cuda-130_arch-x86_64.yaml => all_cuda-131_arch-x86_64.yaml} (88%) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index e7b7d90666..4528f2d00f 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -34,7 +34,7 @@ concurrency: jobs: java-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.1.0 # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: @@ -42,7 +42,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.0.2' + - '13.1.0' with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 7d7ac7dff2..9c7a935df4 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda-13.1.0 if: always() with: needs: ${{ toJSON(needs) }} @@ -42,7 +42,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@cuda-13.1.0 with: files_yaml: | test_java: @@ -52,14 +52,14 @@ jobs: checks: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda-13.1.0 with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" conda-java-tests: needs: [changed-files, checks] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.1.0 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. @@ -68,7 +68,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.0.2' + - '13.1.0' with: build_type: pull-request node_type: "gpu-l4-latest-1" diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index c6019f2de9..16417a21a4 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -25,13 +25,13 @@ on: jobs: conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.1.0 strategy: fail-fast: false matrix: cuda_version: - '12.9.1' - - '13.0.2' + - '13.1.0' with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml similarity index 88% rename from java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml rename to java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml index 489f899a89..2c73e35bb0 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -8,7 +8,7 @@ dependencies: - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api -- cuda-version=13.0 +- cuda-version=13.1 - libcublas-dev - libcurand-dev - libcusolver-dev @@ -17,4 +17,4 @@ dependencies: - maven - ninja - openjdk=22.* -name: all_cuda-130_arch-aarch64 +name: all_cuda-131_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml similarity index 88% rename from java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml rename to java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml index 71cfa010a2..bced33f047 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -8,7 +8,7 @@ dependencies: - cuda-cudart-dev - cuda-nvtx-dev - cuda-profiler-api -- cuda-version=13.0 +- cuda-version=13.1 - libcublas-dev - libcurand-dev - libcusolver-dev @@ -17,4 +17,4 @@ dependencies: - maven - ninja - openjdk=22.* -name: all_cuda-130_arch-x86_64 +name: all_cuda-131_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index f0cb592e08..852e75ff76 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 # Dependency list for https://github.com/rapidsai/dependency-file-generator @@ -6,7 +6,7 @@ files: all: output: conda matrix: - cuda: ["12.9", "13.0"] + cuda: ["12.9", "13.1"] arch: [x86_64, aarch64] includes: - cuda @@ -59,6 +59,10 @@ dependencies: cuda: "13.0" packages: - cuda-version=13.0 + - matrix: + cuda: "13.1" + packages: + - cuda-version=13.1 cuda: common: - output_types: [conda] diff --git a/java/cuvs-lucene/examples/README.md b/java/cuvs-lucene/examples/README.md index acebe9e252..7a38e7b68d 100644 --- a/java/cuvs-lucene/examples/README.md +++ b/java/cuvs-lucene/examples/README.md @@ -16,7 +16,7 @@ cd .. Then do: ```sh -docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.02-cuda13.0.2-ubuntu24.04-py3.13 +docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.02-cuda13.1.0-ubuntu24.04-py3.13 ``` Inside the docker container (and in the `cuvs-lucene's` root directory) do: From 4b3adc84b2ef5180b575dce09ca96efaf2e3d5fe Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Mon, 12 Jan 2026 18:31:06 -0500 Subject: [PATCH 057/105] Fix for result returned was not in topk (NVIDIA/cuvs-lucene#94) Issue: We have noticed a few occurrences of test failures complaining of missing topK values in the results. A deeper level of debugging made me realize that this was happening in specific merge cases due to a silly bug in the merge code path. Essentially, while passing the collection of vectors down the layers, the vectors were getting messed up, causing a corrupt merged segment. Fix: This simple fix corrects the vectors in the collection passed down the layer in the merge code path. Fixes NVIDIA/cuvs-lucene#92 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cuvs-lucene/pull/94 --- .../java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java | 4 ++-- .../cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index 033bbc14b6..7c0174c4ef 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; @@ -526,7 +526,7 @@ private List createListFromMergedVectors(FloatVectorValues mergedVector for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { int ordinal = iter.index(); float[] vector = mergedVectorValues.vectorValue(ordinal); - res.add(vector); + res.add(vector.clone()); } return res; } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index 8528ef4566..1848465e96 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; @@ -737,7 +737,7 @@ private List createListFromMergedVectors(FloatVectorValues mergedVector KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { float[] vector = mergedVectorValues.vectorValue(iter.index()); - vectors.add(vector); + vectors.add(vector.clone()); } return vectors; } From 238f9252360a0e7f93cc2a76137260313f863e9f Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 13 Jan 2026 12:50:07 -0500 Subject: [PATCH 058/105] Setup ThreadLocal based CuVSResources usage (NVIDIA/cuvs-lucene#87) Issue: Currently, the way the instance of `CuVSResources` was set up as one instance of it at the `CuVS2510GPUVectorsFormat` and `Lucene99AcceleratedHNSWVectorsFormat` level as a class-level member, and its instance was used in a trickle-down fashion in the `CuVS2510GPUVectorsReader`, `CuVS2510GPUVectorsWriter`, and `Lucene99AcceleratedHNSWVectorsWriter` which was causing a rare occurence of failure (Stack trace available in NVIDIA/cuvs-lucene#91). Fix: To fix this, I am introducing a ThreadLocal-based mechanism with the new `CuVSResourcesProvider` that gives a better way of providing per-thread copies of `CuVSResources` instances. Fixes NVIDIA/cuvs-lucene#11 Fixes NVIDIA/cuvs-lucene#91 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cuvs-lucene/pull/87 --- .../cuvs/lucene/CuVS2510GPUVectorsFormat.java | 33 ++----- .../cuvs/lucene/CuVS2510GPUVectorsReader.java | 18 ++-- .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 34 ++++--- .../cuvs/lucene/FilterCuVSProvider.java | 6 ++ .../Lucene99AcceleratedHNSWVectorsFormat.java | 48 +--------- .../Lucene99AcceleratedHNSWVectorsWriter.java | 24 +++-- .../ThreadLocalCuVSResourcesProvider.java | 91 +++++++++++++++++++ .../TestAcceleratedHNSWDeletedDocuments.java | 5 +- ...TestCagraToHnswSerializationAndSearch.java | 5 +- ...ializationAndSearchWithFallbackWriter.java | 10 +- ...stCuVSAcceleratedHNSWDeletedDocuments.java | 5 +- .../lucene/TestCuVSAcceleratedHNSWGaps.java | 9 +- .../cuvs/lucene/TestCuVSDeletedDocuments.java | 3 +- .../com/nvidia/cuvs/lucene/TestCuVSGaps.java | 7 +- .../TestCuVSRandomizedHNSWVectorSearch.java | 7 +- .../TestCuVSRandomizedVectorSearch.java | 5 +- .../cuvs/lucene/TestCuVSVectorsFormat.java | 5 +- ...tLucene99AcceleratedHNSWVectorsFormat.java | 5 +- .../com/nvidia/cuvs/lucene/TestMerge.java | 5 +- 19 files changed, 192 insertions(+), 133 deletions(-) create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java index 99639b189a..3048867090 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java @@ -1,12 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; -import static com.nvidia.cuvs.lucene.Utils.cuVSResourcesOrNull; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.assertIsSupported; -import com.nvidia.cuvs.CuVSResources; import com.nvidia.cuvs.LibraryException; import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; import java.io.IOException; @@ -41,7 +40,6 @@ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { static final int DEFAULT_GRAPH_DEGREE = 64; static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; - static CuVSResources resources = cuVSResourcesOrNull(); static final LuceneProvider LUCENE_PROVIDER; static final FlatVectorsFormat FLAT_VECTORS_FORMAT; @@ -98,10 +96,10 @@ public CuVS2510GPUVectorsFormat( */ @Override public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { - checkSupported(); + assertIsSupported(); var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); return new CuVS2510GPUVectorsWriter( - state, cuvsWriterThreads, intGraphDegree, graphDegree, indexType, resources, flatWriter); + state, cuvsWriterThreads, intGraphDegree, graphDegree, indexType, flatWriter); } /** @@ -109,8 +107,8 @@ public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOE */ @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { - checkSupported(); - return new CuVS2510GPUVectorsReader(state, resources, FLAT_VECTORS_FORMAT.fieldsReader(state)); + assertIsSupported(); + return new CuVS2510GPUVectorsReader(state, FLAT_VECTORS_FORMAT.fieldsReader(state)); } /** @@ -130,26 +128,7 @@ public String toString() { sb.append("(cuvsWriterThreads=").append(cuvsWriterThreads); sb.append("intGraphDegree=").append(intGraphDegree); sb.append("graphDegree=").append(graphDegree); - sb.append("resources=").append(resources); sb.append(")"); return sb.toString(); } - - /** - * Tells whether the platform supports cuVS. - * - * @return if cuVS is supported or not - */ - public static boolean supported() { - return resources != null; - } - - /** - * Checks if cuVS is supported and throws {@link UnsupportedOperationException} otherwise. - */ - public static void checkSupported() { - if (!supported()) { - throw new UnsupportedOperationException(); - } - } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java index a016feebd4..7bc29ab4b5 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java @@ -10,6 +10,8 @@ import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_META_CODEC_NAME; import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.VERSION_CURRENT; import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.VERSION_START; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.closeCuVSResourcesInstance; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.getCuVSResourcesInstance; import com.nvidia.cuvs.BruteForceIndex; import com.nvidia.cuvs.BruteForceQuery; @@ -17,7 +19,6 @@ import com.nvidia.cuvs.CagraQuery; import com.nvidia.cuvs.CagraSearchParams; import com.nvidia.cuvs.CuVSMatrix; -import com.nvidia.cuvs.CuVSResources; import java.io.IOException; import java.util.Iterator; import java.util.List; @@ -59,7 +60,6 @@ public class CuVS2510GPUVectorsReader extends KnnVectorsReader { @SuppressWarnings("unused") private static final Logger log = Logger.getLogger(CuVS2510GPUVectorsReader.class.getName()); - private final CuVSResources resources; private final FlatVectorsReader flatVectorsReader; // for reading the raw vectors private final FieldInfos fieldInfos; private final IntObjectHashMap fields; @@ -70,15 +70,12 @@ public class CuVS2510GPUVectorsReader extends KnnVectorsReader { * Initializes the {@link CuVS2510GPUVectorsReader}, checks and loads the index. * * @param state instance of the SegmentReadState - * @param resources instance of the CuVSResources * @param flatReader instance of the FlatVectorsReader * * @throws IOException I/O exception */ - public CuVS2510GPUVectorsReader( - SegmentReadState state, CuVSResources resources, FlatVectorsReader flatReader) + public CuVS2510GPUVectorsReader(SegmentReadState state, FlatVectorsReader flatReader) throws IOException { - this.resources = resources; this.flatVectorsReader = flatReader; this.fieldInfos = state.fieldInfos; this.fields = new IntObjectHashMap<>(); @@ -311,7 +308,7 @@ private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { long off = fieldEntry.cagraIndexOffset(); try (var slice = cuvsIndexInput.slice("cagra index", off, len); var in = new IndexInputInputStream(slice)) { - cagraIndex = CagraIndex.newBuilder(resources).from(in).build(); + cagraIndex = CagraIndex.newBuilder(getCuVSResourcesInstance()).from(in).build(); } } @@ -320,7 +317,7 @@ private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { long off = fieldEntry.bruteForceIndexOffset(); try (var slice = cuvsIndexInput.slice("bf index", off, len); var in = new IndexInputInputStream(slice)) { - bruteForceIndex = BruteForceIndex.newBuilder(resources).from(in).build(); + bruteForceIndex = BruteForceIndex.newBuilder(getCuVSResourcesInstance()).from(in).build(); } } } catch (Throwable t) { @@ -339,6 +336,7 @@ public void close() throws IOException { Stream.of(flatVectorsReader, cuvsIndexInput), stream(cuvsIndices.values().iterator()).map(cursor -> cursor.value)); IOUtils.close(closeableStream::iterator); + closeCuVSResourcesInstance(); } static Stream stream(Iterator iterator) { @@ -443,7 +441,7 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits } var query = - new CagraQuery.Builder(resources) + new CagraQuery.Builder(getCuVSResourcesInstance()) .withTopK(topK) .withSearchParams(searchParams) .withQueryVectors(CuVSMatrix.ofArray(new float[][] {target})) @@ -463,7 +461,7 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits BruteForceIndex bruteforceIndex = cuvsIndex.getBruteforceIndex(); assert bruteforceIndex != null; var queryBuilder = - new BruteForceQuery.Builder(resources) + new BruteForceQuery.Builder(getCuVSResourcesInstance()) .withQueryVectors(new float[][] {target}) .withTopK(topK); BruteForceQuery query = queryBuilder.build(); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index 7c0174c4ef..b2a94a845a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -9,6 +9,8 @@ import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_META_CODEC_EXT; import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.CUVS_META_CODEC_NAME; import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.VERSION_CURRENT; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.closeCuVSResourcesInstance; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.getCuVSResourcesInstance; import static org.apache.lucene.index.VectorEncoding.FLOAT32; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; @@ -19,7 +21,6 @@ import com.nvidia.cuvs.CagraIndexParams; import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.CuVSMatrix; -import com.nvidia.cuvs.CuVSResources; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; @@ -77,11 +78,8 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { private final int cuvsWriterThreads; private final int intGraphDegree; private final int graphDegree; - - private final CuVSResources resources; private final IndexType indexType; - - private final FlatVectorsWriter flatVectorsWriter; // for writing the raw vectors + private final FlatVectorsWriter flatVectorsWriter; private final List fields = new ArrayList<>(); private IndexOutput meta = null, cuvsIndex = null; private final InfoStream infoStream; @@ -143,7 +141,6 @@ public boolean bruteForce() { * @param intGraphDegree the intermediate graph degree for building the CAGRA index * @param graphDegree the graph degree for building the CAGRA index * @param indexType the IndexType - * @param resources instance of the CuVSResources * @param flatVectorsWriter instance of FlatVectorsWriter * * @throws IOException I/O exceptions @@ -154,7 +151,6 @@ public CuVS2510GPUVectorsWriter( int intGraphDegree, int graphDegree, IndexType indexType, - CuVSResources resources, FlatVectorsWriter flatVectorsWriter) throws IOException { super(); @@ -162,7 +158,6 @@ public CuVS2510GPUVectorsWriter( this.cuvsWriterThreads = cuvsWriterThreads; this.intGraphDegree = intGraphDegree; this.graphDegree = graphDegree; - this.resources = resources; this.flatVectorsWriter = flatVectorsWriter; this.infoStream = state.infoStream; @@ -291,7 +286,8 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro try { var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); CuVSMatrix dataset = - Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension(), resources); + Utils.createFloatMatrix( + vectors, fieldInfo.getVectorDimension(), getCuVSResourcesInstance()); writeCagraIndex(cagraIndexOutputStream, dataset); } catch (Throwable t) { Utils.handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); @@ -305,7 +301,8 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro if (indexType.bruteForce()) { var bruteForceIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); CuVSMatrix dataset = - Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension(), resources); + Utils.createFloatMatrix( + vectors, fieldInfo.getVectorDimension(), getCuVSResourcesInstance()); writeBruteForceIndex(bruteForceIndexOutputStream, dataset); bruteForceIndexLength = cuvsIndex.getFilePointer() - bruteForceIndexOffset; } @@ -336,10 +333,14 @@ private void writeCagraIndex(OutputStream os, CuVSMatrix dataset) throws Throwab CagraIndexParams params = cagraIndexParams((int) dataset.size()); long startTime = System.nanoTime(); CagraIndex index = - CagraIndex.newBuilder(resources).withDataset(dataset).withIndexParams(params).build(); + CagraIndex.newBuilder(getCuVSResourcesInstance()) + .withDataset(dataset) + .withIndexParams(params) + .build(); long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - startTime); info("Cagra index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); - Path tmpFile = Files.createTempFile(resources.tempDirectory(), "tmpindex", "cag"); + Path tmpFile = + Files.createTempFile(getCuVSResourcesInstance().tempDirectory(), "tmpindex", "cag"); index.serialize(os, tmpFile); index.close(); } @@ -358,7 +359,10 @@ private void writeBruteForceIndex(OutputStream os, CuVSMatrix dataset) throws Th .build(); long startTime = System.nanoTime(); var index = - BruteForceIndex.newBuilder(resources).withIndexParams(params).withDataset(dataset).build(); + BruteForceIndex.newBuilder(getCuVSResourcesInstance()) + .withIndexParams(params) + .withDataset(dataset) + .build(); long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - startTime); info("bf index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); index.serialize(os); @@ -583,7 +587,8 @@ private void writeMergedCagraIndex(FieldInfo fieldInfo, CagraIndex mergedIndex, var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); // Serialize the merged index - Path tmpFile = Files.createTempFile(resources.tempDirectory(), "mergedindex", "cag"); + Path tmpFile = + Files.createTempFile(getCuVSResourcesInstance().tempDirectory(), "mergedindex", "cag"); mergedIndex.serialize(cagraIndexOutputStream, tmpFile); long cagraIndexLength = cuvsIndex.getFilePointer() - cagraIndexOffset; @@ -656,6 +661,7 @@ public void finish() throws IOException { @Override public void close() throws IOException { IOUtils.close(meta, cuvsIndex, flatVectorsWriter); + closeCuVSResourcesInstance(); } /** diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index d6b8a0d5c0..07e64fb1ff 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -159,4 +159,10 @@ public void enableRMMPooledMemory(int arg0, int arg1) { public void resetRMMPooledMemory() { delegate.resetRMMPooledMemory(); } + + @Override + public HnswIndex hnswIndexBuild(CuVSResources arg0, HnswIndexParams arg1, CuVSMatrix arg2) + throws Throwable { + return delegate.hnswIndexBuild(arg0, arg1, arg2); + } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java index 5128ce7ff2..b8b26b33e7 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java @@ -1,12 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; -import static com.nvidia.cuvs.lucene.Utils.cuVSResourcesOrNull; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; -import com.nvidia.cuvs.CuVSResources; import com.nvidia.cuvs.LibraryException; import java.io.IOException; import java.util.logging.Level; @@ -40,8 +39,6 @@ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { static final String HNSW_INDEX_EXT = "vex"; static final LuceneProvider LUCENE_PROVIDER; - private static CuVSResources resources = cuVSResourcesOrNull(); - private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; private static final Integer MAX_CONN; private static final Integer BEAM_WIDTH; @@ -115,10 +112,10 @@ public Lucene99AcceleratedHNSWVectorsFormat( @Override public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); - if (supported()) { + if (isSupported()) { log.log(Level.FINE, "cuVS is supported so using the Lucene99AcceleratedHNSWVectorsWriter"); return new Lucene99AcceleratedHNSWVectorsWriter( - state, cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, resources, flatWriter); + state, cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, flatWriter); } else { log.log( Level.WARNING, @@ -166,44 +163,7 @@ public String toString() { sb.append("intGraphDegree=").append(intGraphDegree); sb.append("graphDegree=").append(graphDegree); sb.append("hnswLayers=").append(hnswLayers); - sb.append("resources=").append(resources); sb.append(")"); return sb.toString(); } - - /** - * Gets the instance of CuVSResources - * - * @return the instance of CuVSResources - */ - public static CuVSResources getResources() { - return resources; - } - - /** - * Sets the instance of CuVSResources - * - * @param resources the instance of CuVSResources to set - */ - public static void setResources(CuVSResources resources) { - Lucene99AcceleratedHNSWVectorsFormat.resources = resources; - } - - /** - * Tells whether the platform supports cuVS. - * - * @return if cuVS supported or not - */ - public static boolean supported() { - return resources != null; - } - - /** - * Checks if cuVS supported and throws {@link UnsupportedOperationException} otherwise. - */ - public static void checkSupported() { - if (!supported()) { - throw new UnsupportedOperationException(); - } - } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index 1848465e96..0603da6195 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -8,6 +8,8 @@ import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_EXT; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_EXT; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_NAME; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.closeCuVSResourcesInstance; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.getCuVSResourcesInstance; import static org.apache.lucene.index.VectorEncoding.FLOAT32; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; @@ -16,7 +18,6 @@ import com.nvidia.cuvs.CagraIndexParams; import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.CuVSMatrix; -import com.nvidia.cuvs.CuVSResources; import com.nvidia.cuvs.RowView; import java.io.IOException; import java.nio.file.Files; @@ -82,7 +83,6 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { private final int intGraphDegree; private final int graphDegree; private final int hnswLayers; // Number of layers to create in CAGRA->HNSW conversion - private final CuVSResources resources; private final FlatVectorsWriter flatVectorsWriter; // for writing the raw vectors private final List fields = new ArrayList<>(); private final InfoStream infoStream; @@ -110,7 +110,6 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { * @param intGraphDegree the intermediate graph degree to use while building the CAGRA index * @param graphDegree the graph degree to use while building the CAGRA index * @param hnswLayers the number of hnsw layers to construct while building the HNSW graph - * @param resources instance of the {@link com.nvidia.cuvs.CuVSResources} * @param flatVectorsWriter instance of the {@link org.apache.lucene.codecs.hnsw.FlatVectorsWriter} * @throws IOException IOException */ @@ -120,7 +119,6 @@ public Lucene99AcceleratedHNSWVectorsWriter( int intGraphDegree, int graphDegree, int hnswLayers, - CuVSResources resources, FlatVectorsWriter flatVectorsWriter) throws IOException { super(); @@ -128,7 +126,6 @@ public Lucene99AcceleratedHNSWVectorsWriter( this.intGraphDegree = intGraphDegree; this.graphDegree = graphDegree; this.hnswLayers = hnswLayers; - this.resources = resources; this.flatVectorsWriter = flatVectorsWriter; this.infoStream = state.infoStream; @@ -241,7 +238,8 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro try { CuVSMatrix dataset = - Utils.createFloatMatrix(vectors, fieldInfo.getVectorDimension(), resources); + Utils.createFloatMatrix( + vectors, fieldInfo.getVectorDimension(), getCuVSResourcesInstance()); if (dataset.size() < 2) { // Handle single vector case by creating a dummy HNSW graph @@ -253,7 +251,10 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro long startTime = System.nanoTime(); CagraIndexParams params = cagraIndexParams(); CagraIndex cagraIndex = - CagraIndex.newBuilder(resources).withDataset(dataset).withIndexParams(params).build(); + CagraIndex.newBuilder(getCuVSResourcesInstance()) + .withDataset(dataset) + .withIndexParams(params) + .build(); // Get the adjacency list from CAGRA index CuVSMatrix adjacencyListMatrix = cagraIndex.getGraph(); @@ -382,7 +383,10 @@ private CuVSMatrix buildCagraGraphForSubset(float[][] vectors, int[] selectedNod // Build CAGRA index for the subset CagraIndexParams params = cagraIndexParams(); CagraIndex subsetIndex = - CagraIndex.newBuilder(resources).withDataset(subsetDataset).withIndexParams(params).build(); + CagraIndex.newBuilder(getCuVSResourcesInstance()) + .withDataset(subsetDataset) + .withIndexParams(params) + .build(); // Get adjacency list from subset CAGRA index CuVSMatrix cagraGraph = subsetIndex.getGraph(); @@ -792,7 +796,8 @@ private void writeMergedCagraIndex(FieldInfo fieldInfo, CagraIndex mergedIndex, var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); // Serialize the merged index - Path tmpFile = Files.createTempFile(resources.tempDirectory(), "mergedindex", "cag"); + Path tmpFile = + Files.createTempFile(getCuVSResourcesInstance().tempDirectory(), "mergedindex", "cag"); mergedIndex.serialize(cagraIndexOutputStream, tmpFile); // TODO: Path to writeFieldInternal missing. Fix this. @@ -861,6 +866,7 @@ public void finish() throws IOException { @Override public void close() throws IOException { IOUtils.close(cuvsIndex, hnswMeta, hnswVectorIndex, flatVectorsWriter); + closeCuVSResourcesInstance(); } /** diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java new file mode 100644 index 0000000000..9e259e27c1 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java @@ -0,0 +1,91 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nvidia.cuvs.lucene; + +import com.nvidia.cuvs.CuVSResources; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Provides a mechanism to create ThreadLocal based CuVSResource instances. + * + * @since 26.02 + */ +public class ThreadLocalCuVSResourcesProvider { + + private static final Logger log = + Logger.getLogger(ThreadLocalCuVSResourcesProvider.class.getName()); + private static final ThreadLocal cuVSResources; + + static { + cuVSResources = ThreadLocal.withInitial(() -> cuVSResourcesOrNull()); + } + + /** + * Gets an instance of CuVSResources for the accessing thread. + * + * @return an instance of CuVSResources + */ + public static CuVSResources getCuVSResourcesInstance() { + return cuVSResources.get(); + } + + /** + * Sets the instance of CuVSResources + * + * @param resources the instance of CuVSResources to set + */ + public static void setCuVSResourcesInstance(CuVSResources resources) { + cuVSResources.set(resources); + } + + private static CuVSResources cuVSResourcesOrNull() { + try { + return CuVSResources.create(); + } catch (UnsupportedOperationException uoe) { + log.log( + Level.WARNING, + "cuVS is not supported on this platform or java version: " + uoe.getMessage()); + } catch (Throwable t) { + if (t instanceof ExceptionInInitializerError ex) { + t = ex.getCause(); + } + log.log(Level.WARNING, "Exception occurred during creation of cuVS resources. " + t); + } + return null; + } + + /** + * Attempts to close the thread's {@link CuVSResources} instance. + */ + public static void closeCuVSResourcesInstance() { + CuVSResources r = cuVSResources.get(); + if (r != null) { + r.close(); + } + cuVSResources.remove(); + } + + /** + * Checks if cuVS is supported and throws {@link UnsupportedOperationException} otherwise. + * + * @throws UnsupportedOperationException + */ + public static void assertIsSupported() throws UnsupportedOperationException { + if (cuVSResources.get() == null) { + throw new UnsupportedOperationException("cuVS is not supported"); + } + } + + /** + * Checks if cuVS is supported. + * + * @return true if cuVS is supported else false + */ + public static boolean isSupported() { + return cuVSResources.get() != null; + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java index c1814faeed..d164af5560 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java @@ -1,11 +1,12 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; import static com.nvidia.cuvs.lucene.TestUtils.generateRandomVector; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import java.io.IOException; import java.util.ArrayList; @@ -53,7 +54,7 @@ public class TestAcceleratedHNSWDeletedDocuments extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { - assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); random = random(); } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java index 335163409b..1fa566972b 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java @@ -1,10 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; import java.io.File; @@ -52,7 +53,7 @@ public class TestCagraToHnswSerializationAndSearch extends LuceneTestCase { @Before public void beforeTest() throws Exception { - assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); // Fixed seed so that we can validate against the same result. random = new Random(222); indexDirPath = Paths.get(UUID.randomUUID().toString()); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java index 746437d698..3d3755f7de 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java @@ -1,10 +1,12 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.setCuVSResourcesInstance; import static com.nvidia.cuvs.lucene.Utils.cuVSResourcesOrNull; import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; @@ -54,9 +56,9 @@ public class TestCagraToHnswSerializationAndSearchWithFallbackWriter extends Luc @BeforeClass public static void beforeClass() throws Exception { - assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); // Set resources to null to simulate that cuVS is not supported. - Lucene99AcceleratedHNSWVectorsFormat.setResources(null); + setCuVSResourcesInstance(null); // Fixed seed so that we can validate against the same result. random = new Random(222); indexDirPath = Paths.get(UUID.randomUUID().toString()); @@ -165,7 +167,7 @@ public void testCagraToHnswSerializationAndSearchWithFallbackWriter() @AfterClass public static void afterClass() throws Exception { // Reset resources for other tests to work - Lucene99AcceleratedHNSWVectorsFormat.setResources(cuVSResourcesOrNull()); + setCuVSResourcesInstance(cuVSResourcesOrNull()); File indexDirPathFile = indexDirPath.toFile(); if (indexDirPathFile.exists() && indexDirPathFile.isDirectory()) { FileUtils.deleteDirectory(indexDirPathFile); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java index ba53c8b000..d3dc866e9a 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java @@ -1,11 +1,12 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; import static com.nvidia.cuvs.lucene.TestUtils.generateRandomVector; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import java.io.IOException; import java.util.ArrayList; @@ -53,7 +54,7 @@ public class TestCuVSAcceleratedHNSWDeletedDocuments extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { - assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); random = random(); } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java index 6cac680426..b0dae38ed1 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java @@ -1,10 +1,11 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import java.io.IOException; import java.util.List; @@ -61,7 +62,7 @@ public class TestCuVSAcceleratedHNSWGaps extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { - assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); directory = newDirectory(); random = random(); @@ -111,7 +112,7 @@ public static void afterClass() throws Exception { @Test public void testVectorSearchWithAlternatingDocuments() throws IOException { - assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); // Use the first vector (from document 0) as query float[] queryVector = dataset[0]; @@ -144,7 +145,7 @@ public void testVectorSearchWithAlternatingDocuments() throws IOException { @Test public void testVectorSearchWithFilterAndAlternatingDocuments() throws IOException { - assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); // Use the first vector (from document 0) as query float[] queryVector = dataset[0]; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java index 3363a61478..de868c7c25 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java @@ -6,6 +6,7 @@ import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; import static com.nvidia.cuvs.lucene.TestUtils.generateRandomVector; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import java.io.IOException; import java.util.ArrayList; @@ -50,7 +51,7 @@ public class TestCuVSDeletedDocuments extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { - assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); random = random(); } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java index 42d3464c06..a3e2f59a6f 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java @@ -5,6 +5,7 @@ package com.nvidia.cuvs.lucene; import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import java.io.IOException; import java.util.List; @@ -59,7 +60,7 @@ public class TestCuVSGaps extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { - assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); directory = newDirectory(); random = random(); @@ -109,7 +110,7 @@ public static void afterClass() throws Exception { @Test public void testVectorSearchWithAlternatingDocuments() throws IOException { - assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); // Use the first vector (from document 0) as query float[] queryVector = dataset[0]; @@ -143,7 +144,7 @@ public void testVectorSearchWithAlternatingDocuments() throws IOException { @Test public void testVectorSearchWithFilterAndAlternatingDocuments() throws IOException { - assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); // Use the first vector (from document 0) as query float[] queryVector = dataset[0]; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java index 385ca8f581..c2e2f2f6ae 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java @@ -1,11 +1,12 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; import static com.nvidia.cuvs.lucene.TestUtils.generateQueries; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import java.io.IOException; import java.util.ArrayList; @@ -61,7 +62,7 @@ public class TestCuVSRandomizedHNSWVectorSearch extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { - assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); directory = newDirectory(); RandomIndexWriter writer = @@ -177,7 +178,7 @@ private static List> generateExpectedResults( @Test public void testVectorSearchWithFilter() throws IOException { - assumeTrue("cuVS not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); Random random = random(); int topK = Math.min(random.nextInt(TOP_K_LIMIT) + 1, dataset.length); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java index 9e42528327..3e77e2edf2 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java @@ -6,6 +6,7 @@ import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; import static com.nvidia.cuvs.lucene.TestUtils.generateQueries; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import java.io.IOException; import java.util.ArrayList; @@ -58,7 +59,7 @@ public class TestCuVSRandomizedVectorSearch extends LuceneTestCase { @BeforeClass public static void beforeClass() throws Exception { - assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); directory = newDirectory(); RandomIndexWriter writer = @@ -175,7 +176,7 @@ private static List> generateExpectedResults( @Test public void testVectorSearchWithFilter() throws IOException { - assumeTrue("cuVS not supported", CuVS2510GPUVectorsFormat.supported()); + assumeTrue("cuVS not supported", isSupported()); Random random = random(); int topK = Math.min(random.nextInt(TOP_K_LIMIT) + 1, dataset.length); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java index 1b3de41d65..1c75cb2318 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java @@ -1,9 +1,10 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; import java.util.List; @@ -30,7 +31,7 @@ public class TestCuVSVectorsFormat extends BaseKnnVectorsFormatTestCase { @BeforeClass public static void beforeClass() { - assumeTrue("cuVS is not supported", CuVS2510GPUVectorsFormat.supported()); + assumeTrue("cuVS is not supported", isSupported()); } @Override diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java index f834d56471..2428fc5053 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java @@ -1,9 +1,10 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; import java.util.List; @@ -30,7 +31,7 @@ public class TestLucene99AcceleratedHNSWVectorsFormat extends BaseKnnVectorsForm @BeforeClass public static void beforeClass() { - assumeTrue("cuVS is not supported", Lucene99AcceleratedHNSWVectorsFormat.supported()); + assumeTrue("cuVS is not supported", isSupported()); } @Override diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java index 9baea0e629..cdba23b170 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java @@ -1,9 +1,10 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import static org.apache.lucene.tests.util.TestUtil.alwaysKnnVectorsFormat; import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; @@ -60,7 +61,7 @@ public class TestMerge extends LuceneTestCase { @BeforeClass public static void beforeClass() { - assumeTrue("cuVS is not supported", CuVS2510GPUVectorsFormat.supported()); + assumeTrue("cuVS is not supported", isSupported()); } private Directory directory; From 2546fc25e1178ac7e7285736cbdc0cf166f2efd0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 14 Jan 2026 10:19:02 -0600 Subject: [PATCH 059/105] Use main shared-workflows branch (NVIDIA/cuvs-lucene#96) Contributes to https://github.com/rapidsai/build-planning/issues/236 Updates branch for GitHub Actions workflows from `@cuda-13.1.0` to `@main` now that https://github.com/rapidsai/shared-workflows/pull/483 is merged. --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 8 ++++---- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 4528f2d00f..8e21627231 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -34,7 +34,7 @@ concurrency: jobs: java-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.1.0 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 9c7a935df4..988fbfd698 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda-13.1.0 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main if: always() with: needs: ${{ toJSON(needs) }} @@ -42,7 +42,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@cuda-13.1.0 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main with: files_yaml: | test_java: @@ -52,14 +52,14 @@ jobs: checks: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda-13.1.0 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" conda-java-tests: needs: [changed-files, checks] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.1.0 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 16417a21a4..42e7d8af72 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -25,7 +25,7 @@ on: jobs: conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.1.0 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main strategy: fail-fast: false matrix: From 9b87665f06b4471418ada4a1b4aa98d595b8732a Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Fri, 16 Jan 2026 10:11:21 -0600 Subject: [PATCH 060/105] Revert "Prepare release/26.02" This reverts commit 7a0f494e0808bd586f68feb148ae47bb88d7afb3. --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 8 ++++---- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/RAPIDS_BRANCH | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 1d6b556e0c..8e21627231 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -34,7 +34,7 @@ concurrency: jobs: java-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 9b15c22c75..988fbfd698 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.02 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main if: always() with: needs: ${{ toJSON(needs) }} @@ -42,7 +42,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@release/26.02 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main with: files_yaml: | test_java: @@ -52,14 +52,14 @@ jobs: checks: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@release/26.02 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" conda-java-tests: needs: [changed-files, checks] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 12573df4fa..42e7d8af72 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -25,7 +25,7 @@ on: jobs: conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.02 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main strategy: fail-fast: false matrix: diff --git a/java/cuvs-lucene/RAPIDS_BRANCH b/java/cuvs-lucene/RAPIDS_BRANCH index 7503d7b54e..ba2906d066 100644 --- a/java/cuvs-lucene/RAPIDS_BRANCH +++ b/java/cuvs-lucene/RAPIDS_BRANCH @@ -1 +1 @@ -release/26.02 +main From 7a0f494e0808bd586f68feb148ae47bb88d7afb3 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Fri, 16 Jan 2026 10:11:21 -0600 Subject: [PATCH 061/105] Prepare release/26.02 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 8 ++++---- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/RAPIDS_BRANCH | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 8e21627231..1d6b556e0c 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -34,7 +34,7 @@ concurrency: jobs: java-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.02 # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 988fbfd698..9b15c22c75 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.02 if: always() with: needs: ${{ toJSON(needs) }} @@ -42,7 +42,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@release/26.02 with: files_yaml: | test_java: @@ -52,14 +52,14 @@ jobs: checks: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@release/26.02 with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" conda-java-tests: needs: [changed-files, checks] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.02 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 42e7d8af72..12573df4fa 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -25,7 +25,7 @@ on: jobs: conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.02 strategy: fail-fast: false matrix: diff --git a/java/cuvs-lucene/RAPIDS_BRANCH b/java/cuvs-lucene/RAPIDS_BRANCH index ba2906d066..7503d7b54e 100644 --- a/java/cuvs-lucene/RAPIDS_BRANCH +++ b/java/cuvs-lucene/RAPIDS_BRANCH @@ -1 +1 @@ -main +release/26.02 From 2131359610446d304287594de1f9111c1d12901c Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Fri, 16 Jan 2026 10:12:38 -0600 Subject: [PATCH 062/105] Update to 26.04 (NVIDIA/cuvs-lucene#98) This PR updates the repository to version 26.04. This is part of the 26.02 release burndown process. --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 2 +- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/VERSION | 2 +- java/cuvs-lucene/bench/pom.xml | 2 +- java/cuvs-lucene/build.sh | 2 +- .../conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- .../conda/environments/all_cuda-131_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-131_arch-x86_64.yaml | 2 +- java/cuvs-lucene/dependencies.yaml | 2 +- java/cuvs-lucene/examples/README.md | 6 +++--- java/cuvs-lucene/examples/pom.xml | 4 ++-- java/cuvs-lucene/pom.xml | 4 ++-- 14 files changed, 18 insertions(+), 18 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 8e21627231..a48616c421 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -48,7 +48,7 @@ jobs: branch: ${{ inputs.branch }} arch: "amd64" date: ${{ inputs.date }} - container_image: "rapidsai/ci-conda:26.02-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.04-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/build_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 988fbfd698..5a004c0f53 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -73,7 +73,7 @@ jobs: build_type: pull-request node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:26.02-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.04-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 42e7d8af72..ec1f0dfdcb 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -39,5 +39,5 @@ jobs: sha: ${{ inputs.sha }} node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:26.02-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.04-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" diff --git a/java/cuvs-lucene/VERSION b/java/cuvs-lucene/VERSION index 5c33046aca..0bd0e8a95b 100644 --- a/java/cuvs-lucene/VERSION +++ b/java/cuvs-lucene/VERSION @@ -1 +1 @@ -26.02.00 +26.04.00 diff --git a/java/cuvs-lucene/bench/pom.xml b/java/cuvs-lucene/bench/pom.xml index e864793c52..dbc1073b50 100644 --- a/java/cuvs-lucene/bench/pom.xml +++ b/java/cuvs-lucene/bench/pom.xml @@ -31,7 +31,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 26.02.0 + 26.04.0 commons-io diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index fc3c492278..92d9f321df 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -8,7 +8,7 @@ set -e -u -o pipefail ARGS="$*" NUMARGS=$# -VERSION="26.02.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked +VERSION="26.04.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked GROUP_ID="com.nvidia.cuvs.lucene" function hasArg { diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index b66afbc36d..737ec979fb 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -13,7 +13,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.02.* +- libcuvs==26.04.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index ff0c06d71a..9cc0309525 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -13,7 +13,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.02.* +- libcuvs==26.04.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml index 2c73e35bb0..f5d58376fa 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -13,7 +13,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.02.* +- libcuvs==26.04.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml index bced33f047..3180629fd8 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -13,7 +13,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.02.* +- libcuvs==26.04.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index 852e75ff76..2d04b5ff56 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -74,7 +74,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev - - libcuvs==26.02.* + - libcuvs==26.04.* java: common: - output_types: conda diff --git a/java/cuvs-lucene/examples/README.md b/java/cuvs-lucene/examples/README.md index 7a38e7b68d..2e6da591ac 100644 --- a/java/cuvs-lucene/examples/README.md +++ b/java/cuvs-lucene/examples/README.md @@ -16,7 +16,7 @@ cd .. Then do: ```sh -docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.02-cuda13.1.0-ubuntu24.04-py3.13 +docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.04-cuda13.1.0-ubuntu24.04-py3.13 ``` Inside the docker container (and in the `cuvs-lucene's` root directory) do: @@ -26,10 +26,10 @@ Inside the docker container (and in the `cuvs-lucene's` root directory) do: To run Accelerated HNSW example do: ```sh -mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.02.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.AcceleratedHnswExample +mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.04.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.AcceleratedHnswExample ``` To run the Index and Search on GPU example do: ```sh -mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.02.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.IndexAndSearchonGPUExample +mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.04.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.IndexAndSearchonGPUExample ``` diff --git a/java/cuvs-lucene/examples/pom.xml b/java/cuvs-lucene/examples/pom.xml index f92332cebb..768a9a8760 100644 --- a/java/cuvs-lucene/examples/pom.xml +++ b/java/cuvs-lucene/examples/pom.xml @@ -11,7 +11,7 @@ com.nvidia.cuvs.lucene.examples examples - 26.02.0 + 26.04.0 examples @@ -51,7 +51,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 26.02.0 + 26.04.0 diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 010e9b67b3..0011497217 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -12,7 +12,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 26.02.0 + 26.04.0 cuvs-lucene jar @@ -75,7 +75,7 @@ com.nvidia.cuvs cuvs-java - 26.02.0 + 26.04.0 From db06f9d9e94f1b4d33cb1be77cd0b4a09abd8c8c Mon Sep 17 00:00:00 2001 From: Puneet Ahuja <167976912+punAhuja@users.noreply.github.com> Date: Fri, 23 Jan 2026 04:34:28 +0530 Subject: [PATCH 063/105] Add quantization vector formats, writers, and codecs (NVIDIA/cuvs-lucene#57) This PR introduces the following Codecs, Formats, and Writers: - `Lucene101AcceleratedHNSWBinaryQuantizedCodec` - `Lucene99AcceleratedHNSWBinaryQuantizedVectorsFormat` - `Lucene99AcceleratedHNSWBinaryQuantizedVectorsWriter` - `Lucene101AcceleratedHNSWScalarQuantizedCodec` - `Lucene99AcceleratedHNSWScalarQuantizedVectorsFormat` - `Lucene99AcceleratedHNSWScalarQuantizedVectorsWriter` This enables us to build and search larger indexes using the binary and scalar quantization approaches. Fixes NVIDIA/cuvs-lucene#97 Authors: - Puneet Ahuja (https://github.com/punAhuja) - Jake Awe (https://github.com/AyodeAwe) - Vivek Narang (https://github.com/narangvivek10) Approvers: - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cuvs-lucene/pull/57 --- .../cuvs/lucene/AcceleratedHNSWUtils.java | 486 ++++++++++++++++++ ...neAcceleratedHNSWBinaryQuantizedCodec.java | 104 ++++ ...ratedHNSWBinaryQuantizedVectorsFormat.java | 176 +++++++ ...ratedHNSWBinaryQuantizedVectorsWriter.java | 403 +++++++++++++++ ...neAcceleratedHNSWScalarQuantizedCodec.java | 104 ++++ ...ratedHNSWScalarQuantizedVectorsFormat.java | 167 ++++++ ...ratedHNSWScalarQuantizedVectorsWriter.java | 427 +++++++++++++++ .../nvidia/cuvs/lucene/LuceneProvider.java | 119 ++++- .../cuvs/lucene/QuantizedFieldWriter.java | 77 +++ .../java/com/nvidia/cuvs/lucene/Utils.java | 56 +- .../services/org.apache.lucene.codecs.Codec | 4 +- .../org.apache.lucene.codecs.KnnVectorsFormat | 4 +- .../lucene/TestQuantizedVectorsFormats.java | 212 ++++++++ 13 files changed, 2327 insertions(+), 12 deletions(-) create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/QuantizedFieldWriter.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java new file mode 100644 index 0000000000..9f9938d94d --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java @@ -0,0 +1,486 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.getCuVSResourcesInstance; +import static com.nvidia.cuvs.lucene.Utils.createByteMatrixFromArray; + +import com.nvidia.cuvs.CagraIndex; +import com.nvidia.cuvs.CagraIndexParams; +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.CuVSMatrix; +import com.nvidia.cuvs.RowView; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Random; +import java.util.SortedSet; +import java.util.TreeSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.VectorSimilarityFunction; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.hnsw.HnswGraph; +import org.apache.lucene.util.hnsw.HnswGraph.NodesIterator; +import org.apache.lucene.util.hnsw.NeighborArray; +import org.apache.lucene.util.packed.DirectMonotonicWriter; + +public class AcceleratedHNSWUtils { + + public enum QuantizationType { + BINARY, + SCALAR + } + + private static final LuceneProvider LUCENE_PROVIDER; + private static final List VECTOR_SIMILARITY_FUNCTIONS; + + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + VECTOR_SIMILARITY_FUNCTIONS = LUCENE_PROVIDER.getSimilarityFunctions(); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + + /** + * Creates a dummy HNSW graph for a single vector. + * The graph will have 1 level with 1 node and no neighbors. + */ + public static GPUBuiltHnswGraph createSingleVectorHnswGraph(int size, int dimensions) + throws Throwable { + // Create adjacency list for single node with no neighbors + int[][] singleNodeAdjacency = new int[][] {{-1}}; // -1 indicates no neighbors + + // Create CuVSMatrix from the adjacency list + CuVSMatrix adjacencyMatrix = CuVSMatrix.ofArray(singleNodeAdjacency); + + // Create layer data for single-level graph + List layerNodes = new ArrayList<>(); + List layerAdjacencies = new ArrayList<>(); + + // Layer 0: contains all nodes (just the single node) + layerNodes.add(null); // Layer 0 contains all nodes, so we don't need to store node list + layerAdjacencies.add(adjacencyMatrix); + + // Create the single-layer graph + return new GPUBuiltHnswGraph(size, dimensions, layerNodes, layerAdjacencies); + } + + /** + * Creates a multi-layer HNSW graph with dynamic number of layers. + * M = cagraGraphDegree/2 + * Each layer contains 1/M nodes from the previous layer + * Creates layers until the highest layer has ≤ M nodes + */ + public static GPUBuiltHnswGraph createMultiLayerHnswGraph( + FieldInfo fieldInfo, + int size, + int dimensions, + CuVSMatrix adjacencyListMatrix, + List vectors, + int hnswLayers, + int graphDegree, + CagraIndexParams params, + QuantizationType quantization) + throws Throwable { + + // Calculate M as cagraGraphDegree/2 + int M = graphDegree / 2; + + // Store all layers data + List layerNodes = new ArrayList<>(); + List layerAdjacencies = new ArrayList<>(); + + // Layer 0: Use full CAGRA adjacency list + layerNodes.add(null); // Layer 0 contains all nodes, so we don't need to store node list + layerAdjacencies.add(adjacencyListMatrix); + + int currentLayerSize = size; + int layerIndex = 1; + Random random = new Random(); + + while (layerIndex < hnswLayers && currentLayerSize > 1) { + // Calculate size for next layer (1/M of current layer) + int nextLayerSize = Math.max(2, currentLayerSize / M); + // Select nodes for this layer + SortedSet selectedNodesSet = new TreeSet<>(); + + if (layerIndex == 1) { + // Select from all nodes (Layer 0) + while (selectedNodesSet.size() < nextLayerSize) { + selectedNodesSet.add(random.nextInt(size)); + } + } else { + // Select from previous layer nodes + int[] prevLayerNodes = layerNodes.get(layerNodes.size() - 1); + while (selectedNodesSet.size() < nextLayerSize) { + int idx = random.nextInt(prevLayerNodes.length); + selectedNodesSet.add(prevLayerNodes[idx]); + } + } + + // Convert to sorted array + int[] selectedNodes = + selectedNodesSet.stream().mapToInt(Integer::intValue).sorted().toArray(); + + layerNodes.add(selectedNodes); + + // Extract vectors for selected nodes + int bytesPerVector = (dimensions + 7) / 8; + byte[][] selectedVectors = new byte[nextLayerSize][]; + for (int i = 0; i < nextLayerSize; i++) { + selectedVectors[i] = vectors.get(selectedNodes[i]); + } + + // Build CAGRA graph for this layer + layerAdjacencies.add( + buildCagraGraphForSubset( + selectedVectors, selectedNodes, bytesPerVector, params, dimensions, quantization)); + + // Update for next iteration + currentLayerSize = nextLayerSize; + layerIndex++; + + // Use different seed for each layer + random = new Random(new Random().nextLong()); + } + + // Create the multi-layer graph with all layers + return new GPUBuiltHnswGraph(size, dimensions, layerNodes, layerAdjacencies); + } + + /** + * Builds a CAGRA graph for a subset of binary quantized vectors + */ + private static CuVSMatrix buildCagraGraphForSubset( + byte[][] vectors, + int[] selectedNodes, + int bytesPerVector, + CagraIndexParams params, + int dimensions, + QuantizationType quantization) + throws Throwable { + + CuVSMatrix subsetDataset; + + if (quantization == QuantizationType.BINARY) { + subsetDataset = + createByteMatrixFromArray(vectors, bytesPerVector, getCuVSResourcesInstance()); + } else { + subsetDataset = createByteMatrixFromArray(vectors, dimensions, getCuVSResourcesInstance()); + } + + // Build CAGRA index for the subset + CagraIndex subsetIndex = + CagraIndex.newBuilder(getCuVSResourcesInstance()) + .withDataset(subsetDataset) + .withIndexParams(params) + .build(); + + // Get adjacency list from subset CAGRA index + CuVSMatrix cagraGraph = subsetIndex.getGraph(); + + long numNodes = cagraGraph.size(); + long degree = cagraGraph.columns(); + + // Create a re-mapped adjacency list + int[][] remappedAdjacency = new int[(int) numNodes][(int) degree]; + + for (int i = 0; i < numNodes; i++) { + RowView rv = cagraGraph.getRow(i); + for (int j = 0; j < degree && j < rv.size(); j++) { + int subsetIndex1 = rv.getAsInt(j); + // Map subset index to original node ID + if (subsetIndex1 >= 0 && subsetIndex1 < selectedNodes.length) { + remappedAdjacency[i][j] = selectedNodes[subsetIndex1]; + } else { + // Invalid index, use self-reference + remappedAdjacency[i][j] = selectedNodes[i]; + } + } + } + + subsetIndex.close(); + return CuVSMatrix.ofArray(remappedAdjacency); + } + + /** + * Returns a 2D array of offsets (information written while writing the meta info) + * + * @param graph instance of GPUBuiltHnswGraph + * @param vectorIndex instance of IndexOutput + * @return a 2D array of offsets + * @throws IOException I/O Exceptions + */ + public static int[][] writeGraph(GPUBuiltHnswGraph graph, IndexOutput vectorIndex) + throws IOException { + // write vectors' neighbors on each level into the vectorIndex file + int countOnLevel0 = graph.size(); + int[][] offsets = new int[graph.numLevels()][]; + int[] scratch = new int[graph.maxConn() * 2]; + for (int level = 0; level < graph.numLevels(); level++) { + int[] sortedNodes = NodesIterator.getSortedNodes(graph.getNodesOnLevel(level)); + offsets[level] = new int[sortedNodes.length]; + int nodeOffsetId = 0; + + for (int node : sortedNodes) { + // Get node neighbors + NeighborArray neighbors = graph.getNeighbors(level, node); + // Get the size of the neighbor array + int size = neighbors.size(); + // Write size in VInt as the neighbors list is typically small + long offsetStart = vectorIndex.getFilePointer(); + // Get neighbors + int[] nnodes = neighbors.nodes(); + // Sort them + Arrays.sort(nnodes, 0, size); + // Now that we have sorted, do delta encoding to minimize the required bits to store the + // information + int actualSize = 0; + if (size > 0) { + scratch[0] = nnodes[0]; + actualSize = 1; + } + // De-duplication + for (int i = 1; i < size; i++) { + assert nnodes[i] < countOnLevel0 : "node too large: " + nnodes[i] + ">=" + countOnLevel0; + // Sorting step helps here + if (nnodes[i - 1] == nnodes[i]) { + continue; + } + scratch[actualSize++] = nnodes[i] - nnodes[i - 1]; + } + // Write the size after duplicates are removed + vectorIndex.writeVInt(actualSize); + // Write de-duplicated neighbors + for (int i = 0; i < actualSize; i++) { + vectorIndex.writeVInt(scratch[i]); + } + offsets[level][nodeOffsetId++] = + Math.toIntExact(vectorIndex.getFilePointer() - offsetStart); + } + } + // Return offsets (information written while writing the meta info) + return offsets; + } + + /** + * Writes the meta information for the index. + * + * @param vectorIndex instance of IndexOutput + * @param meta instance of IndexOutput + * @param field instance of FieldInfo + * @param vectorIndexOffset vector index offset + * @param vectorIndexLength vector index length + * @param count the count of vectors + * @param graph instance of HnswGraph + * @param graphLevelNodeOffsets graph level node offsets + * @throws IOException I/O Exceptions + */ + public static void writeMeta( + IndexOutput vectorIndex, + IndexOutput meta, + FieldInfo field, + long vectorIndexOffset, + long vectorIndexLength, + int count, + HnswGraph graph, + int[][] graphLevelNodeOffsets, + int graphDegree) + throws IOException { + + meta.writeInt(field.number); + meta.writeInt(field.getVectorEncoding().ordinal()); + meta.writeInt(distFuncToOrd(field.getVectorSimilarityFunction())); + meta.writeVLong(vectorIndexOffset); + meta.writeVLong(vectorIndexLength); + meta.writeVInt(field.getVectorDimension()); + meta.writeInt(count); + meta.writeVInt(graphDegree / 2); // M = cagraGraphDegree/2 + + // write graph nodes on each level + if (graph == null) { + meta.writeVInt(0); + } else { + meta.writeVInt(graph.numLevels()); + long valueCount = 0; + for (int level = 0; level < graph.numLevels(); level++) { + NodesIterator nodesOnLevel = graph.getNodesOnLevel(level); + valueCount += nodesOnLevel.size(); + if (level > 0) { + int[] nol = new int[nodesOnLevel.size()]; + int numberConsumed = nodesOnLevel.consume(nol); + Arrays.sort(nol); + assert numberConsumed == nodesOnLevel.size(); + meta.writeVInt(nol.length); // number of nodes on a level + for (int i = nodesOnLevel.size() - 1; i > 0; --i) { + nol[i] -= nol[i - 1]; + } + for (int n : nol) { + meta.writeVInt(n); + } + } else { + assert nodesOnLevel.size() == count : "Level 0 expects to have all nodes"; + } + } + + long start = vectorIndex.getFilePointer(); + meta.writeLong(start); + meta.writeVInt(16); // DIRECT_MONOTONIC_BLOCK_SHIFT); + + final DirectMonotonicWriter memoryOffsetsWriter = + DirectMonotonicWriter.getInstance(meta, vectorIndex, valueCount, 16); + long cumulativeOffsetSum = 0; + for (int[] levelOffsets : graphLevelNodeOffsets) { + for (int v : levelOffsets) { + memoryOffsetsWriter.add(cumulativeOffsetSum); + cumulativeOffsetSum += v; + } + } + + memoryOffsetsWriter.finish(); + meta.writeLong(vectorIndex.getFilePointer() - start); + } + } + + public static int distFuncToOrd(VectorSimilarityFunction func) { + for (int i = 0; i < VECTOR_SIMILARITY_FUNCTIONS.size(); i++) { + if (VECTOR_SIMILARITY_FUNCTIONS.get(i).equals(func)) { + return (byte) i; + } + } + throw new IllegalArgumentException("invalid distance function: " + func); + } + + /** + * A utility method to print info/debugging messages using InfoStream. + * + * @param msg the debugging message to print + */ + public static void printInfoStream(InfoStream infoStream, String component, String msg) { + if (infoStream.isEnabled(component)) { + infoStream.message(component, msg); + } + } + + /** + * Writes an empty meta information for the field. + * + * @param fieldInfo instance of FieldInfo + * @throws IOException I/O Exceptions + */ + public static void writeEmpty(FieldInfo fieldInfo, IndexOutput op) throws IOException { + writeMeta(null, op, fieldInfo, 0, 0, 0, null, null, 0); + } + + /** + * Builds an instance of CagraIndexParams. + * + * @return instance of CagraIndexParams + */ + public static CagraIndexParams cagraIndexParams( + int cuvsWriterThreads, int intGraphDegree, int graphDegree) { + return new CagraIndexParams.Builder() + .withNumWriterThreads(cuvsWriterThreads) + .withIntermediateGraphDegree(intGraphDegree) + .withGraphDegree(graphDegree) + .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) + .build(); + } + + /** + * Quantizes FLOAT32 vectors to binary (1 bit per dimension, packed into bytes). + * Binary quantization: each dimension is compared to a centroid (mean of all values for that dimension). + * If value > centroid, bit = 1, else bit = 0. + * Bits are packed: 8 dimensions per byte. + * + * @param floatVectors A list of float vectors + * @return A list of byte binary representation for the input vectors + */ + public static List quantizeFloatVectorsToBinary(List floatVectors) { + if (floatVectors.isEmpty()) { + return new ArrayList<>(); + } + + int dimensions = floatVectors.get(0).length; + int numVectors = floatVectors.size(); + int bytesPerVector = (dimensions + 7) / 8; + + float[] centroids = new float[dimensions]; + for (float[] vector : floatVectors) { + for (int d = 0; d < dimensions; d++) { + centroids[d] += vector[d]; + } + } + for (int d = 0; d < dimensions; d++) { + centroids[d] /= numVectors; + } + + List quantizedVectors = new ArrayList<>(numVectors); + for (float[] vector : floatVectors) { + byte[] quantized = new byte[bytesPerVector]; + for (int d = 0; d < dimensions; d++) { + boolean bit = vector[d] > centroids[d]; + int byteIndex = d / 8; + int bitIndex = d % 8; + if (bit) { + quantized[byteIndex] |= (1 << bitIndex); + } + } + quantizedVectors.add(quantized); + } + + return quantizedVectors; + } + + /** + * Scalar quantization. + * + * @param floatVectors A list of float vectors + * @return A list of byte scalar representation for the input vectors + */ + public static List quantizeFloatVectorsToScalar(List floatVectors) { + if (floatVectors.isEmpty()) { + return new ArrayList<>(); + } + + int dimensions = floatVectors.get(0).length; + int numVectors = floatVectors.size(); + + float[] minPerDim = new float[dimensions]; + float[] maxPerDim = new float[dimensions]; + Arrays.fill(minPerDim, Float.MAX_VALUE); + Arrays.fill(maxPerDim, Float.MIN_VALUE); + + for (float[] vector : floatVectors) { + for (int d = 0; d < dimensions; d++) { + minPerDim[d] = Math.min(minPerDim[d], vector[d]); + maxPerDim[d] = Math.max(maxPerDim[d], vector[d]); + } + } + + List quantizedVectors = new ArrayList<>(numVectors); + for (float[] vector : floatVectors) { + byte[] quantized = new byte[dimensions]; + for (int d = 0; d < dimensions; d++) { + float range = maxPerDim[d] - minPerDim[d]; + if (range > 0) { + float normalized = (vector[d] - minPerDim[d]) / range; + int quantizedValue = Math.round(normalized * 127.0f) - 64; + quantized[d] = (byte) Math.max(-64, Math.min(63, quantizedValue)); + } else { + quantized[d] = 0; + } + } + quantizedVectors.add(quantized); + } + + return quantizedVectors; + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java new file mode 100644 index 0000000000..17336e3220 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java @@ -0,0 +1,104 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.DEFAULT_GRAPH_DEGREE; +import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.DEFAULT_HNSW_GRAPH_LAYERS; +import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.DEFAULT_INTERMEDIATE_GRAPH_DEGREE; +import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.DEFAULT_WRITER_THREADS; + +import com.nvidia.cuvs.LibraryException; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; + +/** + * CuVS based codec for GPU based vector search + * + * @since 26.02 + */ +public class LuceneAcceleratedHNSWBinaryQuantizedCodec extends FilterCodec { + + private static final Logger log = + Logger.getLogger(LuceneAcceleratedHNSWBinaryQuantizedCodec.class.getName()); + private static final String NAME = "Lucene101AcceleratedHNSWBinaryQuantizedCodec"; + private static final LuceneProvider LUCENE99_PROVIDER; + private static final Integer DEFAULT_MAX_CONN; + private static final Integer DEFAULT_BEAM_WIDTH; + + private KnnVectorsFormat format; + + static { + try { + LUCENE99_PROVIDER = LuceneProvider.getInstance("99"); + DEFAULT_MAX_CONN = LUCENE99_PROVIDER.getStaticIntParam("DEFAULT_MAX_CONN"); + DEFAULT_BEAM_WIDTH = LUCENE99_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + + public LuceneAcceleratedHNSWBinaryQuantizedCodec() throws Exception { + this(NAME, LuceneProvider.getCodec("101")); + } + + public LuceneAcceleratedHNSWBinaryQuantizedCodec(String name, Codec delegate) { + super(name, delegate); + initializeFormatDefaultValues(); + } + + public LuceneAcceleratedHNSWBinaryQuantizedCodec( + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + int maxConn, + int beamWidth) + throws Exception { + this(NAME, LuceneProvider.getCodec("101")); + initializeFormat( + cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); + } + + private void initializeFormatDefaultValues() { + initializeFormat( + DEFAULT_WRITER_THREADS, + DEFAULT_INTERMEDIATE_GRAPH_DEGREE, + DEFAULT_GRAPH_DEGREE, + DEFAULT_HNSW_GRAPH_LAYERS, + DEFAULT_MAX_CONN, + DEFAULT_BEAM_WIDTH); + } + + private void initializeFormat( + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + int maxConn, + int beamWidth) { + try { + format = + new LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat( + cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); + setKnnFormat(format); + } catch (LibraryException ex) { + log.log( + Level.SEVERE, + "Couldn't load native library, possible classloader issue. " + ex.getMessage()); + } + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return format; + } + + public void setKnnFormat(KnnVectorsFormat format) { + this.format = format; + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java new file mode 100644 index 0000000000..fb74d97f78 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java @@ -0,0 +1,176 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; + +import com.nvidia.cuvs.LibraryException; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; + +/** + * cuVS based Binary Quantized KnnVectorsFormat for indexing on GPU and searching on the CPU. + * + * @since 26.02 + */ +public class LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat extends KnnVectorsFormat { + + private static final Logger log = + Logger.getLogger(LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.class.getName()); + private static final LuceneProvider LUCENE102_PROVIDER; + private static final LuceneProvider LUCENE99_PROVIDER; + private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; + private static final Integer DEFAULT_MAX_CONN; + private static final Integer DEFAULT_BEAM_WIDTH; + + public static final int DEFAULT_WRITER_THREADS = 32; + public static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; + public static final int DEFAULT_GRAPH_DEGREE = 64; + public static final int DEFAULT_HNSW_GRAPH_LAYERS = 2; + + private final int maxDimensions = 4096; + private final int cuvsWriterThreads; + private final int intGraphDegree; + private final int graphDegree; + private final int hnswLayers; + private final int maxConn; + private final int beamWidth; + + static { + try { + LUCENE99_PROVIDER = LuceneProvider.getInstance("99"); + LUCENE102_PROVIDER = LuceneProvider.getInstance("102"); + FLAT_VECTORS_FORMAT = + LUCENE102_PROVIDER.getLuceneFlatVectorsFormatInstance(DefaultFlatVectorScorer.INSTANCE); + DEFAULT_MAX_CONN = LUCENE99_PROVIDER.getStaticIntParam("DEFAULT_MAX_CONN"); + DEFAULT_BEAM_WIDTH = LUCENE99_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + + /** + * Initializes {@link LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat} with default values. + * + * @throws LibraryException if the native library fails to load + */ + public LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat() { + this( + DEFAULT_WRITER_THREADS, + DEFAULT_INTERMEDIATE_GRAPH_DEGREE, + DEFAULT_GRAPH_DEGREE, + DEFAULT_HNSW_GRAPH_LAYERS, + DEFAULT_MAX_CONN, + DEFAULT_BEAM_WIDTH); + } + + /** + * Initializes {@link LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat} with the given threads, graph degree, etc. + * + * @param cuvsWriterThreads number of cuVS threads to use while building the CAGRA index + * @param intGraphDegree the intermediate graph degree while building the CAGRA index + * @param graphDegree the graph degree to use while building the CAGRA index + * @param hnswLayers the number of HNSW layers to construct in the HNSW graph + * @param maxConn the maximum connections for the HNSW graph + * @param beamWidth the beam width to use while building the HNSW graph + */ + public LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat( + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + int maxConn, + int beamWidth) { + super("Lucene99AcceleratedHNSWBinaryQuantizedVectorsFormat"); + + assert cuvsWriterThreads > 0 + : "cuvsWriterThreads must be greater than zero, but is: " + cuvsWriterThreads; + assert intGraphDegree > 0 + : "intGraphDegree must be greater than zero, but is: " + intGraphDegree; + assert graphDegree > 0 : "graphDegree must be greater than zero, but is: " + graphDegree; + assert hnswLayers > 0 : "hnswLayers must be greater than zero, but is: " + hnswLayers; + assert maxConn > 0 : "maxConn must be greater than zero, but is: " + maxConn; + assert beamWidth > 0 : "beamWidth must be greater than zero, but is: " + beamWidth; + + this.cuvsWriterThreads = cuvsWriterThreads; + this.intGraphDegree = intGraphDegree; + this.graphDegree = graphDegree; + this.hnswLayers = hnswLayers; + this.maxConn = maxConn; + this.beamWidth = beamWidth; + } + + /** + * Returns a KnnVectorsWriter to write the binary quantized vectors to the index. + */ + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); + if (isSupported()) { + log.log( + Level.FINE, + "cuVS is supported so using the Lucene99AcceleratedHNSWBinaryQuantizedVectorsWriter"); + return new LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter( + state, cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, flatWriter); + } else { + try { + // Fallback to Lucene's Lucene102HnswBinaryQuantizedVectorsFormat format + log.log( + Level.WARNING, + "GPU based indexing not supported, falling back to using the" + + " Lucene102HnswBinaryQuantizedVectorsFormat"); + KnnVectorsFormat fallbackFormat = + LUCENE102_PROVIDER.getLuceneHnswBinaryQuantizedVectorsFormatInstance( + maxConn, beamWidth); + return fallbackFormat.fieldsWriter(state); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + } + + /** + * Returns a KnnVectorsReader to read the binary quantized vectors from the index. + */ + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + try { + return LUCENE99_PROVIDER.getLuceneHnswVectorsReaderInstance( + state, FLAT_VECTORS_FORMAT.fieldsReader(state)); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + /** + * Returns the maximum number of vector dimensions supported by this codec for the given field name. + */ + @Override + public int getMaxDimensions(String fieldName) { + return maxDimensions; + } + + /** + * Returns a string containing the meta information like hnsw layers, graph degree etc. + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(this.getClass().getSimpleName()); + sb.append("(cuvsWriterThreads=").append(cuvsWriterThreads); + sb.append("intGraphDegree=").append(intGraphDegree); + sb.append("graphDegree=").append(graphDegree); + sb.append("hnswLayers=").append(hnswLayers); + sb.append(")"); + return sb.toString(); + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java new file mode 100644 index 0000000000..b55ed98248 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java @@ -0,0 +1,403 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.cagraIndexParams; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.createMultiLayerHnswGraph; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.createSingleVectorHnswGraph; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.printInfoStream; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.quantizeFloatVectorsToBinary; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.writeEmpty; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.writeGraph; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.writeMeta; +import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_CODEC_NAME; +import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_EXT; +import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_EXT; +import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_NAME; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.closeCuVSResourcesInstance; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.getCuVSResourcesInstance; +import static org.apache.lucene.index.VectorEncoding.FLOAT32; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; + +import com.nvidia.cuvs.CagraIndex; +import com.nvidia.cuvs.CagraIndexParams; +import com.nvidia.cuvs.CuVSMatrix; +import com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.QuantizationType; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Sorter; +import org.apache.lucene.index.Sorter.DocMap; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.InfoStream; + +/** + * This class extends upon the KnnVectorsWriter to enable the creation of GPU-based accelerated + * vector search indexes. + * + * @since 26.02 + */ +public class LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter extends KnnVectorsWriter { + + private static final long SHALLOW_RAM_BYTES_USED = + shallowSizeOfInstance(LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.class); + private static final String COMPONENT = "Lucene99AcceleratedHNSWBinaryQuantizedVectorsWriter"; + + private final int cuvsWriterThreads; + private final int intGraphDegree; + private final int graphDegree; + private final int hnswLayers; + private final FlatVectorsWriter flatVectorsWriter; + private final List fields = new ArrayList<>(); + private final InfoStream infoStream; + private IndexOutput hnswMeta = null, hnswVectorIndex = null; + private boolean finished; + private String vemFileName; + private String vexFileName; + + /** + * Initializes {@link LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter} + * + * @param state instance of the {@link org.apache.lucene.index.SegmentWriteState} + * @param cuvsWriterThreads number of cuVS threads to use while building the intermediate CAGRA index + * @param intGraphDegree the intermediate graph degree to use while building the CAGRA index + * @param graphDegree the graph degree to use while building the CAGRA index + * @param hnswLayers the number of hnsw layers to construct while building the HNSW graph + * @param flatVectorsWriter instance of the {@link org.apache.lucene.codecs.hnsw.FlatVectorsWriter} + * @throws IOException IOException + */ + public LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter( + SegmentWriteState state, + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + FlatVectorsWriter flatVectorsWriter) + throws IOException { + super(); + this.cuvsWriterThreads = cuvsWriterThreads; + this.intGraphDegree = intGraphDegree; + this.graphDegree = graphDegree; + this.hnswLayers = hnswLayers; + this.flatVectorsWriter = flatVectorsWriter; + this.infoStream = state.infoStream; + + vemFileName = + IndexFileNames.segmentFileName( + state.segmentInfo.name, state.segmentSuffix, HNSW_META_CODEC_EXT); + + vexFileName = + IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, HNSW_INDEX_EXT); + + boolean success = false; + try { + hnswMeta = state.directory.createOutput(vemFileName, state.context); + hnswVectorIndex = state.directory.createOutput(vexFileName, state.context); + + CodecUtil.writeIndexHeader( + hnswMeta, + HNSW_META_CODEC_NAME, + Lucene99HnswVectorsFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix); + CodecUtil.writeIndexHeader( + hnswVectorIndex, + HNSW_INDEX_CODEC_NAME, + Lucene99HnswVectorsFormat.VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix); + + success = true; + printInfoStream( + infoStream, COMPONENT, "Lucene99AcceleratedHNSWBinaryQuantizedVectorsWriter opened"); + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + /** + * Add new field for indexing. + */ + @Override + public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { + VectorEncoding encoding = fieldInfo.getVectorEncoding(); + if (encoding != FLOAT32) { + throw new IllegalArgumentException("expected float32, got:" + encoding); + } + var writer = Objects.requireNonNull(flatVectorsWriter.addField(fieldInfo)); + var cuvsFieldWriter = new QuantizedFieldWriter(QuantizationType.BINARY, fieldInfo, writer); + fields.add(cuvsFieldWriter); + return writer; + } + + /** + * Utility method for building index metadata information string object. + * + * @param size index size + * @param args additional metadata information + * @return the string representation of the metadata information + */ + static String indexMsg(int size, int... args) { + StringBuilder sb = new StringBuilder("cagra index params"); + sb.append(": size=").append(size); + sb.append(", intGraphDegree=").append(args[0]); + sb.append(", actualIntGraphDegree=").append(args[1]); + sb.append(", graphDegree=").append(args[2]); + sb.append(", actualGraphDegree=").append(args[3]); + return sb.toString(); + } + + /** + * Builds the intermediate CAGRA index and builds and writes the HNSW index for binary quantized vectors. + * Binary quantized vectors are stored as packed bits (1 bit per dimension, 8 dimensions per byte). + * + * @param fieldInfo instance of FieldInfo that has the field description + * @param vectors binary quantized vectors (packed bits as bytes) + * @throws IOException + */ + private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOException { + if (vectors.size() == 0) { + writeEmpty(fieldInfo, hnswMeta); + return; + } + + try { + int dimensions = fieldInfo.getVectorDimension(); + int bytesPerVector = (dimensions + 7) / 8; + + CuVSMatrix dataset = + Utils.createByteMatrix(vectors, bytesPerVector, getCuVSResourcesInstance()); + + if (dataset.size() < 2) { + writeSingleVectorGraph(fieldInfo, vectors); + return; + } + + CagraIndexParams params = cagraIndexParams(cuvsWriterThreads, intGraphDegree, graphDegree); + CagraIndex cagraIndex = + CagraIndex.newBuilder(getCuVSResourcesInstance()) + .withDataset(dataset) + .withIndexParams(params) + .build(); + + CuVSMatrix adjacencyListMatrix = cagraIndex.getGraph(); + int size = (int) dataset.size(); + + // Create multi-layer HNSW graph from CAGRA + GPUBuiltHnswGraph hnswGraph = + createMultiLayerHnswGraph( + fieldInfo, + size, + dimensions, + adjacencyListMatrix, + vectors, + hnswLayers, + graphDegree, + params, + QuantizationType.BINARY); + + long vectorIndexOffset = hnswVectorIndex.getFilePointer(); + // Write the graph to the vector index + int[][] graphLevelNodeOffsets = writeGraph(hnswGraph, hnswVectorIndex); + long vectorIndexLength = hnswVectorIndex.getFilePointer() - vectorIndexOffset; + + // Write metadata + writeMeta( + hnswVectorIndex, + hnswMeta, + fieldInfo, + vectorIndexOffset, + vectorIndexLength, + size, + hnswGraph, + graphLevelNodeOffsets, + graphDegree); + + cagraIndex.close(); + + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + + /** + * Build the indexes and writes it to the disk. + */ + @Override + public void flush(int maxDoc, DocMap sortMap) throws IOException { + flatVectorsWriter.flush(maxDoc, sortMap); + for (var field : fields) { + if (sortMap == null) { + writeField(field); + } else { + writeSortingField(field, sortMap); + } + } + } + + /** + * Builds the index and writes it to the disk. + * + * @param fieldData + * @throws IOException + */ + private void writeField(QuantizedFieldWriter fieldData) throws IOException { + writeFieldInternal(fieldData.fieldInfo(), fieldData.getVectors()); + } + + /** + * Builds the index and writes it to the disk. + * + * @param fieldData instance of BinaryQuantizedGPUFieldWriter + * @param sortMap instance of the DocMap + * @throws IOException + */ + private void writeSortingField(QuantizedFieldWriter fieldData, Sorter.DocMap sortMap) + throws IOException { + + DocsWithFieldSet oldDocsWithFieldSet = fieldData.getDocsWithFieldSet(); + final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; // new ord to old ord + mapOldOrdToNewOrd(oldDocsWithFieldSet, sortMap, null, new2OldOrd, null); + + List sortedVectors = new ArrayList(); + List vectors = fieldData.getVectors(); + for (int i = 0; i < vectors.size(); i++) { + sortedVectors.add(vectors.get(new2OldOrd[i])); + } + + writeFieldInternal(fieldData.fieldInfo(), sortedVectors); + } + + /** + * Builds and writes a single vector graph. + * + * @param fieldInfo instance of FieldInfo + * @param vectors the list of binary quantized vectors + * @throws IOException I/O Exceptions + */ + private void writeSingleVectorGraph(FieldInfo fieldInfo, List vectors) + throws IOException { + // Workaround for CAGRA not supporting single vector indexes + try { + int size = 1; + int dimensions = fieldInfo.getVectorDimension(); + + // Create a dummy HNSW graph for a single vector + GPUBuiltHnswGraph hnswGraph = createSingleVectorHnswGraph(size, dimensions); + + long vectorIndexOffset = hnswVectorIndex.getFilePointer(); + + // Write the graph to the vector index + int[][] graphLevelNodeOffsets = writeGraph(hnswGraph, hnswVectorIndex); + + long vectorIndexLength = hnswVectorIndex.getFilePointer() - vectorIndexOffset; + + // Write metadata + writeMeta( + hnswVectorIndex, + hnswMeta, + fieldInfo, + vectorIndexOffset, + vectorIndexLength, + size, + hnswGraph, + graphLevelNodeOffsets, + graphDegree); + + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + + /** + * Write field for merging. + */ + @Override + public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + flatVectorsWriter.mergeOneField(fieldInfo, mergeState); + vectorBasedMerge(fieldInfo, mergeState); + } + + /** + * Fallback method that rebuilds indexes from merged vectors. + * Used when native CAGRA merge() is not possible. + */ + private void vectorBasedMerge(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + try { + FloatVectorValues mergedVectorValues = + KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); + + if (mergedVectorValues != null) { + List floatVectors = new ArrayList<>(); + KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); + for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { + floatVectors.add(mergedVectorValues.vectorValue(iter.index()).clone()); + } + writeFieldInternal(fieldInfo, quantizeFloatVectorsToBinary(floatVectors)); + } + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + + /** + * Called once at the end before close. + */ + @Override + public void finish() throws IOException { + if (finished) { + throw new IllegalStateException("already finished"); + } + finished = true; + flatVectorsWriter.finish(); + if (hnswMeta != null) { + // write end of fields marker + hnswMeta.writeInt(-1); + CodecUtil.writeFooter(hnswMeta); + } + if (hnswVectorIndex != null) { + CodecUtil.writeFooter(hnswVectorIndex); + } + } + + /** + * Closes the resources. + */ + @Override + public void close() throws IOException { + IOUtils.close(hnswMeta, hnswVectorIndex, flatVectorsWriter); + closeCuVSResourcesInstance(); + } + + /** + * Returns the memory usage of this object in bytes. + */ + @Override + public long ramBytesUsed() { + long total = SHALLOW_RAM_BYTES_USED; + for (var field : fields) { + total += field.ramBytesUsed(); + } + return total; + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java new file mode 100644 index 0000000000..2b23a9c57c --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java @@ -0,0 +1,104 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.DEFAULT_GRAPH_DEGREE; +import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.DEFAULT_HNSW_GRAPH_LAYERS; +import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.DEFAULT_INTERMEDIATE_GRAPH_DEGREE; +import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.DEFAULT_WRITER_THREADS; + +import com.nvidia.cuvs.LibraryException; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; + +/** + * CuVS based codec for GPU based vector search + * + * @since 26.02 + */ +public class LuceneAcceleratedHNSWScalarQuantizedCodec extends FilterCodec { + + private static final Logger log = + Logger.getLogger(LuceneAcceleratedHNSWScalarQuantizedCodec.class.getName()); + private static final String NAME = "Lucene101AcceleratedHNSWScalarQuantizedCodec"; + private static final LuceneProvider LUCENE99_PROVIDER; + private static final Integer DEFAULT_MAX_CONN; + private static final Integer DEFAULT_BEAM_WIDTH; + + private KnnVectorsFormat format; + + static { + try { + LUCENE99_PROVIDER = LuceneProvider.getInstance("99"); + DEFAULT_MAX_CONN = LUCENE99_PROVIDER.getStaticIntParam("DEFAULT_MAX_CONN"); + DEFAULT_BEAM_WIDTH = LUCENE99_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + + public LuceneAcceleratedHNSWScalarQuantizedCodec() throws Exception { + this(NAME, LuceneProvider.getCodec("101")); + } + + public LuceneAcceleratedHNSWScalarQuantizedCodec(String name, Codec delegate) { + super(name, delegate); + initializeFormatDefaultValues(); + } + + public LuceneAcceleratedHNSWScalarQuantizedCodec( + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + int maxConn, + int beamWidth) + throws Exception { + this(NAME, LuceneProvider.getCodec("101")); + initializeFormat( + cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); + } + + private void initializeFormatDefaultValues() { + initializeFormat( + DEFAULT_WRITER_THREADS, + DEFAULT_INTERMEDIATE_GRAPH_DEGREE, + DEFAULT_GRAPH_DEGREE, + DEFAULT_HNSW_GRAPH_LAYERS, + DEFAULT_MAX_CONN, + DEFAULT_BEAM_WIDTH); + } + + private void initializeFormat( + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + int maxConn, + int beamWidth) { + try { + format = + new LuceneAcceleratedHNSWScalarQuantizedVectorsFormat( + cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); + setKnnFormat(format); + } catch (LibraryException ex) { + log.log( + Level.SEVERE, + "Couldn't load native library, possible classloader issue. " + ex.getMessage()); + } + } + + @Override + public KnnVectorsFormat knnVectorsFormat() { + return format; + } + + public void setKnnFormat(KnnVectorsFormat format) { + this.format = format; + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java new file mode 100644 index 0000000000..e194e0a99b --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java @@ -0,0 +1,167 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; + +import com.nvidia.cuvs.LibraryException; +import java.io.IOException; +import java.util.logging.Logger; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; + +/** + * cuVS based Scalar Quantized KnnVectorsFormat for indexing on GPU and searching on the CPU. + * + * @since 26.02 + */ +public class LuceneAcceleratedHNSWScalarQuantizedVectorsFormat extends KnnVectorsFormat { + + private static final Logger log = + Logger.getLogger(LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.class.getName()); + private static final LuceneProvider LUCENE_PROVIDER; + private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; + private static final Integer DEFAULT_MAX_CONN; + private static final Integer DEFAULT_BEAM_WIDTH; + + public static final int DEFAULT_WRITER_THREADS = 32; + public static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; + public static final int DEFAULT_GRAPH_DEGREE = 64; + public static final int DEFAULT_HNSW_GRAPH_LAYERS = 2; + + private final int maxDimensions = 4096; + private final int cuvsWriterThreads; + private final int intGraphDegree; + private final int graphDegree; + private final int hnswLayers; + private final int maxConn; + private final int beamWidth; + + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + DEFAULT_MAX_CONN = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_MAX_CONN"); + DEFAULT_BEAM_WIDTH = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); + FLAT_VECTORS_FORMAT = LUCENE_PROVIDER.getLuceneScalarQuantizedVectorsFormatInstance(); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + + /** + * Initializes {@link LuceneAcceleratedHNSWScalarQuantizedVectorsFormat} with default values. + * + * @throws LibraryException if the native library fails to load + */ + public LuceneAcceleratedHNSWScalarQuantizedVectorsFormat() { + this( + DEFAULT_WRITER_THREADS, + DEFAULT_INTERMEDIATE_GRAPH_DEGREE, + DEFAULT_GRAPH_DEGREE, + DEFAULT_HNSW_GRAPH_LAYERS, + DEFAULT_MAX_CONN, + DEFAULT_BEAM_WIDTH); + } + + /** + * Initializes {@link LuceneAcceleratedHNSWScalarQuantizedVectorsFormat} with the given threads, graph degree, etc. + * + * @param cuvsWriterThreads number of cuVS threads to use while building the CAGRA index + * @param intGraphDegree the intermediate graph degree while building the CAGRA index + * @param graphDegree the graph degree to use while building the CAGRA index + * @param hnswLayers the number of HNSW layers to construct in the HNSW graph + * @param maxConn the maximum connections for the HNSW graph + * @param beamWidth the beam width to use while building the HNSW graph + */ + public LuceneAcceleratedHNSWScalarQuantizedVectorsFormat( + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + int maxConn, + int beamWidth) { + super("Lucene99AcceleratedHNSWScalarQuantizedVectorsFormat"); + + assert cuvsWriterThreads > 0 + : "cuvsWriterThreads must be greater than zero, but is: " + cuvsWriterThreads; + assert intGraphDegree > 0 + : "intGraphDegree must be greater than zero, but is: " + intGraphDegree; + assert graphDegree > 0 : "graphDegree must be greater than zero, but is: " + graphDegree; + assert hnswLayers > 0 : "hnswLayers must be greater than zero, but is: " + hnswLayers; + assert maxConn > 0 : "maxConn must be greater than zero, but is: " + maxConn; + assert beamWidth > 0 : "beamWidth must be greater than zero, but is: " + beamWidth; + + this.cuvsWriterThreads = cuvsWriterThreads; + this.intGraphDegree = intGraphDegree; + this.graphDegree = graphDegree; + this.hnswLayers = hnswLayers; + this.maxConn = maxConn; + this.beamWidth = beamWidth; + } + + /** + * Returns a KnnVectorsWriter to write the scalar quantized vectors to the index. + */ + @Override + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); + if (isSupported()) { + log.info("cuVS is supported so using the Lucene99AcceleratedHNSWQuantizedVectorsWriter"); + return new LuceneAcceleratedHNSWScalarQuantizedVectorsWriter( + state, cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, flatWriter); + } else { + try { + // Fallback to Lucene's Lucene99HnswScalarQuantizedVectorsFormat + log.warning( + "GPU based indexing not supported, falling back to using the" + + " Lucene99HnswScalarQuantizedVectorsFormat"); + KnnVectorsFormat fallbackFormat = + LUCENE_PROVIDER.getLuceneHnswScalarQuantizedVectorsFormatInstance(beamWidth, maxConn); + return fallbackFormat.fieldsWriter(state); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + } + + /** + * Returns a KnnVectorsReader to read the scalar quantized vectors from the index. + */ + @Override + public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { + try { + return LUCENE_PROVIDER.getLuceneHnswVectorsReaderInstance( + state, FLAT_VECTORS_FORMAT.fieldsReader(state)); + } catch (Exception e) { + throw new RuntimeException(e.getMessage()); + } + } + + /** + * Returns the maximum number of vector dimensions supported by this Codec for the given field name. + */ + @Override + public int getMaxDimensions(String fieldName) { + return maxDimensions; + } + + /** + * Returns a string containing the meta information like hnsw layers, graph degree etc. + */ + @Override + public String toString() { + StringBuilder sb = new StringBuilder(this.getClass().getSimpleName()); + sb.append("(cuvsWriterThreads=").append(cuvsWriterThreads); + sb.append("intGraphDegree=").append(intGraphDegree); + sb.append("graphDegree=").append(graphDegree); + sb.append("hnswLayers=").append(hnswLayers); + sb.append(")"); + return sb.toString(); + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java new file mode 100644 index 0000000000..e13102b096 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java @@ -0,0 +1,427 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.cagraIndexParams; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.createMultiLayerHnswGraph; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.createSingleVectorHnswGraph; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.printInfoStream; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.quantizeFloatVectorsToScalar; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.writeEmpty; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.writeGraph; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.writeMeta; +import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_CODEC_NAME; +import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_EXT; +import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_EXT; +import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_NAME; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.closeCuVSResourcesInstance; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.getCuVSResourcesInstance; +import static org.apache.lucene.index.VectorEncoding.FLOAT32; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; +import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; + +import com.nvidia.cuvs.CagraIndex; +import com.nvidia.cuvs.CagraIndexParams; +import com.nvidia.cuvs.CuVSMatrix; +import com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.QuantizationType; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.codecs.KnnVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.index.MergeState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Sorter; +import org.apache.lucene.index.Sorter.DocMap; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.InfoStream; + +/** + * This class extends upon the KnnVectorsWriter to enable the creation of GPU-based accelerated + * vector search indexes. + * + * @since 26.02 + */ +public class LuceneAcceleratedHNSWScalarQuantizedVectorsWriter extends KnnVectorsWriter { + + private static final long SHALLOW_RAM_BYTES_USED = + shallowSizeOfInstance(LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.class); + private static final String COMPONENT = "Lucene99AcceleratedHNSWQuantizedVectorsWriter"; + private static final LuceneProvider LUCENE_PROVIDER; + private static final Integer VERSION_CURRENT; + + private final int cuvsWriterThreads; + private final int intGraphDegree; + private final int graphDegree; + private final int hnswLayers; + private final FlatVectorsWriter flatVectorsWriter; + private final List fields = new ArrayList<>(); + private final InfoStream infoStream; + private IndexOutput hnswMeta = null, hnswVectorIndex = null; + private boolean finished; + private String vemFileName; + private String vexFileName; + + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + VERSION_CURRENT = LUCENE_PROVIDER.getStaticIntParam("VERSION_CURRENT"); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + + /** + * Initializes {@link LuceneAcceleratedHNSWScalarQuantizedVectorsWriter} + * + * @param state instance of the {@link org.apache.lucene.index.SegmentWriteState} + * @param cuvsWriterThreads number of cuVS threads to use while building the intermediate CAGRA index + * @param intGraphDegree the intermediate graph degree to use while building the CAGRA index + * @param graphDegree the graph degree to use while building the CAGRA index + * @param hnswLayers the number of hnsw layers to construct while building the HNSW graph + * @param flatVectorsWriter instance of the {@link org.apache.lucene.codecs.hnsw.FlatVectorsWriter} + * @throws IOException IOException + */ + public LuceneAcceleratedHNSWScalarQuantizedVectorsWriter( + SegmentWriteState state, + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + int hnswLayers, + FlatVectorsWriter flatVectorsWriter) + throws IOException { + super(); + this.cuvsWriterThreads = cuvsWriterThreads; + this.intGraphDegree = intGraphDegree; + this.graphDegree = graphDegree; + this.hnswLayers = hnswLayers; + this.flatVectorsWriter = flatVectorsWriter; + this.infoStream = state.infoStream; + + vemFileName = + IndexFileNames.segmentFileName( + state.segmentInfo.name, state.segmentSuffix, HNSW_META_CODEC_EXT); + + vexFileName = + IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, HNSW_INDEX_EXT); + + boolean success = false; + try { + + hnswMeta = state.directory.createOutput(vemFileName, state.context); + hnswVectorIndex = state.directory.createOutput(vexFileName, state.context); + + CodecUtil.writeIndexHeader( + hnswMeta, + HNSW_META_CODEC_NAME, + VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix); + CodecUtil.writeIndexHeader( + hnswVectorIndex, + HNSW_INDEX_CODEC_NAME, + VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix); + success = true; + printInfoStream( + infoStream, COMPONENT, "Lucene99AcceleratedHNSWQuantizedVectorsWriter opened"); + } finally { + if (success == false) { + IOUtils.closeWhileHandlingException(this); + } + } + } + + /** + * Add new field for indexing. + */ + @Override + public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { + VectorEncoding encoding = fieldInfo.getVectorEncoding(); + if (encoding != FLOAT32) { + throw new IllegalArgumentException("expected float32, got:" + encoding); + } + var writer = Objects.requireNonNull(flatVectorsWriter.addField(fieldInfo)); + var cuvsFieldWriter = new QuantizedFieldWriter(QuantizationType.SCALAR, fieldInfo, writer); + fields.add(cuvsFieldWriter); + return writer; + } + + /** + * Utility method for building index metadata information string object. + * + * @param size index size + * @param args additional metadata information + * @return the string representation of the metadata information + */ + static String indexMsg(int size, int... args) { + StringBuilder sb = new StringBuilder("cagra index params"); + sb.append(": size=").append(size); + sb.append(", intGraphDegree=").append(args[0]); + sb.append(", actualIntGraphDegree=").append(args[1]); + sb.append(", graphDegree=").append(args[2]); + sb.append(", actualGraphDegree=").append(args[3]); + return sb.toString(); + } + + private static byte signedToUnsignedByte(byte signedByte) { + return (byte) (signedByte & 0xFF); + } + + private static byte[] convertSignedToUnsigned(byte[] signedVector) { + byte[] unsignedVector = new byte[signedVector.length]; + for (int i = 0; i < signedVector.length; i++) { + unsignedVector[i] = signedToUnsignedByte(signedVector[i]); + } + return unsignedVector; + } + + /** + * Builds the intermediate CAGRA index and builds and writes the HNSW index. + * + * @param fieldInfo instance of FieldInfo that has the field description + * @param vectors quantized vectors + * @throws IOException + */ + private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOException { + if (vectors.size() == 0) { + writeEmpty(fieldInfo, hnswMeta); + return; + } + + try { + int dimensions = fieldInfo.getVectorDimension(); + + // Convert 7-bit signed bytes to 8-bit unsigned bytes for cuVS compatibility + List unsignedVectors = new ArrayList<>(vectors.size()); + for (byte[] signedVector : vectors) { + unsignedVectors.add(convertSignedToUnsigned(signedVector)); + } + + // Create CuVSMatrix with BYTE data type (unsigned bytes) + CuVSMatrix dataset = + Utils.createByteMatrix(unsignedVectors, dimensions, getCuVSResourcesInstance()); + + if (dataset.size() < 2) { + writeSingleVectorGraph(fieldInfo, unsignedVectors); + return; + } + + CagraIndexParams params = cagraIndexParams(cuvsWriterThreads, intGraphDegree, graphDegree); + CagraIndex cagraIndex = + CagraIndex.newBuilder(getCuVSResourcesInstance()) + .withDataset(dataset) + .withIndexParams(params) + .build(); + + CuVSMatrix adjacencyListMatrix = cagraIndex.getGraph(); + + int size = (int) dataset.size(); + GPUBuiltHnswGraph hnswGraph = + createMultiLayerHnswGraph( + fieldInfo, + size, + dimensions, + adjacencyListMatrix, + unsignedVectors, + hnswLayers, + graphDegree, + params, + QuantizationType.SCALAR); + + long vectorIndexOffset = hnswVectorIndex.getFilePointer(); + + // Write the graph to the vector index + int[][] graphLevelNodeOffsets = writeGraph(hnswGraph, hnswVectorIndex); + + long vectorIndexLength = hnswVectorIndex.getFilePointer() - vectorIndexOffset; + + // Write metadata + writeMeta( + hnswVectorIndex, + hnswMeta, + fieldInfo, + vectorIndexOffset, + vectorIndexLength, + size, + hnswGraph, + graphLevelNodeOffsets, + graphDegree); + + cagraIndex.close(); + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + + /** + * Build the indexes and writes it to the disk. + */ + @Override + public void flush(int maxDoc, DocMap sortMap) throws IOException { + flatVectorsWriter.flush(maxDoc, sortMap); + for (var field : fields) { + if (sortMap == null) { + writeField(field); + } else { + writeSortingField(field, sortMap); + } + } + } + + /** + * Builds the index and writes it to the disk. + * + * @param fieldData + * @throws IOException + */ + private void writeField(QuantizedFieldWriter fieldData) throws IOException { + writeFieldInternal(fieldData.fieldInfo(), fieldData.getVectors()); + } + + /** + * Builds the index and writes it to the disk. + * + * @param fieldData instance of ScalarQuantizedGPUFieldWriter + * @param sortMap instance of the DocMap + * @throws IOException + */ + private void writeSortingField(QuantizedFieldWriter fieldData, Sorter.DocMap sortMap) + throws IOException { + + DocsWithFieldSet oldDocsWithFieldSet = fieldData.getDocsWithFieldSet(); + final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; // new ord to old ord + mapOldOrdToNewOrd(oldDocsWithFieldSet, sortMap, null, new2OldOrd, null); + + List sortedVectors = new ArrayList(); + for (int i = 0; i < fieldData.getVectors().size(); i++) { + sortedVectors.add(fieldData.getVectors().get(new2OldOrd[i])); + } + + writeFieldInternal(fieldData.fieldInfo(), sortedVectors); + } + + /** + * Builds and writes a single vector graph. + * + * @param fieldInfo instance of FieldInfo + * @param vectors the list of scalar quantized vectors (already converted to unsigned) + * @throws IOException I/O Exceptions + */ + private void writeSingleVectorGraph(FieldInfo fieldInfo, List vectors) + throws IOException { + // Workaround for CAGRA not supporting single vector indexes + try { + int size = 1; + int dimensions = fieldInfo.getVectorDimension(); + + // Create a dummy HNSW graph for a single vector + GPUBuiltHnswGraph hnswGraph = createSingleVectorHnswGraph(size, dimensions); + + long vectorIndexOffset = hnswVectorIndex.getFilePointer(); + // Write the graph to the vector index + int[][] graphLevelNodeOffsets = writeGraph(hnswGraph, hnswVectorIndex); + long vectorIndexLength = hnswVectorIndex.getFilePointer() - vectorIndexOffset; + + // Write metadata + writeMeta( + hnswVectorIndex, + hnswMeta, + fieldInfo, + vectorIndexOffset, + vectorIndexLength, + size, + hnswGraph, + graphLevelNodeOffsets, + graphDegree); + + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + + /** + * Write field for merging. + */ + @Override + public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + flatVectorsWriter.mergeOneField(fieldInfo, mergeState); + vectorBasedMerge(fieldInfo, mergeState); + } + + /** + * Fallback method that rebuilds indexes from merged vectors. + * Used when native CAGRA merge() is not possible. + */ + private void vectorBasedMerge(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + try { + FloatVectorValues mergedVectorValues = + KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState); + + if (mergedVectorValues != null) { + List floatVectors = new ArrayList<>(); + KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); + for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { + floatVectors.add(mergedVectorValues.vectorValue(iter.index()).clone()); + } + writeFieldInternal(fieldInfo, quantizeFloatVectorsToScalar(floatVectors)); + } + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + + /** + * Called once at the end before close. + */ + @Override + public void finish() throws IOException { + if (finished) { + throw new IllegalStateException("already finished"); + } + finished = true; + flatVectorsWriter.finish(); + if (hnswMeta != null) { + // write end of fields marker + hnswMeta.writeInt(-1); + CodecUtil.writeFooter(hnswMeta); + } + if (hnswVectorIndex != null) { + CodecUtil.writeFooter(hnswVectorIndex); + } + } + + /** + * Closes the resources. + */ + @Override + public void close() throws IOException { + IOUtils.close(hnswMeta, hnswVectorIndex, flatVectorsWriter); + closeCuVSResourcesInstance(); + } + + /** + * Returns the memory usage of this object in bytes. + */ + @Override + public long ramBytesUsed() { + long total = SHALLOW_RAM_BYTES_USED; + for (var field : fields) { + total += field.ramBytesUsed(); + } + return total; + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java index f7f90abe92..7635e32323 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; @@ -34,30 +34,50 @@ public class LuceneProvider { private static final String BASE = "org.apache.lucene."; private static String codecs = "codecs.lucene."; - private static String backwardCodecs = "backward_codecs.lucene."; + private static String fallbackCodecs = "backward_codecs.lucene."; - private static String luceneflatVectorsFormat = + private static String luceneFlatVectorsFormat = BASE + codecs + "LuceneFlatVectorsFormat"; private static String luceneFlatVectorsFormatFallback = - BASE + backwardCodecs + "LuceneFlatVectorsFormat"; + BASE + fallbackCodecs + "LuceneFlatVectorsFormat"; private static String luceneHnswVectorsFormat = BASE + codecs + "LuceneHnswVectorsFormat"; private static String luceneHnswVectorsFormatFallback = - BASE + backwardCodecs + "LuceneHnswVectorsFormat"; + BASE + fallbackCodecs + "LuceneHnswVectorsFormat"; private static String luceneHnswVectorsReader = BASE + codecs + "LuceneHnswVectorsReader"; private static String luceneHnswVectorsReaderFallback = - BASE + backwardCodecs + "LuceneHnswVectorsReader"; + BASE + fallbackCodecs + "LuceneHnswVectorsReader"; private static String luceneHnswVectorsWriter = BASE + codecs + "LuceneHnswVectorsWriter"; private static String luceneHnswVectorsWriterFallback = - BASE + backwardCodecs + "LuceneHnswVectorsWriter"; + BASE + fallbackCodecs + "LuceneHnswVectorsWriter"; + + private static String luceneBinaryQuantizedVectorsFormat = + BASE + codecs + "LuceneBinaryQuantizedVectorsFormat"; + private static String luceneBinaryQuantizedVectorsFormatFallback = + BASE + fallbackCodecs + "LuceneBinaryQuantizedVectorsFormat"; + + private static String luceneHnswBinaryQuantizedVectorsFormat = + BASE + codecs + "LuceneHnswBinaryQuantizedVectorsFormat"; + private static String luceneHnswBinaryQuantizedVectorsFormatFallback = + BASE + fallbackCodecs + "LuceneHnswBinaryQuantizedVectorsFormat"; + + private static String luceneScalarQuantizedVectorsFormat = + BASE + codecs + "LuceneScalarQuantizedVectorsFormat"; + private static String luceneScalarQuantizedVectorsFormatFallback = + BASE + fallbackCodecs + "LuceneScalarQuantizedVectorsFormat"; + + private static String luceneHnswScalarQuantizedVectorsFormat = + BASE + codecs + "LuceneHnswScalarQuantizedVectorsFormat"; + private static String luceneHnswScalarQuantizedVectorsFormatFallback = + BASE + fallbackCodecs + "LuceneHnswScalarQuantizedVectorsFormat"; private static String luceneCodec = BASE + codecs + "LuceneCodec"; - private static String luceneCodecFallback = BASE + backwardCodecs + "LuceneCodec"; + private static String luceneCodecFallback = BASE + fallbackCodecs + "LuceneCodec"; private static LuceneProvider instance; @@ -67,6 +87,10 @@ public class LuceneProvider { private Class hnswVectorsFormat; private Class hnswVectorsReader; private Class hnswVectorsWriter; + private Class binaryQuantizedVectorsFormat; + private Class hnswBinaryQuantizedVectorsFormat; + private Class scalarQuantizedVectorsFormat; + private Class hnswScalarQuantizedVectorsFormat; public static LuceneProvider getInstance(String version) throws ClassNotFoundException { if (instance == null) { @@ -78,7 +102,7 @@ public static LuceneProvider getInstance(String version) throws ClassNotFoundExc private LuceneProvider(String version) throws ClassNotFoundException { flatVectorsFormat = loadClass( - setVersion(luceneflatVectorsFormat, version), + setVersion(luceneFlatVectorsFormat, version), setVersion(luceneFlatVectorsFormatFallback, version)); hnswVectorsFormat = loadClass( @@ -92,6 +116,27 @@ private LuceneProvider(String version) throws ClassNotFoundException { loadClass( setVersion(luceneHnswVectorsWriter, version), setVersion(luceneHnswVectorsWriterFallback, version)); + scalarQuantizedVectorsFormat = + loadClass( + setVersion(luceneScalarQuantizedVectorsFormat, version), + setVersion(luceneScalarQuantizedVectorsFormatFallback, version)); + + hnswScalarQuantizedVectorsFormat = + loadClass( + setVersion(luceneHnswScalarQuantizedVectorsFormat, version), + setVersion(luceneHnswScalarQuantizedVectorsFormatFallback, version)); + + // TODO: Find a better way if possible, but as a separate initiative. + if ("102".equals(version)) { + binaryQuantizedVectorsFormat = + loadClass( + setVersion(luceneBinaryQuantizedVectorsFormat, version), + setVersion(luceneBinaryQuantizedVectorsFormatFallback, version)); + hnswBinaryQuantizedVectorsFormat = + loadClass( + setVersion(luceneHnswBinaryQuantizedVectorsFormat, version), + setVersion(luceneHnswBinaryQuantizedVectorsFormatFallback, version)); + } } private static String setVersion(String pkg, String version) { @@ -199,4 +244,60 @@ public List getSimilarityFunctions() throw e; } } + + public FlatVectorsFormat getluceneBinaryQuantizedVectorsFormatInstance() throws Exception { + try { + Constructor luceneBinaryQuantizedVectorsFormatConstructor = + binaryQuantizedVectorsFormat.getConstructor(); + return (FlatVectorsFormat) luceneBinaryQuantizedVectorsFormatConstructor.newInstance(); + } catch (Exception e) { + log.log( + Level.SEVERE, + "Unable to initialize LuceneBinaryQuantizedVectorsFormat: " + e.getMessage()); + throw e; + } + } + + public FlatVectorsFormat getLuceneHnswBinaryQuantizedVectorsFormatInstance( + int maxConn, int beamWidth) throws Exception { + try { + Constructor luceneHnswBinaryQuantizedVectorsFormatConstructor = + hnswBinaryQuantizedVectorsFormat.getConstructor(Integer.TYPE, Integer.TYPE); + return (FlatVectorsFormat) + luceneHnswBinaryQuantizedVectorsFormatConstructor.newInstance(maxConn, beamWidth); + } catch (Exception e) { + log.log( + Level.SEVERE, + "Unable to initialize LuceneBinaryQuantizedVectorsFormat: " + e.getMessage()); + throw e; + } + } + + public FlatVectorsFormat getLuceneScalarQuantizedVectorsFormatInstance() throws Exception { + try { + Constructor luceneScalarQuantizedVectorsFormatConstructor = + scalarQuantizedVectorsFormat.getConstructor(); + return (FlatVectorsFormat) luceneScalarQuantizedVectorsFormatConstructor.newInstance(); + } catch (Exception e) { + log.log( + Level.SEVERE, + "Unable to initialize LuceneScalarQuantizedVectorsFormat: " + e.getMessage()); + throw e; + } + } + + public FlatVectorsFormat getLuceneHnswScalarQuantizedVectorsFormatInstance( + int beamWidth, int maxConn) throws Exception { + try { + Constructor luceneHnswScalarQuantizedVectorsFormatConstructor = + hnswScalarQuantizedVectorsFormat.getConstructor(Integer.TYPE, Integer.TYPE); + return (FlatVectorsFormat) + luceneHnswScalarQuantizedVectorsFormatConstructor.newInstance(beamWidth, maxConn); + } catch (Exception e) { + log.log( + Level.SEVERE, + "Unable to initialize LuceneHnswScalarQuantizedVectorsFormat: " + e.getMessage()); + throw e; + } + } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/QuantizedFieldWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/QuantizedFieldWriter.java new file mode 100644 index 0000000000..8a6bbfed27 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/QuantizedFieldWriter.java @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.quantizeFloatVectorsToBinary; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.quantizeFloatVectorsToScalar; + +import com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.QuantizationType; +import java.io.IOException; +import java.util.List; +import org.apache.lucene.codecs.KnnFieldVectorsWriter; +import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; +import org.apache.lucene.index.DocsWithFieldSet; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.util.RamUsageEstimator; + +public class QuantizedFieldWriter extends KnnFieldVectorsWriter { + + private static final long SHALLOW_SIZE = + RamUsageEstimator.shallowSizeOfInstance(QuantizedFieldWriter.class); + + private final FieldInfo fieldInfo; + private final FlatFieldVectorsWriter flatFieldVectorsWriter; + private int lastDocID = -1; + private QuantizationType quantizationType; + + @SuppressWarnings("unchecked") + public QuantizedFieldWriter( + QuantizationType quantizationType, + FieldInfo fieldInfo, + FlatFieldVectorsWriter flatFieldVectorsWriter) { + this.quantizationType = quantizationType; + this.fieldInfo = fieldInfo; + this.flatFieldVectorsWriter = (FlatFieldVectorsWriter) flatFieldVectorsWriter; + } + + @Override + public void addValue(int docID, Object vectorValue) throws IOException { + if (docID == lastDocID) { + throw new IllegalArgumentException( + "VectorValuesField \"" + + fieldInfo.name + + "\" appears more than once in this document (only one value is allowed per" + + " field)"); + } + flatFieldVectorsWriter.addValue(docID, (float[]) vectorValue); + } + + List getVectors() { + if (quantizationType == QuantizationType.BINARY) { + return quantizeFloatVectorsToBinary(flatFieldVectorsWriter.getVectors()); + } else { + return quantizeFloatVectorsToScalar(flatFieldVectorsWriter.getVectors()); + } + } + + FieldInfo fieldInfo() { + return fieldInfo; + } + + DocsWithFieldSet getDocsWithFieldSet() { + return flatFieldVectorsWriter.getDocsWithFieldSet(); + } + + @Override + public Object copyValue(Object vectorValue) { + throw new UnsupportedOperationException(); + } + + @Override + public long ramBytesUsed() { + return SHALLOW_SIZE + flatFieldVectorsWriter.ramBytesUsed(); + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java index 74d2cbc558..21898f8ee4 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; @@ -65,6 +65,60 @@ static CuVSMatrix createFloatMatrix(List data, int dimensions, CuVSReso return builder.build(); } + /** + * A method to build a CuVSMatrix from a list of byte vectors (for binary quantized vectors). + * + * Uses CuVSMatrix.Builder to copy vectors directly to device memory + * without creating intermediate heap arrays. + * + * @param data The byte vectors (packed bits for binary quantization) + * @param bytesPerVector The number of bytes in each vector + * @param resources The CuVS resources for device matrix creation + * @return an instance of CuVSMatrix with BYTE data type + */ + static CuVSMatrix createByteMatrix( + List data, int bytesPerVector, CuVSResources resources) { + // Use Builder pattern to avoid intermediate byte[][] allocation + // and copy directly from List to device memory + CuVSMatrix.Builder builder = + CuVSMatrix.deviceBuilder( + resources, + data.size(), // rows (number of vectors) + bytesPerVector, // columns (bytes per vector) + CuVSMatrix.DataType.BYTE); + + // Add vectors one by one - builder copies directly to device memory + for (byte[] vector : data) { + builder.addVector(vector); + } + + return builder.build(); + } + + /** + * A method to build a CuVSMatrix from a 2D byte array (for binary quantized vectors). + * + * @param data The 2D byte array (packed bits for binary quantization) + * @param bytesPerVector The number of bytes in each vector + * @param resources The CuVS resources for device matrix creation + * @return an instance of CuVSMatrix with BYTE data type + */ + static CuVSMatrix createByteMatrixFromArray( + byte[][] data, int bytesPerVector, CuVSResources resources) { + CuVSMatrix.Builder builder = + CuVSMatrix.deviceBuilder( + resources, + data.length, // rows (number of vectors) + bytesPerVector, // columns (bytes per vector) + CuVSMatrix.DataType.BYTE); + + // Add vectors one by one - builder copies directly to device memory + for (byte[] vector : data) { + builder.addVector(vector); + } + return builder.build(); + } + /** * A utility method to convert nanoseconds to milliseconds. * diff --git a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index f01d2e8da9..faa0684c04 100644 --- a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -1,5 +1,7 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 com.nvidia.cuvs.lucene.Lucene101AcceleratedHNSWCodec com.nvidia.cuvs.lucene.CuVS2510GPUSearchCodec +com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWBinaryQuantizedCodec +com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWScalarQuantizedCodec diff --git a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat index 4e20fcb599..6625ac72a4 100644 --- a/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat +++ b/java/cuvs-lucene/src/main/resources/META-INF/services/org.apache.lucene.codecs.KnnVectorsFormat @@ -1,7 +1,9 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat +com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat +com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWScalarQuantizedVectorsFormat diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java new file mode 100644 index 0000000000..4c3c280993 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java @@ -0,0 +1,212 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; +import static org.apache.lucene.index.VectorSimilarityFunction.COSINE; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.VectorEncoding; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.BeforeClass; +import org.junit.Ignore; + +@SuppressSysoutChecks(bugUrl = "") +public class TestQuantizedVectorsFormats extends BaseKnnVectorsFormatTestCase { + + private static final Logger log = Logger.getLogger(TestQuantizedVectorsFormats.class.getName()); + + private static KnnVectorsFormat knnVectorsFormat; + + public TestQuantizedVectorsFormats(@Name("knnVectorsWriter") KnnVectorsFormat knnVectorsFormat) { + TestQuantizedVectorsFormats.knnVectorsFormat = knnVectorsFormat; + } + + @ParametersFactory + public static List parameters() { + return Arrays.asList( + new Object[][] { + {new LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat()}, + {new LuceneAcceleratedHNSWScalarQuantizedVectorsFormat()} + }); + } + + @BeforeClass + public static void beforeClass() { + assumeTrue("cuVS is not supported so skipping these tests", isSupported()); + } + + @Override + protected Codec getCodec() { + log.log(Level.FINE, "Running tests for: " + knnVectorsFormat.getName()); + return TestUtil.alwaysKnnVectorsFormat(knnVectorsFormat); + } + + public void testMergeTwoSegsWithASingleDocPerSeg() throws Exception { + final int R = 2, D = 128; + float[][] f = new float[R][D]; + final String F = "f"; + for (int i = 0; i < R; i++) { + f[i] = randomVector(D); + } + + try (Directory dir = newDirectory(new ByteBuffersDirectory()); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + for (int i = 0; i < R; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.NO)); + doc.add(new KnnFloatVectorField(F, f[i], EUCLIDEAN)); + w.addDocument(doc); + w.commit(); + } + w.flush(); + + try (DirectoryReader reader = DirectoryReader.open(w)) { + List subReaders = reader.leaves(); + assertEquals(2, subReaders.size()); + for (int i = 0; i < R; i++) { + assertEquals(1, subReaders.get(i).reader().getFloatVectorValues(F).size()); + } + } + w.forceMerge(1); + + try (DirectoryReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues values = r.getFloatVectorValues(F); + assertNotNull(values); + assertEquals(R, values.size()); + for (int i = 0; i < R; i++) { + assertArrayEquals(f[i], values.vectorValue(i), 0.0f); + } + } + } + } + + public void testTwoVectorFieldsPerDoc() throws Exception { + final int R = 2, D = 128; + final String F1 = "f1", F2 = "f2"; + float[][] f1 = new float[R][D]; + float[][] f2 = new float[R][D]; + + for (int i = 0; i < R; i++) { + f1[i] = randomVector(D); + f2[i] = randomVector(D); + } + + try (Directory dir = newDirectory(new ByteBuffersDirectory()); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + + for (int i = 0; i < R; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.NO)); + doc.add(new KnnFloatVectorField(F1, f1[i], EUCLIDEAN)); + doc.add(new KnnFloatVectorField(F2, f2[i], EUCLIDEAN)); + w.addDocument(doc); + } + w.forceMerge(1); + + try (DirectoryReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + FloatVectorValues values = r.getFloatVectorValues(F1); + assertNotNull(values); + assertEquals(R, values.size()); + for (int i = 0; i < R; i++) { + assertArrayEquals(f1[i], values.vectorValue(i), 0.0f); + } + + values = r.getFloatVectorValues(F2); + assertNotNull(values); + assertEquals(R, values.size()); + for (int i = 0; i < R; i++) { + assertArrayEquals(f2[i], values.vectorValue(i), 0.0f); + } + } + } + } + + public void testCosineSimilarity() throws Exception { + final int R = 2, D = 128; + final String F = "f"; + float[][] f = new float[R][D]; + for (int i = 0; i < R; i++) { + f[i] = randomVector(D); + } + + try (Directory dir = newDirectory(new ByteBuffersDirectory()); + IndexWriter w = new IndexWriter(dir, newIndexWriterConfig())) { + + for (int i = 0; i < R; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.NO)); + doc.add(new KnnFloatVectorField(F, f[i], COSINE)); + w.addDocument(doc); + } + w.forceMerge(1); + + try (DirectoryReader reader = DirectoryReader.open(w)) { + LeafReader r = getOnlyLeafReader(reader); + + FloatVectorValues values = r.getFloatVectorValues(F); + assertNotNull(values); + assertEquals(R, values.size()); + + float[] queryVector = randomVector(D); + var topDocs = r.searchNearestVectors(F, queryVector, 2, null, 10); + assertTrue("Should return at least one result", topDocs.scoreDocs.length > 0); + assertTrue("Scores should be non-negative", topDocs.scoreDocs[0].score >= 0); + } + } + } + + @Override + protected VectorEncoding randomVectorEncoding() { + return VectorEncoding.FLOAT32; + } + + @Ignore + @Override + public void testByteVectorScorerIteration() {} + + @Ignore + @Override + public void testEmptyByteVectorData() {} + + @Ignore + @Override + public void testMergingWithDifferentByteKnnFields() {} + + @Ignore + @Override + public void testMismatchedFields() {} + + @Ignore + @Override + public void testRandomBytes() {} + + @Ignore + @Override + public void testSortedIndexBytes() {} +} From 4222db2196350b9fcde99847087667d1fae5bdc6 Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Mon, 26 Jan 2026 14:03:26 -0500 Subject: [PATCH 064/105] fix(build): build package on merge to `release/*` branch (NVIDIA/cuvs-lucene#101) ## Description We should be building packages when commits are merged into the `release/` branches, otherwise projects can get stuck waiting for nightlies. Additionally, some packages like `rapids-dask-dependency` don't get built in the nightly runs. xref: https://github.com/rapidsai/build-planning/issues/224 --- java/cuvs-lucene/.github/workflows/build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 1d6b556e0c..94bd6ce708 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -4,6 +4,7 @@ on: push: branches: - "main" + - "release/*" tags: - v[0-9][0-9].[0-9][0-9].[0-9][0-9] workflow_dispatch: From 3d3043bb3d5ef9b6c0ce622873f03972fdf82933 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 27 Jan 2026 21:50:36 -0500 Subject: [PATCH 065/105] Make CAGRA graph build algo configurable (NVIDIA/cuvs-lucene#73) There was a need to let the user select which CAGRA build algorithm to use. The code changes in this PR allow an additional `CagraGraphBuildAlgo` parameter to be passed via the Codecs and Formats. Fixes NVIDIA/cuvs-lucene#58 32cbd7b are all the changes from NVIDIA/cuvs-lucene#72, NVIDIA/cuvs-lucene#71, and NVIDIA/cuvs-lucene#69 PRs. Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/73 --- .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 72 ++++++++++++----- .../cuvs/lucene/CuVS2510GPUVectorsFormat.java | 21 ++++- .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 7 +- .../lucene/Lucene101AcceleratedHNSWCodec.java | 30 +++---- .../Lucene99AcceleratedHNSWVectorsFormat.java | 13 ++- .../Lucene99AcceleratedHNSWVectorsWriter.java | 6 +- ...TestCagraToHnswSerializationAndSearch.java | 14 +--- ...ializationAndSearchWithFallbackWriter.java | 12 +-- .../com/nvidia/cuvs/lucene/TestMerge.java | 80 ++++++++++++++----- 9 files changed, 170 insertions(+), 85 deletions(-) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index 57b1e3c9cf..4e4f52e3a3 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -1,12 +1,12 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.LibraryException; import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; -import java.lang.reflect.InvocationTargetException; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; @@ -27,46 +27,74 @@ public class CuVS2510GPUSearchCodec extends FilterCodec { private static final int DEFAULT_CUVS_WRITER_THREADS = 1; private static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; private static final int DEFAULT_GRAPH_DEGREE = 64; + private static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = + CagraGraphBuildAlgo.NN_DESCENT; private static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; private KnnVectorsFormat format; /** * Default constructor for {@link CuVS2510GPUSearchCodec} - * @throws InvocationTargetException - * @throws IllegalArgumentException - * @throws IllegalAccessException - * @throws InstantiationException - * @throws SecurityException - * @throws NoSuchMethodException - * @throws ClassNotFoundException + * + * @throws Exception */ - public CuVS2510GPUSearchCodec() - throws ClassNotFoundException, - NoSuchMethodException, - SecurityException, - InstantiationException, - IllegalAccessException, - IllegalArgumentException, - InvocationTargetException { + public CuVS2510GPUSearchCodec() throws Exception { this(NAME, LuceneProvider.getCodec("101")); } /** - * Constructor for the {@link CuVS2510GPUSearchCodec} + * Initialize {@link CuVS2510GPUSearchCodec} with default parameter values. * * @param name the name of the codec * @param delegate the delegate codec */ public CuVS2510GPUSearchCodec(String name, Codec delegate) { super(name, delegate); + initializeFormat( + DEFAULT_INTERMEDIATE_GRAPH_DEGREE, + DEFAULT_GRAPH_DEGREE, + DEFAULT_CUVS_WRITER_THREADS, + DEFAULT_CAGRA_GRAPH_BUILD_ALGO, + DEFAULT_INDEX_TYPE); + } + + /** + * Initialize the codec with custom parameter values. + * + * @param cuvsWriterThreads The number of cuVS writer threads to use while indexing. + * @param intermediateGraphDegree The intermediate graph degree to use while indexing. + * @param graphDegree The graph degree to use while indexing. + * @param cagraGraphBuildAlgo The CAGRA graph build algo to use. + * @param indexType Type of Index to build. + * + * @throws Exception Exception raised when initializing the codec. + */ + public CuVS2510GPUSearchCodec( + int cuvsWriterThreads, + int intermediateGraphDegree, + int graphDegree, + CagraGraphBuildAlgo cagraGraphBuildAlgo, + IndexType indexType) + throws Exception { + this(NAME, LuceneProvider.getCodec("101")); + initializeFormat( + cuvsWriterThreads, intermediateGraphDegree, graphDegree, cagraGraphBuildAlgo, indexType); + } + + private void initializeFormat( + int cuvsWriterThreads, + int intermediateGraphDegree, + int graphDegree, + CagraGraphBuildAlgo cagraGraphBuildAlgo, + IndexType indexType) { try { format = new CuVS2510GPUVectorsFormat( - DEFAULT_CUVS_WRITER_THREADS, - DEFAULT_INTERMEDIATE_GRAPH_DEGREE, - DEFAULT_GRAPH_DEGREE, - DEFAULT_INDEX_TYPE); + cuvsWriterThreads, + intermediateGraphDegree, + graphDegree, + cagraGraphBuildAlgo, + indexType); setKnnFormat(format); } catch (LibraryException ex) { log.log( diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java index 3048867090..62a89f4276 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java @@ -6,6 +6,7 @@ import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.assertIsSupported; +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.LibraryException; import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; import java.io.IOException; @@ -38,6 +39,7 @@ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { static final int DEFAULT_WRITER_THREADS = 32; static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; static final int DEFAULT_GRAPH_DEGREE = 64; + static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = CagraGraphBuildAlgo.NN_DESCENT; static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; static final LuceneProvider LUCENE_PROVIDER; @@ -47,6 +49,7 @@ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { final int cuvsWriterThreads; final int intGraphDegree; final int graphDegree; + final CagraGraphBuildAlgo cagraGraphBuildAlgo; final CuVS2510GPUVectorsWriter.IndexType indexType; // the index type to build, when writing static { @@ -69,6 +72,7 @@ public CuVS2510GPUVectorsFormat() { DEFAULT_WRITER_THREADS, DEFAULT_INTERMEDIATE_GRAPH_DEGREE, DEFAULT_GRAPH_DEGREE, + DEFAULT_CAGRA_GRAPH_BUILD_ALGO, DEFAULT_INDEX_TYPE); } @@ -78,16 +82,22 @@ public CuVS2510GPUVectorsFormat() { * @param cuvsWriterThreads the number of cuVS writer threads to use * @param intGraphDegree the intermediate graph degree for building the CAGRA index * @param graphDegree the graph degree for building the CAGRA index + * @param cagraGraphBuildAlgo the CAGRA graph build algorithm to use * @param indexType the {@link com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType} * * @throws LibraryException if the native library fails to load */ public CuVS2510GPUVectorsFormat( - int cuvsWriterThreads, int intGraphDegree, int graphDegree, IndexType indexType) { + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + CagraGraphBuildAlgo cagraGraphBuildAlgo, + IndexType indexType) { super("CuVS2510GPUVectorsFormat"); this.cuvsWriterThreads = cuvsWriterThreads; this.intGraphDegree = intGraphDegree; this.graphDegree = graphDegree; + this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; this.indexType = indexType; } @@ -99,7 +109,13 @@ public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOE assertIsSupported(); var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); return new CuVS2510GPUVectorsWriter( - state, cuvsWriterThreads, intGraphDegree, graphDegree, indexType, flatWriter); + state, + cuvsWriterThreads, + intGraphDegree, + graphDegree, + cagraGraphBuildAlgo, + indexType, + flatWriter); } /** @@ -128,6 +144,7 @@ public String toString() { sb.append("(cuvsWriterThreads=").append(cuvsWriterThreads); sb.append("intGraphDegree=").append(intGraphDegree); sb.append("graphDegree=").append(graphDegree); + sb.append("cagraGraphBuildAlgo=").append(cagraGraphBuildAlgo); sb.append(")"); return sb.toString(); } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index b2a94a845a..ca6a05a524 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -78,6 +78,8 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { private final int cuvsWriterThreads; private final int intGraphDegree; private final int graphDegree; + private final CagraGraphBuildAlgo cagraGraphBuildAlgo; + private final IndexType indexType; private final FlatVectorsWriter flatVectorsWriter; private final List fields = new ArrayList<>(); @@ -140,6 +142,7 @@ public boolean bruteForce() { * @param cuvsWriterThreads the number of cuVS writer threads * @param intGraphDegree the intermediate graph degree for building the CAGRA index * @param graphDegree the graph degree for building the CAGRA index + * @param cagraGraphBuildAlgo the CAGRA graph build algorithm to use * @param indexType the IndexType * @param flatVectorsWriter instance of FlatVectorsWriter * @@ -150,6 +153,7 @@ public CuVS2510GPUVectorsWriter( int cuvsWriterThreads, int intGraphDegree, int graphDegree, + CagraGraphBuildAlgo cagraGraphBuildAlgo, IndexType indexType, FlatVectorsWriter flatVectorsWriter) throws IOException { @@ -158,6 +162,7 @@ public CuVS2510GPUVectorsWriter( this.cuvsWriterThreads = cuvsWriterThreads; this.intGraphDegree = intGraphDegree; this.graphDegree = graphDegree; + this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; this.flatVectorsWriter = flatVectorsWriter; this.infoStream = state.infoStream; @@ -243,7 +248,7 @@ private CagraIndexParams cagraIndexParams(int size) { .withNumWriterThreads(cuvsWriterThreads) .withIntermediateGraphDegree(intGraphDegree) .withGraphDegree(graphDegree) - .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) + .withCagraGraphBuildAlgo(cagraGraphBuildAlgo) .build(); } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java index 9b0db6930b..76e92b76c7 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java @@ -1,11 +1,10 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; import com.nvidia.cuvs.LibraryException; -import java.lang.reflect.InvocationTargetException; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; @@ -42,14 +41,7 @@ public class Lucene101AcceleratedHNSWCodec extends FilterCodec { } } - public Lucene101AcceleratedHNSWCodec() - throws ClassNotFoundException, - NoSuchMethodException, - SecurityException, - InstantiationException, - IllegalAccessException, - IllegalArgumentException, - InvocationTargetException { + public Lucene101AcceleratedHNSWCodec() throws Exception { this(NAME, LuceneProvider.getCodec("101")); } @@ -65,13 +57,7 @@ public Lucene101AcceleratedHNSWCodec( int hnswLayers, int maxConn, int beamWidth) - throws ClassNotFoundException, - NoSuchMethodException, - SecurityException, - InstantiationException, - IllegalAccessException, - IllegalArgumentException, - InvocationTargetException { + throws Exception { this(NAME, LuceneProvider.getCodec("101")); initializeFormat( cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); @@ -106,11 +92,21 @@ private void initializeFormat( } } + /** + * Get the configured {@link KnnVectorsFormat} + * + * @return the instance of the {@link KnnVectorsFormat} + */ @Override public KnnVectorsFormat knnVectorsFormat() { return format; } + /** + * Set the {@link KnnVectorsFormat} + * + * @param format the {@link KnnVectorsFormat} to set + */ public void setKnnFormat(KnnVectorsFormat format) { this.format = format; } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java index b8b26b33e7..a2a1f2f0b7 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java @@ -6,6 +6,7 @@ import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.LibraryException; import java.io.IOException; import java.util.logging.Level; @@ -31,6 +32,7 @@ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { static final int DEFAULT_WRITER_THREADS = 32; static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; static final int DEFAULT_GRAPH_DEGREE = 64; + static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = CagraGraphBuildAlgo.NN_DESCENT; static final int DEFAULT_HNSW_GRAPH_LAYERS = 1; static final String HNSW_META_CODEC_NAME = "Lucene99HnswVectorsFormatMeta"; @@ -48,6 +50,8 @@ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { private final int cuvsWriterThreads; private final int intGraphDegree; private final int graphDegree; + // This default setting will be removed once I figure out the root cause of a related bug. + private final CagraGraphBuildAlgo cagraGraphBuildAlgo = DEFAULT_CAGRA_GRAPH_BUILD_ALGO; private final int hnswLayers; private final int maxConn; private final int beamWidth; @@ -115,7 +119,13 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException if (isSupported()) { log.log(Level.FINE, "cuVS is supported so using the Lucene99AcceleratedHNSWVectorsWriter"); return new Lucene99AcceleratedHNSWVectorsWriter( - state, cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, flatWriter); + state, + cuvsWriterThreads, + intGraphDegree, + graphDegree, + cagraGraphBuildAlgo, + hnswLayers, + flatWriter); } else { log.log( Level.WARNING, @@ -162,6 +172,7 @@ public String toString() { sb.append("(cuvsWriterThreads=").append(cuvsWriterThreads); sb.append("intGraphDegree=").append(intGraphDegree); sb.append("graphDegree=").append(graphDegree); + sb.append("cagraGraphBuildAlgo=").append(cagraGraphBuildAlgo); sb.append("hnswLayers=").append(hnswLayers); sb.append(")"); return sb.toString(); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index 0603da6195..d5a944c184 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -82,6 +82,7 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { private final int cuvsWriterThreads; private final int intGraphDegree; private final int graphDegree; + private final CagraGraphBuildAlgo cagraGraphBuildAlgo; private final int hnswLayers; // Number of layers to create in CAGRA->HNSW conversion private final FlatVectorsWriter flatVectorsWriter; // for writing the raw vectors private final List fields = new ArrayList<>(); @@ -109,6 +110,7 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { * @param cuvsWriterThreads number of cuVS threads to use while building the intermediate CAGRA index * @param intGraphDegree the intermediate graph degree to use while building the CAGRA index * @param graphDegree the graph degree to use while building the CAGRA index + * @param cagraGraphBuildAlgo the CAGRA graph build algorithm to use * @param hnswLayers the number of hnsw layers to construct while building the HNSW graph * @param flatVectorsWriter instance of the {@link org.apache.lucene.codecs.hnsw.FlatVectorsWriter} * @throws IOException IOException @@ -118,6 +120,7 @@ public Lucene99AcceleratedHNSWVectorsWriter( int cuvsWriterThreads, int intGraphDegree, int graphDegree, + CagraGraphBuildAlgo cagraGraphBuildAlgo, int hnswLayers, FlatVectorsWriter flatVectorsWriter) throws IOException { @@ -125,6 +128,7 @@ public Lucene99AcceleratedHNSWVectorsWriter( this.cuvsWriterThreads = cuvsWriterThreads; this.intGraphDegree = intGraphDegree; this.graphDegree = graphDegree; + this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; this.hnswLayers = hnswLayers; this.flatVectorsWriter = flatVectorsWriter; this.infoStream = state.infoStream; @@ -207,7 +211,7 @@ private CagraIndexParams cagraIndexParams() { .withNumWriterThreads(cuvsWriterThreads) .withIntermediateGraphDegree(intGraphDegree) .withGraphDegree(graphDegree) - .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) + .withCagraGraphBuildAlgo(cagraGraphBuildAlgo) .build(); } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java index 1fa566972b..45cefeec7b 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java @@ -9,8 +9,6 @@ import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -60,16 +58,8 @@ public void beforeTest() throws Exception { } @Test - public void testCagraToHnswSerializationAndSearch() - throws IOException, - ClassNotFoundException, - NoSuchMethodException, - SecurityException, - InstantiationException, - IllegalAccessException, - IllegalArgumentException, - InvocationTargetException { - Codec codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); + public void testCagraToHnswSerializationAndSearch() throws Exception { + Codec codec = new Lucene101AcceleratedHNSWCodec(1, 128, 64, 3, 16, 100); IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); final int COMMIT_FREQ = 2000; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java index 3d3755f7de..81e2627b8d 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java @@ -11,8 +11,6 @@ import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; @@ -65,15 +63,7 @@ public static void beforeClass() throws Exception { } @Test - public void testCagraToHnswSerializationAndSearchWithFallbackWriter() - throws IOException, - ClassNotFoundException, - NoSuchMethodException, - SecurityException, - InstantiationException, - IllegalAccessException, - IllegalArgumentException, - InvocationTargetException { + public void testCagraToHnswSerializationAndSearchWithFallbackWriter() throws Exception { Codec codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java index cdba23b170..8e64e9fb1b 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java @@ -7,9 +7,13 @@ import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; import static org.apache.lucene.tests.util.TestUtil.alwaysKnnVectorsFormat; +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Random; import java.util.logging.Level; @@ -59,6 +63,18 @@ public class TestMerge extends LuceneTestCase { private static final int MAX_VECTOR_DIMENSION = 256; private static final int TOP_K_LIMIT = 64; + private static CagraGraphBuildAlgo cagraGraphBuildAlgo; + + public TestMerge(@Name("cagraGraphBuildAlgo") CagraGraphBuildAlgo cagraGraphBuildAlgo) { + TestMerge.cagraGraphBuildAlgo = cagraGraphBuildAlgo; + } + + @ParametersFactory + public static List parameters() { + return Arrays.asList( + new Object[][] {{CagraGraphBuildAlgo.NN_DESCENT}, {CagraGraphBuildAlgo.IVF_PQ}}); + } + @BeforeClass public static void beforeClass() { assumeTrue("cuVS is not supported", isSupported()); @@ -94,7 +110,10 @@ public void tearDown() throws Exception { **/ @Test public void testMergeManyDocumentsMultipleSegments() throws IOException { - log.log(Level.FINE, "Starting testMergeManyDocumentsMultipleSegments"); + log.log( + Level.FINE, + "Starting testMergeManyDocumentsMultipleSegments with CagraGraphBuildAlgo: " + + cagraGraphBuildAlgo); // Randomize configuration parameters int maxBufferedDocs = 5 + random().nextInt(16); // 5-20 docs per buffer @@ -118,9 +137,12 @@ public void testMergeManyDocumentsMultipleSegments() throws IOException { + ", vectorProbability=" + vectorProbability); + CuVS2510GPUVectorsFormat format = + new CuVS2510GPUVectorsFormat(32, 128, 64, cagraGraphBuildAlgo, IndexType.CAGRA); + IndexWriterConfig config = new IndexWriterConfig() - .setCodec(alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat())) + .setCodec(alwaysKnnVectorsFormat(format)) .setMaxBufferedDocs(maxBufferedDocs) // Randomized buffer size .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); @@ -212,7 +234,10 @@ public void testMergeManyDocumentsMultipleSegments() throws IOException { **/ @Test public void testMergeWithIndexSorting() throws IOException { - log.log(Level.FINE, "Starting testMergeWithIndexSorting with text-based sorting"); + log.log( + Level.FINE, + "Starting testMergeWithIndexSorting with text-based sorting with CagraGraphBuildAlgo: " + + cagraGraphBuildAlgo); // Randomize sort field type SortField.Type sortType = random().nextBoolean() ? SortField.Type.STRING : SortField.Type.LONG; @@ -246,9 +271,12 @@ public void testMergeWithIndexSorting() throws IOException { + ", vectorProbability=" + vectorProbability); + CuVS2510GPUVectorsFormat format = + new CuVS2510GPUVectorsFormat(32, 128, 64, cagraGraphBuildAlgo, IndexType.CAGRA); + IndexWriterConfig config = new IndexWriterConfig() - .setCodec(alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat())) + .setCodec(alwaysKnnVectorsFormat(format)) .setIndexSort(indexSort) // This automatically enables sorting during merges .setMergePolicy(mergePolicy) .setMaxBufferedDocs(maxBufferedDocs) @@ -455,15 +483,20 @@ public void testMergeWithIndexSorting() throws IOException { **/ @Test public void testMergeWithMissingVectors() throws IOException { - log.log(Level.FINE, "Starting testMergeWithMissingVectors"); + log.log( + Level.FINE, + "Starting testMergeWithMissingVectors with CagraGraphBuildAlgo: " + cagraGraphBuildAlgo); // Randomize configuration int maxBufferedDocs = 10 + random().nextInt(11); // 10-20 docs per buffer int numSegments = 3 + random().nextInt(3); // 3-5 segments + CuVS2510GPUVectorsFormat format = + new CuVS2510GPUVectorsFormat(32, 128, 64, cagraGraphBuildAlgo, IndexType.CAGRA); + IndexWriterConfig config = new IndexWriterConfig() - .setCodec(alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat())) + .setCodec(alwaysKnnVectorsFormat(format)) .setMaxBufferedDocs(maxBufferedDocs) .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); @@ -583,7 +616,9 @@ public void testMergeWithMissingVectors() throws IOException { **/ @Test public void testMergeWithDeletions() throws IOException { - log.log(Level.FINE, "Starting testMergeWithDeletions"); + log.log( + Level.FINE, + "Starting testMergeWithDeletions with CagraGraphBuildAlgo: " + cagraGraphBuildAlgo); // Randomize configuration parameters int maxBufferedDocs = 15 + random().nextInt(11); // 15-25 docs per buffer @@ -605,9 +640,12 @@ public void testMergeWithDeletions() throws IOException { + ", deletionProbability=" + deletionProbability); + CuVS2510GPUVectorsFormat format = + new CuVS2510GPUVectorsFormat(32, 128, 64, cagraGraphBuildAlgo, IndexType.CAGRA); + IndexWriterConfig config = new IndexWriterConfig() - .setCodec(alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat())) + .setCodec(alwaysKnnVectorsFormat(format)) .setMaxBufferedDocs(maxBufferedDocs) .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); @@ -725,7 +763,9 @@ public void testMergeWithDeletions() throws IOException { * */ @Test public void testMergeBruteForceIndex() throws IOException { - log.log(Level.FINE, "Starting testMergeBruteForceIndex"); + log.log( + Level.FINE, + "Starting testMergeBruteForceIndex with CagraGraphBuildAlgo: " + cagraGraphBuildAlgo); // Randomize configuration parameters int maxBufferedDocs = 8 + random().nextInt(8); // 8-15 docs per buffer @@ -747,10 +787,7 @@ public void testMergeBruteForceIndex() throws IOException { // Configure with brute force index type CuVS2510GPUVectorsFormat bruteForceFormat = new CuVS2510GPUVectorsFormat( - 32, // writer threads - 128, // intermediate graph degree - 64, // graph degree - IndexType.BRUTE_FORCE); // Use brute force index + 32, 128, 64, cagraGraphBuildAlgo, IndexType.BRUTE_FORCE); // Use brute force index IndexWriterConfig config = new IndexWriterConfig() @@ -880,7 +917,10 @@ public void testMergeBruteForceIndex() throws IOException { * */ @Test public void testMergeCagraAndBruteForceIndex() throws IOException { - log.log(Level.FINE, "Starting testMergeCagraAndBruteForceIndex"); + log.log( + Level.FINE, + "Starting testMergeCagraAndBruteForceIndex with CagraGraphBuildAlgo: " + + cagraGraphBuildAlgo); // Use moderate dataset size int maxBufferedDocs = 15 + random().nextInt(10); // 15-24 docs per buffer @@ -903,9 +943,10 @@ public void testMergeCagraAndBruteForceIndex() throws IOException { // Configure with CAGRA + brute force combined index type CuVS2510GPUVectorsFormat combinedFormat = new CuVS2510GPUVectorsFormat( - 32, // writer threads - 128, // intermediate graph degree - 64, // graph degree + 32, + 128, + 64, + cagraGraphBuildAlgo, IndexType.CAGRA_AND_BRUTE_FORCE); // Use combined CAGRA + brute force IndexWriterConfig config = @@ -1082,9 +1123,12 @@ public void testLargeScaleMerge() throws IOException { + ", totalDocuments=" + totalDocuments); + CuVS2510GPUVectorsFormat format = + new CuVS2510GPUVectorsFormat(32, 128, 64, cagraGraphBuildAlgo, IndexType.CAGRA); + IndexWriterConfig config = new IndexWriterConfig() - .setCodec(alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat())) + .setCodec(alwaysKnnVectorsFormat(format)) .setMaxBufferedDocs(maxBufferedDocs) .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH); From 5f97c26e6a86d8d103551d86b02b2d59c5d6aaae Mon Sep 17 00:00:00 2001 From: "Corey J. Nolet" Date: Tue, 27 Jan 2026 22:05:47 -0500 Subject: [PATCH 066/105] Revise README with build and test details (NVIDIA/cuvs-lucene#108) Updated README to include building and testing instructions. Authors: - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Ben Frederickson (https://github.com/benfred) URL: https://github.com/rapidsai/cuvs-lucene/pull/108 --- java/cuvs-lucene/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/java/cuvs-lucene/README.md b/java/cuvs-lucene/README.md index bae113a8fc..d88d3401a9 100644 --- a/java/cuvs-lucene/README.md +++ b/java/cuvs-lucene/README.md @@ -6,24 +6,24 @@ This is a project for using [cuVS](https://github.com/rapidsai/cuvs), NVIDIA's G This library provides a new [KnnVectorFormat](https://lucene.apache.org/core/10_3_1/core/org/apache/lucene/codecs/KnnVectorsFormat.html) which can be plugged into a Lucene codec. -> [!CAUTION] -> This is not production ready yet. +## Building ### Prerequisites - [CUDA 12.0+](https://developer.nvidia.com/cuda-toolkit-archive), - [Maven 3.9.6+](https://maven.apache.org/download.cgi), - [JDK 22](https://jdk.java.net/archive/) -### Building ```sh mvn clean compile package ``` The artifacts would be built and available in the target / folder. -### Run Tests +### Running Tests ```sh export LD_LIBRARY_PATH={ PATH TO YOUR LOCAL libcuvs_c.so }:$LD_LIBRARY_PATH && mvn clean test ``` +## Contributing + > [!NOTE] > The code style format is automatically enforced (including the missing license header, if any) using the [Spotless maven plugin](https://github.com/diffplug/spotless/tree/main/plugin-maven). This currently happens in the maven's `validate` stage. From 3bb8782faf08346f3a5d871b7a1a57f094ee2f90 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Wed, 28 Jan 2026 13:42:51 -0500 Subject: [PATCH 067/105] Update readme (NVIDIA/cuvs-lucene#111) Cherry-picked commit from NVIDIA/cuvs-lucene#108 to make similar changes on the `release/26.02` branch. @cjnolet FYI. Authors: - Vivek Narang (https://github.com/narangvivek10) - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/111 --- java/cuvs-lucene/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/java/cuvs-lucene/README.md b/java/cuvs-lucene/README.md index bae113a8fc..d88d3401a9 100644 --- a/java/cuvs-lucene/README.md +++ b/java/cuvs-lucene/README.md @@ -6,24 +6,24 @@ This is a project for using [cuVS](https://github.com/rapidsai/cuvs), NVIDIA's G This library provides a new [KnnVectorFormat](https://lucene.apache.org/core/10_3_1/core/org/apache/lucene/codecs/KnnVectorsFormat.html) which can be plugged into a Lucene codec. -> [!CAUTION] -> This is not production ready yet. +## Building ### Prerequisites - [CUDA 12.0+](https://developer.nvidia.com/cuda-toolkit-archive), - [Maven 3.9.6+](https://maven.apache.org/download.cgi), - [JDK 22](https://jdk.java.net/archive/) -### Building ```sh mvn clean compile package ``` The artifacts would be built and available in the target / folder. -### Run Tests +### Running Tests ```sh export LD_LIBRARY_PATH={ PATH TO YOUR LOCAL libcuvs_c.so }:$LD_LIBRARY_PATH && mvn clean test ``` +## Contributing + > [!NOTE] > The code style format is automatically enforced (including the missing license header, if any) using the [Spotless maven plugin](https://github.com/diffplug/spotless/tree/main/plugin-maven). This currently happens in the maven's `validate` stage. From cb4fe5a0eac72beae6bca66534844f4ae9d9e80a Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Wed, 28 Jan 2026 23:24:52 -0500 Subject: [PATCH 068/105] Add a multi-threaded test for searching on the GPU (NVIDIA/cuvs-lucene#104) Fixes NVIDIA/cuvs-lucene#103 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/104 --- .../TestMultithreadedCuVSGPUSearch.java | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMultithreadedCuVSGPUSearch.java diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMultithreadedCuVSGPUSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMultithreadedCuVSGPUSearch.java new file mode 100644 index 0000000000..25dc13b88a --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMultithreadedCuVSGPUSearch.java @@ -0,0 +1,141 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Random; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.English; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.apache.lucene.tests.util.TestUtil; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestMultithreadedCuVSGPUSearch extends LuceneTestCase { + + private static final Logger log = + Logger.getLogger(TestMultithreadedCuVSGPUSearch.class.getName()); + private static final Codec codec = + TestUtil.alwaysKnnVectorsFormat(new CuVS2510GPUVectorsFormat()); + private static final String VECTOR_FIELD = "vectors"; + + private static Directory directory; + private static Random random; + private static BlockingQueue queries; + private static int numQueries; + private static int topK; + private static int numThreads; + + @BeforeClass + public static void beforeClass() throws IOException { + assumeTrue("cuVS not supported", isSupported()); + random = random(); + directory = newDirectory(new ByteBuffersDirectory()); + IndexWriterConfig config = new IndexWriterConfig().setCodec(codec); + IndexWriter writer = new IndexWriter(directory, config); + + int datasetSize = random.nextInt(500, 2000); + int dimensions = random.nextInt(64, 256); + topK = random.nextInt(2, 30); + log.log(Level.FINE, "Using topK as: " + topK); + numThreads = random.nextInt(2, 8); + log.log(Level.FINE, "Generating a dataset with " + datasetSize + " vectors"); + float[][] dataset = generateDataset(random, datasetSize, dimensions); + numQueries = random.nextInt(100, 500); + log.log(Level.FINE, "Generating a query set with " + numQueries + " queries"); + float[][] queryVectors = generateDataset(random, numQueries, dimensions); + queries = new ArrayBlockingQueue<>(numQueries, true, Arrays.asList(queryVectors)); + + log.log(Level.FINE, "Indexing " + datasetSize + " vectors"); + for (int i = 0; i < datasetSize; i++) { + Document doc = new Document(); + doc.add(new StringField("id", String.valueOf(i), Field.Store.YES)); + doc.add(newTextField("text_field", English.intToEnglish(i), Field.Store.YES)); + doc.add(new KnnFloatVectorField(VECTOR_FIELD, dataset[i], EUCLIDEAN)); + writer.addDocument(doc); + } + writer.commit(); + writer.close(); + } + + @Test + public void testMultithreadedCuVSGPUSearch() throws Exception { + DirectoryReader reader = DirectoryReader.open(directory); + ExecutorService executorService = Executors.newFixedThreadPool(numThreads); + IndexSearcher searcher = new IndexSearcher(reader); + CountDownLatch latch = new CountDownLatch(numThreads); + AtomicInteger totalSuccessfulQueries = new AtomicInteger(); + log.log(Level.FINE, "Using " + numThreads + " threads"); + for (int i = 0; i < numThreads; i++) { + executorService.execute( + new Runnable() { + public void run() { + try { + float[] queryVector; + String threadName = Thread.currentThread().getName(); + while ((queryVector = queries.poll()) != null) { + log.log(Level.FINE, "Thread: " + threadName + ", queue size: " + queries.size()); + log.log(Level.FINER, "Query: " + Arrays.toString(queryVector)); + GPUKnnFloatVectorQuery query = + new GPUKnnFloatVectorQuery(VECTOR_FIELD, queryVector, topK, null, topK, 1); + ScoreDoc[] hits = searcher.search(query, topK).scoreDocs; + totalSuccessfulQueries.addAndGet(hits.length == topK ? 1 : 0); + } + } catch (Exception e) { + e.printStackTrace(); + } finally { + latch.countDown(); + } + } + }); + } + latch.await(); + executorService.shutdown(); + executorService.close(); + reader.close(); + log.log( + Level.FINE, + "Number queries that returned topK values: " + totalSuccessfulQueries.intValue()); + assertEquals( + "All search queries did not return topK results", + totalSuccessfulQueries.intValue(), + numQueries); + } + + @AfterClass + public static void afterClass() throws IOException { + if (directory != null) directory.close(); + directory = null; + log.log(Level.FINE, "Test finished"); + } +} From 7a2bd19cb773e3e0f8e2744af0930bb4f7dde6bf Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 3 Feb 2026 16:30:39 -0600 Subject: [PATCH 069/105] use CUDA 13.1.1, update pre-commit hooks (NVIDIA/cuvs-lucene#116) Contributes to https://github.com/rapidsai/build-planning/issues/248 Updates CUDA 13.1.x CI configurations and docs from 13.1.0 to 13.1.1 Also: * updates all `pre-commit` hooks with `pre-commit autoupdate` (might as well, since we're burning a CI run anyway) * declares compiler dependencies (matching cuVS's): https://github.com/rapidsai/cuvs-lucene/pull/116#issuecomment-3843355504 Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Gil Forsyth (https://github.com/gforsyth) - Vivek Narang (https://github.com/narangvivek10) - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cuvs-lucene/pull/116 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 2 +- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/.pre-commit-config.yaml | 6 +++--- .../environments/all_cuda-129_arch-aarch64.yaml | 5 +++++ .../environments/all_cuda-129_arch-x86_64.yaml | 5 +++++ .../environments/all_cuda-131_arch-aarch64.yaml | 5 +++++ .../environments/all_cuda-131_arch-x86_64.yaml | 5 +++++ java/cuvs-lucene/dependencies.yaml | 16 ++++++++++++++++ java/cuvs-lucene/examples/README.md | 2 +- 10 files changed, 43 insertions(+), 7 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 8c9d48e390..d32385a53a 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -43,7 +43,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.1.0' + - '13.1.1' with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 5a004c0f53..b1fdd1bfd0 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -68,7 +68,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.1.0' + - '13.1.1' with: build_type: pull-request node_type: "gpu-l4-latest-1" diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index ec1f0dfdcb..c387f58d9a 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -31,7 +31,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.1.0' + - '13.1.1' with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} diff --git a/java/cuvs-lucene/.pre-commit-config.yaml b/java/cuvs-lucene/.pre-commit-config.yaml index 03ee2970c1..ed7392ef09 100644 --- a/java/cuvs-lucene/.pre-commit-config.yaml +++ b/java/cuvs-lucene/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 repos: @@ -10,7 +10,7 @@ repos: - id: end-of-file-fixer - id: check-symlinks - repo: https://github.com/rapidsai/pre-commit-hooks - rev: v1.2.1 + rev: v1.4.2 hooks: - id: verify-copyright name: verify-copyright @@ -24,7 +24,7 @@ repos: dependencies[.]yaml$| ^[.]pre-commit-config[.]yaml$ - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.20.0 + rev: v1.20.2 hooks: - id: rapids-dependency-file-generator args: ["--clean", "--warn-all", "--strict"] diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index 737ec979fb..8bd3b94861 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -4,11 +4,15 @@ channels: - rapidsai-nightly - conda-forge dependencies: +- c-compiler - cmake>=3.30.4 - cuda-cudart-dev +- cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api - cuda-version=12.9 +- cxx-compiler +- gcc_linux-aarch64=14.* - libcublas-dev - libcurand-dev - libcusolver-dev @@ -17,4 +21,5 @@ dependencies: - maven - ninja - openjdk=22.* +- sysroot_linux-aarch64==2.28 name: all_cuda-129_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index 9cc0309525..aca12acb97 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -4,11 +4,15 @@ channels: - rapidsai-nightly - conda-forge dependencies: +- c-compiler - cmake>=3.30.4 - cuda-cudart-dev +- cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api - cuda-version=12.9 +- cxx-compiler +- gcc_linux-64=14.* - libcublas-dev - libcurand-dev - libcusolver-dev @@ -17,4 +21,5 @@ dependencies: - maven - ninja - openjdk=22.* +- sysroot_linux-64==2.28 name: all_cuda-129_arch-x86_64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml index f5d58376fa..461c75751b 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -4,11 +4,15 @@ channels: - rapidsai-nightly - conda-forge dependencies: +- c-compiler - cmake>=3.30.4 - cuda-cudart-dev +- cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api - cuda-version=13.1 +- cxx-compiler +- gcc_linux-aarch64=14.* - libcublas-dev - libcurand-dev - libcusolver-dev @@ -17,4 +21,5 @@ dependencies: - maven - ninja - openjdk=22.* +- sysroot_linux-aarch64==2.28 name: all_cuda-131_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml index 3180629fd8..6f16b0c8ef 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -4,11 +4,15 @@ channels: - rapidsai-nightly - conda-forge dependencies: +- c-compiler - cmake>=3.30.4 - cuda-cudart-dev +- cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api - cuda-version=13.1 +- cxx-compiler +- gcc_linux-64=14.* - libcublas-dev - libcurand-dev - libcusolver-dev @@ -17,4 +21,5 @@ dependencies: - maven - ninja - openjdk=22.* +- sysroot_linux-64==2.28 name: all_cuda-131_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index 2d04b5ff56..99cec2a1ff 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -79,7 +79,23 @@ dependencies: common: - output_types: conda packages: + - c-compiler - cmake>=3.30.4 + - cuda-nvcc - maven - ninja - openjdk=22.* + - cxx-compiler + specific: + - output_types: conda + matrices: + - matrix: + arch: x86_64 + packages: + - gcc_linux-64=14.* + - sysroot_linux-64==2.28 + - matrix: + arch: aarch64 + packages: + - gcc_linux-aarch64=14.* + - sysroot_linux-aarch64==2.28 diff --git a/java/cuvs-lucene/examples/README.md b/java/cuvs-lucene/examples/README.md index 2e6da591ac..a9917fb00c 100644 --- a/java/cuvs-lucene/examples/README.md +++ b/java/cuvs-lucene/examples/README.md @@ -16,7 +16,7 @@ cd .. Then do: ```sh -docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.04-cuda13.1.0-ubuntu24.04-py3.13 +docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.04-cuda13.1.1-ubuntu24.04-py3.13 ``` Inside the docker container (and in the `cuvs-lucene's` root directory) do: From 3d65177a70be91fa07a31c787beccc45060d8e2e Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Tue, 3 Feb 2026 17:57:53 -0500 Subject: [PATCH 070/105] Setup parameter validation and encapsulate parameters (NVIDIA/cuvs-lucene#110) Introducing a ~~[Hibernate Validator](https://hibernate.org/validator/) based~~ simple validation setup for input parameters. Additionally, with these code changes, I intend to logically group input parameters by encapsulating them in `AcceleratedHNSWParams` and `GPUSearchParams` for AcceleratedHNSW and Search on GPU API, respectively. This approach provides a cleaner and more scalable way to deal with input parameters. Code changes also include updating examples accordingly. Fixes NVIDIA/cuvs-lucene#105 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cuvs-lucene/pull/110 --- .../examples/AcceleratedHnswExample.java | 4 +- .../examples/IndexAndSearchonGPUExample.java | 4 +- .../cuvs/lucene/AcceleratedHNSWParams.java | 289 ++++++++++++++++++ .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 51 +--- .../cuvs/lucene/CuVS2510GPUVectorsFormat.java | 77 +---- .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 41 +-- .../nvidia/cuvs/lucene/GPUSearchParams.java | 238 +++++++++++++++ .../lucene/Lucene101AcceleratedHNSWCodec.java | 50 +-- .../Lucene99AcceleratedHNSWVectorsFormat.java | 91 +----- .../Lucene99AcceleratedHNSWVectorsWriter.java | 63 ++-- ...neAcceleratedHNSWBinaryQuantizedCodec.java | 53 +--- ...ratedHNSWBinaryQuantizedVectorsFormat.java | 77 +---- ...ratedHNSWBinaryQuantizedVectorsWriter.java | 51 +--- ...neAcceleratedHNSWScalarQuantizedCodec.java | 49 +-- ...ratedHNSWScalarQuantizedVectorsFormat.java | 78 +---- ...ratedHNSWScalarQuantizedVectorsWriter.java | 52 +--- .../lucene/TestAcceleratedHNSWParams.java | 98 ++++++ ...TestCagraToHnswSerializationAndSearch.java | 3 +- ...ializationAndSearchWithFallbackWriter.java | 9 +- .../cuvs/lucene/TestGPUSearchParams.java | 83 +++++ .../com/nvidia/cuvs/lucene/TestMerge.java | 52 ++-- 21 files changed, 881 insertions(+), 632 deletions(-) create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java diff --git a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java index 8ac19071a8..b0cf575c5a 100644 --- a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java +++ b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java @@ -7,6 +7,7 @@ import static com.nvidia.cuvs.lucene.examples.Utils.generateDataset; import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; +import com.nvidia.cuvs.lucene.AcceleratedHNSWParams; import com.nvidia.cuvs.lucene.Lucene101AcceleratedHNSWCodec; import java.io.File; import java.nio.file.Path; @@ -44,7 +45,8 @@ public class AcceleratedHnswExample { public static void main(String[] args) throws Exception { - Codec codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); + AcceleratedHNSWParams params = new AcceleratedHNSWParams.Builder().build(); + Codec codec = new Lucene101AcceleratedHNSWCodec(params); IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); random = new Random(222); diff --git a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java index 8c889620c5..834544fec9 100644 --- a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java +++ b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java @@ -9,6 +9,7 @@ import com.nvidia.cuvs.lucene.CuVS2510GPUSearchCodec; import com.nvidia.cuvs.lucene.GPUKnnFloatVectorQuery; +import com.nvidia.cuvs.lucene.GPUSearchParams; import java.io.File; import java.nio.file.Path; import java.nio.file.Paths; @@ -45,7 +46,8 @@ public class IndexAndSearchonGPUExample { public static void main(String[] args) throws Exception { - Codec codec = new CuVS2510GPUSearchCodec(); + GPUSearchParams params = new GPUSearchParams.Builder().build(); + Codec codec = new CuVS2510GPUSearchCodec(params); IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); random = new Random(222); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java new file mode 100644 index 0000000000..c32f63b90e --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java @@ -0,0 +1,289 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nvidia.cuvs.lucene; + +public class AcceleratedHNSWParams { + + /* + * TODO: Update boundaries for all parameters when a consensus is reached. + * Issue: https://github.com/rapidsai/cuvs-lucene/issues/99 + */ + private static final int MIN_WRITER_THREADS = 1; + private static final int MAX_WRITER_THREADS = 32; + private static final int MIN_INT_GRAPH_DEG = 2; + private static final int MAX_INT_GRAPH_DEG = 128; + private static final int MIN_GRAPH_DEG = 1; + private static final int MAX_GRAPH_DEG = 64; + private static final int MIN_HNSW_LAYERS = 1; + private static final int MAX_HNSW_LAYERS = 5; + private static final int MIN_MAX_CONN = 1; + private static final int MAX_MAX_CONN = 512; + private static final int MIN_BEAM_WIDTH = 1; + private static final int MAX_BEAM_WIDTH = 512; + + private final int writerThreads; + private final int intermediateGraphDegree; + private final int graphdegree; + private final int hnswLayers; + private final int maxConn; + private final int beamWidth; + + /** + * Constructs an instance of {@link GPUSearchParams} with specific parameter values. + * + * @param writerThreads Number of cuVS writer threads to use. + * @param intermediateGraphDegree The intermediate graph degree while building the CAGRA index. + * @param graphdegree The graph degree to use while building the CAGRA index. + * @param indexType The type of index to build - CAGRA, BRUTEFORCE, or both. + * @param hnswLayers The number of HNSW layers to build in the HNSW index. + * @param maxConn The max connection parameter used when building HNSW index with the fallback mechanism. + * @param beamWidth The beam width parameter used when building HNSW index with the fallback mechanism. + */ + private AcceleratedHNSWParams( + int writerThreads, + int intermediateGraphDegree, + int graphdegree, + int hnswLayers, + int maxConn, + int beamWidth) { + this.writerThreads = writerThreads; + this.intermediateGraphDegree = intermediateGraphDegree; + this.graphdegree = graphdegree; + this.hnswLayers = hnswLayers; + this.maxConn = maxConn; + this.beamWidth = beamWidth; + } + + /** + * Get the cuVS writer threads parameter + * + * @return cuVS writer threads parameter + */ + public int getWriterThreads() { + return writerThreads; + } + + /** + * Get the intermediate graph degree + * + * @return the graph degree parameter + */ + public int getIntermediateGraphDegree() { + return intermediateGraphDegree; + } + + /** + * Get the graph degree + * + * @return the graph degree parameter + */ + public int getGraphdegree() { + return graphdegree; + } + + /** + * Get the number of HNSW layers + * + * @return the number of HNSW layers + */ + public int getHnswLayers() { + return hnswLayers; + } + + /** + * Get the max connection parameter + * + * @return the max connection parameter + */ + public int getMaxConn() { + return maxConn; + } + + /** + * Get the beam width parameter + * + * @return the beam width parameter + */ + public int getBeamWidth() { + return beamWidth; + } + + @Override + public String toString() { + return "AcceleratedHNSWParams [writerThreads=" + + writerThreads + + ", intermediateGraphDegree=" + + intermediateGraphDegree + + ", graphdegree=" + + graphdegree + + ", hnswLayers=" + + hnswLayers + + ", maxConn=" + + maxConn + + ", beamWidth=" + + beamWidth + + "]"; + } + + /** + * Builder class for creating an instance of {@link GPUSearchParams} + */ + public static class Builder { + + private int writerThreads = 1; + private int intermediateGraphDegree = 128; + private int graphdegree = 64; + private int hnswLayers = 2; + private int maxConn = 8; + private int beamWidth = 16; + + /** + * Set the number of cuVS writer threads while building the index + * Valid range - Minimum: {@value MIN_WRITER_THREADS}, Maximum: {@value MAX_WRITER_THREADS} + * Default value - 64 + * + * @param writerThreads + * @return instance of {@link Builder} + */ + public Builder withWriterThreads(int writerThreads) { + this.writerThreads = writerThreads; + return this; + } + + /** + * Set the intermediate graph degree to use while building CAGRA index + * Valid range - Minimum: {@value MIN_INT_GRAPH_DEG}, Maximum: {@value MAX_INT_GRAPH_DEG} + * Default value - 128 + * + * @param intermediateGraphDegree + * @return instance of {@link Builder} + */ + public Builder withIntermediateGraphDegree(int intermediateGraphDegree) { + this.intermediateGraphDegree = intermediateGraphDegree; + return this; + } + + /** + * Set the graph degree to use while building CAGRA index + * Valid range - Minimum: {@value MIN_GRAPH_DEG}, Maximum: {@value MAX_GRAPH_DEG} + * Default value - 64 + * + * @param graphDegree + * @return instance of {@link Builder} + */ + public Builder withGraphDegree(int graphDegree) { + this.graphdegree = graphDegree; + return this; + } + + /** + * Set the number of HNSW layers to construct while building the HNSW index + * Valid range - Minimum: {@value MIN_HNSW_LAYERS}, Maximum: {@value MAX_HNSW_LAYERS} + * Default value - 2 + * + * @param hnswLayers the number of HNSW layers + * @return instance of {@link Builder} + */ + public Builder withHNSWLayer(int hnswLayers) { + this.hnswLayers = hnswLayers; + return this; + } + + /** + * Set the max connections parameter while building HNSW index with fallback mechanism + * Valid range - Minimum: {@value MIN_MAX_CONN}, Maximum: {@value MAX_MAX_CONN} + * Default value - 8 + * + * @param maxConn the max connections parameter + * @return instance of {@link Builder} + */ + public Builder withMaxConn(int maxConn) { + this.maxConn = maxConn; + return this; + } + + /** + * Set the beam width parameter while building HNSW index with fallback mechanism + * Valid range - Minimum: {@value MIN_BEAM_WIDTH}, Maximum: {@value MAX_BEAM_WIDTH} + * Default value - 16 + * + * @param beamWidth the beam width parameter + * @return instance of {@link Builder} + */ + public Builder withBeamWidth(int beamWidth) { + this.beamWidth = beamWidth; + return this; + } + + /** + * Validates the input parameters. + * + * @throws IllegalArgumentException + */ + private void validate() throws IllegalArgumentException { + if (writerThreads < MIN_WRITER_THREADS || writerThreads > MAX_WRITER_THREADS) { + throw new IllegalArgumentException( + "writerThreads not in valid range. Valid range: [" + + MIN_WRITER_THREADS + + ", " + + MAX_WRITER_THREADS + + "]"); + } + if (intermediateGraphDegree < MIN_INT_GRAPH_DEG + || intermediateGraphDegree > MAX_INT_GRAPH_DEG) { + throw new IllegalArgumentException( + "intermediateGraphDegree not in valid range. Valid range: [" + + MIN_INT_GRAPH_DEG + + ", " + + MAX_INT_GRAPH_DEG + + "]"); + } + if (graphdegree < MIN_GRAPH_DEG || graphdegree > MAX_GRAPH_DEG) { + throw new IllegalArgumentException( + "graphdegree not in valid range. Valid range: [" + + MIN_GRAPH_DEG + + ", " + + MAX_GRAPH_DEG + + "]"); + } + if (hnswLayers < MIN_HNSW_LAYERS || hnswLayers > MAX_HNSW_LAYERS) { + throw new IllegalArgumentException( + "hnswLayers not in valid range. Valid range: [" + + MIN_HNSW_LAYERS + + ", " + + MAX_HNSW_LAYERS + + "]"); + } + if (maxConn < MIN_MAX_CONN || maxConn > MAX_MAX_CONN) { + throw new IllegalArgumentException( + "maxConn not in valid range. Valid range: [" + + MIN_MAX_CONN + + ", " + + MAX_MAX_CONN + + "]"); + } + if (beamWidth < MIN_BEAM_WIDTH || beamWidth > MAX_BEAM_WIDTH) { + throw new IllegalArgumentException( + "beamWidth not in valid range. Valid range: [" + + MIN_BEAM_WIDTH + + ", " + + MAX_BEAM_WIDTH + + "]"); + } + } + + /** + * Create an instance of {@link AcceleratedHNSWParams} + * + * @return instance of {@link AcceleratedHNSWParams} + */ + public AcceleratedHNSWParams build() { + validate(); + return new AcceleratedHNSWParams( + writerThreads, intermediateGraphDegree, graphdegree, hnswLayers, maxConn, beamWidth); + } + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index 4e4f52e3a3..321f1c1ca1 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -4,9 +4,7 @@ */ package com.nvidia.cuvs.lucene; -import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.LibraryException; -import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.codecs.Codec; @@ -23,14 +21,6 @@ public class CuVS2510GPUSearchCodec extends FilterCodec { private static final Logger log = Logger.getLogger(CuVS2510GPUSearchCodec.class.getName()); private static final String NAME = "CuVS2510GPUSearchCodec"; - - private static final int DEFAULT_CUVS_WRITER_THREADS = 1; - private static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; - private static final int DEFAULT_GRAPH_DEGREE = 64; - private static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = - CagraGraphBuildAlgo.NN_DESCENT; - private static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; - private KnnVectorsFormat format; /** @@ -40,6 +30,7 @@ public class CuVS2510GPUSearchCodec extends FilterCodec { */ public CuVS2510GPUSearchCodec() throws Exception { this(NAME, LuceneProvider.getCodec("101")); + initializeFormat(new GPUSearchParams.Builder().build()); } /** @@ -50,51 +41,23 @@ public CuVS2510GPUSearchCodec() throws Exception { */ public CuVS2510GPUSearchCodec(String name, Codec delegate) { super(name, delegate); - initializeFormat( - DEFAULT_INTERMEDIATE_GRAPH_DEGREE, - DEFAULT_GRAPH_DEGREE, - DEFAULT_CUVS_WRITER_THREADS, - DEFAULT_CAGRA_GRAPH_BUILD_ALGO, - DEFAULT_INDEX_TYPE); + initializeFormat(new GPUSearchParams.Builder().build()); } /** * Initialize the codec with custom parameter values. * - * @param cuvsWriterThreads The number of cuVS writer threads to use while indexing. - * @param intermediateGraphDegree The intermediate graph degree to use while indexing. - * @param graphDegree The graph degree to use while indexing. - * @param cagraGraphBuildAlgo The CAGRA graph build algo to use. - * @param indexType Type of Index to build. - * + * @param params An instance of {@link GPUSearchParams} * @throws Exception Exception raised when initializing the codec. */ - public CuVS2510GPUSearchCodec( - int cuvsWriterThreads, - int intermediateGraphDegree, - int graphDegree, - CagraGraphBuildAlgo cagraGraphBuildAlgo, - IndexType indexType) - throws Exception { + public CuVS2510GPUSearchCodec(GPUSearchParams params) throws Exception { this(NAME, LuceneProvider.getCodec("101")); - initializeFormat( - cuvsWriterThreads, intermediateGraphDegree, graphDegree, cagraGraphBuildAlgo, indexType); + initializeFormat(params); } - private void initializeFormat( - int cuvsWriterThreads, - int intermediateGraphDegree, - int graphDegree, - CagraGraphBuildAlgo cagraGraphBuildAlgo, - IndexType indexType) { + private void initializeFormat(GPUSearchParams params) { try { - format = - new CuVS2510GPUVectorsFormat( - cuvsWriterThreads, - intermediateGraphDegree, - graphDegree, - cagraGraphBuildAlgo, - indexType); + format = new CuVS2510GPUVectorsFormat(params); setKnnFormat(format); } catch (LibraryException ex) { log.log( diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java index 62a89f4276..be7e6ab772 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java @@ -6,9 +6,7 @@ import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.assertIsSupported; -import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.LibraryException; -import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; import java.io.IOException; import java.util.logging.Logger; import org.apache.lucene.codecs.KnnVectorsFormat; @@ -26,32 +24,21 @@ */ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { - static final Logger log = Logger.getLogger(CuVS2510GPUVectorsFormat.class.getName()); + @SuppressWarnings("unused") + private static final Logger log = Logger.getLogger(CuVS2510GPUVectorsFormat.class.getName()); + + private static final int maxDimensions = 4096; + private static final LuceneProvider LUCENE_PROVIDER; + private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; + private GPUSearchParams gpuSearchParams; static final String CUVS_META_CODEC_NAME = "Lucene102CuVSVectorsFormatMeta"; static final String CUVS_META_CODEC_EXT = "vemc"; static final String CUVS_INDEX_CODEC_NAME = "Lucene102CuVSVectorsFormatIndex"; static final String CUVS_INDEX_EXT = "vcag"; - static final int VERSION_START = 0; static final int VERSION_CURRENT = VERSION_START; - static final int DEFAULT_WRITER_THREADS = 32; - static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; - static final int DEFAULT_GRAPH_DEGREE = 64; - static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = CagraGraphBuildAlgo.NN_DESCENT; - static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; - - static final LuceneProvider LUCENE_PROVIDER; - static final FlatVectorsFormat FLAT_VECTORS_FORMAT; - - final int maxDimensions = 4096; - final int cuvsWriterThreads; - final int intGraphDegree; - final int graphDegree; - final CagraGraphBuildAlgo cagraGraphBuildAlgo; - final CuVS2510GPUVectorsWriter.IndexType indexType; // the index type to build, when writing - static { try { LUCENE_PROVIDER = LuceneProvider.getInstance("99"); @@ -68,37 +55,18 @@ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { * @throws LibraryException if the native library fails to load */ public CuVS2510GPUVectorsFormat() { - this( - DEFAULT_WRITER_THREADS, - DEFAULT_INTERMEDIATE_GRAPH_DEGREE, - DEFAULT_GRAPH_DEGREE, - DEFAULT_CAGRA_GRAPH_BUILD_ALGO, - DEFAULT_INDEX_TYPE); + this(new GPUSearchParams.Builder().build()); } /** * Initializes the {@link CuVS2510GPUVectorsFormat} with the given threads, graph degree, etc. * - * @param cuvsWriterThreads the number of cuVS writer threads to use - * @param intGraphDegree the intermediate graph degree for building the CAGRA index - * @param graphDegree the graph degree for building the CAGRA index - * @param cagraGraphBuildAlgo the CAGRA graph build algorithm to use - * @param indexType the {@link com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType} - * + * @param gpuSearchParams An instance of {@link GPUSearchParams} * @throws LibraryException if the native library fails to load */ - public CuVS2510GPUVectorsFormat( - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - CagraGraphBuildAlgo cagraGraphBuildAlgo, - IndexType indexType) { + public CuVS2510GPUVectorsFormat(GPUSearchParams gpuSearchParams) { super("CuVS2510GPUVectorsFormat"); - this.cuvsWriterThreads = cuvsWriterThreads; - this.intGraphDegree = intGraphDegree; - this.graphDegree = graphDegree; - this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; - this.indexType = indexType; + this.gpuSearchParams = gpuSearchParams; } /** @@ -108,14 +76,7 @@ public CuVS2510GPUVectorsFormat( public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { assertIsSupported(); var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); - return new CuVS2510GPUVectorsWriter( - state, - cuvsWriterThreads, - intGraphDegree, - graphDegree, - cagraGraphBuildAlgo, - indexType, - flatWriter); + return new CuVS2510GPUVectorsWriter(state, gpuSearchParams, flatWriter); } /** @@ -134,18 +95,4 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException public int getMaxDimensions(String fieldName) { return maxDimensions; } - - /** - * Returns a string containing information like cuvsWriterThreads, intGraphDegree, etc. - */ - @Override - public String toString() { - StringBuilder sb = new StringBuilder(this.getClass().getSimpleName()); - sb.append("(cuvsWriterThreads=").append(cuvsWriterThreads); - sb.append("intGraphDegree=").append(intGraphDegree); - sb.append("graphDegree=").append(graphDegree); - sb.append("cagraGraphBuildAlgo=").append(cagraGraphBuildAlgo); - sb.append(")"); - return sb.toString(); - } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index ca6a05a524..cba32e21de 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -19,7 +19,6 @@ import com.nvidia.cuvs.BruteForceIndexParams; import com.nvidia.cuvs.CagraIndex; import com.nvidia.cuvs.CagraIndexParams; -import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.CuVSMatrix; import java.io.IOException; import java.io.OutputStream; @@ -75,12 +74,8 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { // we attempt to build a Cagra index static final int MIN_CAGRA_INDEX_SIZE = 2; - private final int cuvsWriterThreads; - private final int intGraphDegree; - private final int graphDegree; - private final CagraGraphBuildAlgo cagraGraphBuildAlgo; + private final GPUSearchParams gpuSearchParams; - private final IndexType indexType; private final FlatVectorsWriter flatVectorsWriter; private final List fields = new ArrayList<>(); private IndexOutput meta = null, cuvsIndex = null; @@ -139,30 +134,16 @@ public boolean bruteForce() { * Initializes {@link CuVS2510GPUVectorsWriter}. * * @param state instance of the SegmentWriteState - * @param cuvsWriterThreads the number of cuVS writer threads - * @param intGraphDegree the intermediate graph degree for building the CAGRA index - * @param graphDegree the graph degree for building the CAGRA index - * @param cagraGraphBuildAlgo the CAGRA graph build algorithm to use - * @param indexType the IndexType + * @param gpuSearchParams An instance of {@link GPUSearchParams} * @param flatVectorsWriter instance of FlatVectorsWriter * * @throws IOException I/O exceptions */ public CuVS2510GPUVectorsWriter( - SegmentWriteState state, - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - CagraGraphBuildAlgo cagraGraphBuildAlgo, - IndexType indexType, - FlatVectorsWriter flatVectorsWriter) + SegmentWriteState state, GPUSearchParams gpuSearchParams, FlatVectorsWriter flatVectorsWriter) throws IOException { super(); - this.indexType = indexType; - this.cuvsWriterThreads = cuvsWriterThreads; - this.intGraphDegree = intGraphDegree; - this.graphDegree = graphDegree; - this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; + this.gpuSearchParams = gpuSearchParams; this.flatVectorsWriter = flatVectorsWriter; this.infoStream = state.infoStream; @@ -245,10 +226,10 @@ private CagraIndexParams cagraIndexParams(int size) { } return new CagraIndexParams.Builder() - .withNumWriterThreads(cuvsWriterThreads) - .withIntermediateGraphDegree(intGraphDegree) - .withGraphDegree(graphDegree) - .withCagraGraphBuildAlgo(cagraGraphBuildAlgo) + .withNumWriterThreads(gpuSearchParams.getWriterThreads()) + .withIntermediateGraphDegree(gpuSearchParams.getIntermediateGraphDegree()) + .withGraphDegree(gpuSearchParams.getGraphdegree()) + .withCagraGraphBuildAlgo(gpuSearchParams.getCagraGraphBuildAlgo()) .build(); } @@ -280,9 +261,9 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro // workaround for the minimum number of vectors for Cagra IndexType indexType = - this.indexType.cagra() && vectors.size() < MIN_CAGRA_INDEX_SIZE + gpuSearchParams.getIndexType().cagra() && vectors.size() < MIN_CAGRA_INDEX_SIZE ? IndexType.BRUTE_FORCE - : this.indexType; + : gpuSearchParams.getIndexType(); try { @@ -613,7 +594,7 @@ private void writeMergedCagraIndex(FieldInfo fieldInfo, CagraIndex mergedIndex, public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { flatVectorsWriter.mergeOneField(fieldInfo, mergeState); - if (indexType.cagra() && !indexType.bruteForce()) { + if (gpuSearchParams.getIndexType().cagra() && !gpuSearchParams.getIndexType().bruteForce()) { // Since CAGRA merge does not support merging of indexes with purging of deletes, // we fallback to vector-based re-indexing. Issue: // https://github.com/rapidsai/cuvs/issues/1253 diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java new file mode 100644 index 0000000000..dade15d2ee --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java @@ -0,0 +1,238 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nvidia.cuvs.lucene; + +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; +import java.util.Objects; + +public class GPUSearchParams { + + /* + * TODO: Update boundaries for all parameters when a consensus is reached. + * Issue: https://github.com/rapidsai/cuvs-lucene/issues/99 + */ + private static final int MIN_WRITER_THREADS = 1; + private static final int MAX_WRITER_THREADS = 32; + private static final int MIN_INT_GRAPH_DEG = 2; + private static final int MAX_INT_GRAPH_DEG = 128; + private static final int MIN_GRAPH_DEG = 1; + private static final int MAX_GRAPH_DEG = 64; + + private final int writerThreads; + private final int intermediateGraphDegree; + private final int graphdegree; + private final CagraGraphBuildAlgo cagraGraphBuildAlgo; + private final IndexType indexType; + + /** + * Constructs an instance of {@link GPUSearchParams} with specific parameter values. + * + * @param writerThreads Number of cuVS writer threads to use. + * @param intermediateGraphDegree The intermediate graph degree while building the CAGRA index. + * @param graphdegree The graph degree to use while building the CAGRA index. + * @param cagraGraphBuildAlgo The CAGRA build algorithm to use. + * @param indexType The type of index to build - CAGRA, BRUTEFORCE, or both. + */ + private GPUSearchParams( + int writerThreads, + int intermediateGraphDegree, + int graphdegree, + CagraGraphBuildAlgo cagraGraphBuildAlgo, + IndexType indexType) { + super(); + this.writerThreads = writerThreads; + this.intermediateGraphDegree = intermediateGraphDegree; + this.graphdegree = graphdegree; + this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; + this.indexType = indexType; + } + + /** + * Get the cuVS writer threads parameter + * + * @return cuVS writer threads parameter + */ + public int getWriterThreads() { + return writerThreads; + } + + /** + * Get the intermediate graph degree + * + * @return the graph degree parameter + */ + public int getIntermediateGraphDegree() { + return intermediateGraphDegree; + } + + /** + * Get the graph degree + * + * @return the graph degree parameter + */ + public int getGraphdegree() { + return graphdegree; + } + + /** + * Get the CAGRA build algorithm parameter value + * + * @return the CAGRA build algorithm parameter value + */ + public CagraGraphBuildAlgo getCagraGraphBuildAlgo() { + return cagraGraphBuildAlgo; + } + + /** + * Get the index type parameter + * + * @return the index type parameter + */ + public IndexType getIndexType() { + return indexType; + } + + @Override + public String toString() { + return "GPUSearchParams [writerThreads=" + + writerThreads + + ", intermediateGraphDegree=" + + intermediateGraphDegree + + ", graphdegree=" + + graphdegree + + ", cagraGraphBuildAlgo=" + + cagraGraphBuildAlgo + + ", indexType=" + + indexType + + "]"; + } + + /** + * Builder class for creating an instance of {@link GPUSearchParams} + */ + public static class Builder { + + private int writerThreads = 1; + private int intermediateGraphDegree = 128; + private int graphdegree = 64; + private CagraGraphBuildAlgo cagraGraphBuildAlgo = CagraGraphBuildAlgo.NN_DESCENT; + private IndexType indexType = IndexType.CAGRA; + + /** + * Set the number of cuVS writer threads while building the index + * Valid range - Minimum: {@value MIN_WRITER_THREADS}, Maximum: {@value MAX_WRITER_THREADS} + * Default value - 64 + * + * @param writerThreads the number of cuVS writer threads + * @return instance of {@link Builder} + */ + public Builder withWriterThreads(int writerThreads) { + this.writerThreads = writerThreads; + return this; + } + + /** + * Set the intermediate graph degree to use while building CAGRA index + * Valid range - Minimum: {@value MIN_INT_GRAPH_DEG}, Maximum: {@value MAX_INT_GRAPH_DEG} + * Default value - 128 + * + * @param intermediateGraphDegree the intermediate graph degree parameter + * @return instance of {@link Builder} + */ + public Builder withIntermediateGraphDegree(int intermediateGraphDegree) { + this.intermediateGraphDegree = intermediateGraphDegree; + return this; + } + + /** + * Set the graph degree to use while building CAGRA index + * Valid range - Minimum: {@value MIN_GRAPH_DEG}, Maximum: {@value MAX_GRAPH_DEG} + * Default value - 64 + * + * @param graphDegree the graph degree parameter + * @return instance of {@link Builder} + */ + public Builder withGraphDegree(int graphDegree) { + this.graphdegree = graphDegree; + return this; + } + + /** + * Set the CAGRA build algorithm. + * Cannot be null, defaults to NN_DESCENT + * + * @param cagraGraphBuildAlgo the CAGRA build algorithm to use + * @return instance of {@link Builder} + */ + public Builder withCagraGraphBuildAlgo(CagraGraphBuildAlgo cagraGraphBuildAlgo) { + this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; + return this; + } + + /** + * Set the type of index to build - CAGRA, BRUTEFORCE, or both. + * Cannot be null, defaults to CAGRA + * + * @param indexType the type of index to build + * @return instance of {@link Builder} + */ + public Builder withIndexType(IndexType indexType) { + this.indexType = indexType; + return this; + } + + /** + * Validates the input parameters. + * + * @throws IllegalArgumentException + */ + private void validate() throws IllegalArgumentException { + if (writerThreads < MIN_WRITER_THREADS || writerThreads > MAX_WRITER_THREADS) { + throw new IllegalArgumentException( + "writerThreads not in valid range. Valid range: [" + + MIN_WRITER_THREADS + + ", " + + MAX_WRITER_THREADS + + "]"); + } + if (intermediateGraphDegree < MIN_INT_GRAPH_DEG + || intermediateGraphDegree > MAX_INT_GRAPH_DEG) { + throw new IllegalArgumentException( + "intermediateGraphDegree not in valid range. Valid range: [" + + MIN_INT_GRAPH_DEG + + ", " + + MAX_INT_GRAPH_DEG + + "]"); + } + if (graphdegree < MIN_GRAPH_DEG || graphdegree > MAX_GRAPH_DEG) { + throw new IllegalArgumentException( + "graphdegree not in valid range. Valid range: [" + + MIN_GRAPH_DEG + + ", " + + MAX_GRAPH_DEG + + "]"); + } + if (Objects.isNull(cagraGraphBuildAlgo)) { + throw new IllegalArgumentException("cagraGraphBuildAlgo cannot be null."); + } + if (Objects.isNull(indexType)) { + throw new IllegalArgumentException("indexType cannot be null."); + } + } + + /** + * Creates and returns an instance of {@link GPUSearchParams} + * + * @return instance of {@link GPUSearchParams} + */ + public GPUSearchParams build() { + validate(); + return new GPUSearchParams( + writerThreads, intermediateGraphDegree, graphdegree, cagraGraphBuildAlgo, indexType); + } + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java index 76e92b76c7..86004141ec 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java @@ -19,28 +19,9 @@ public class Lucene101AcceleratedHNSWCodec extends FilterCodec { private static final Logger log = Logger.getLogger(Lucene101AcceleratedHNSWCodec.class.getName()); - - private static final int DEFAULT_CUVS_WRITER_THREADS = 1; - private static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; - private static final int DEFAULT_GRAPH_DEGREE = 64; - private static final int DEFAULT_HNSW_LAYERS = 1; private static final String NAME = "Lucene101AcceleratedHNSWCodec"; - private static final LuceneProvider lucene99Provider; - private static final Integer maxConn; - private static final Integer beamWidth; - private KnnVectorsFormat format; - static { - try { - lucene99Provider = LuceneProvider.getInstance("99"); - maxConn = lucene99Provider.getStaticIntParam("DEFAULT_MAX_CONN"); - beamWidth = lucene99Provider.getStaticIntParam("DEFAULT_BEAM_WIDTH"); - } catch (Exception e) { - throw new ExceptionInInitializerError(e.getMessage()); - } - } - public Lucene101AcceleratedHNSWCodec() throws Exception { this(NAME, LuceneProvider.getCodec("101")); } @@ -50,40 +31,19 @@ public Lucene101AcceleratedHNSWCodec(String name, Codec delegate) { initializeFormatDefaultValues(); } - public Lucene101AcceleratedHNSWCodec( - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - int hnswLayers, - int maxConn, - int beamWidth) + public Lucene101AcceleratedHNSWCodec(AcceleratedHNSWParams acceleratedHNSWParams) throws Exception { this(NAME, LuceneProvider.getCodec("101")); - initializeFormat( - cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); + initializeFormat(acceleratedHNSWParams); } private void initializeFormatDefaultValues() { - initializeFormat( - DEFAULT_CUVS_WRITER_THREADS, - DEFAULT_INTERMEDIATE_GRAPH_DEGREE, - DEFAULT_GRAPH_DEGREE, - DEFAULT_HNSW_LAYERS, - maxConn, - beamWidth); + initializeFormat(new AcceleratedHNSWParams.Builder().build()); } - private void initializeFormat( - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - int hnswLayers, - int maxConn, - int beamWidth) { + private void initializeFormat(AcceleratedHNSWParams acceleratedHNSWParams) { try { - format = - new Lucene99AcceleratedHNSWVectorsFormat( - cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); + format = new Lucene99AcceleratedHNSWVectorsFormat(acceleratedHNSWParams); setKnnFormat(format); } catch (LibraryException ex) { log.log( diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java index a2a1f2f0b7..233313ec86 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java @@ -6,7 +6,6 @@ import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; -import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.LibraryException; import java.io.IOException; import java.util.logging.Level; @@ -28,12 +27,10 @@ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { private static final Logger log = Logger.getLogger(Lucene99AcceleratedHNSWVectorsFormat.class.getName()); - - static final int DEFAULT_WRITER_THREADS = 32; - static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; - static final int DEFAULT_GRAPH_DEGREE = 64; - static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = CagraGraphBuildAlgo.NN_DESCENT; - static final int DEFAULT_HNSW_GRAPH_LAYERS = 1; + private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; + private static final Integer NUM_MERGE_WORKERS; + private static final int maxDimensions = 4096; + private AcceleratedHNSWParams acceleratedHNSWParams; static final String HNSW_META_CODEC_NAME = "Lucene99HnswVectorsFormatMeta"; static final String HNSW_META_CODEC_EXT = "vem"; @@ -41,26 +38,9 @@ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { static final String HNSW_INDEX_EXT = "vex"; static final LuceneProvider LUCENE_PROVIDER; - private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; - private static final Integer MAX_CONN; - private static final Integer BEAM_WIDTH; - private static final Integer NUM_MERGE_WORKERS; - - private final int maxDimensions = 4096; - private final int cuvsWriterThreads; - private final int intGraphDegree; - private final int graphDegree; - // This default setting will be removed once I figure out the root cause of a related bug. - private final CagraGraphBuildAlgo cagraGraphBuildAlgo = DEFAULT_CAGRA_GRAPH_BUILD_ALGO; - private final int hnswLayers; - private final int maxConn; - private final int beamWidth; - static { try { LUCENE_PROVIDER = LuceneProvider.getInstance("99"); - MAX_CONN = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_MAX_CONN"); - BEAM_WIDTH = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); NUM_MERGE_WORKERS = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); FLAT_VECTORS_FORMAT = LUCENE_PROVIDER.getLuceneFlatVectorsFormatInstance(DefaultFlatVectorScorer.INSTANCE); @@ -70,44 +50,22 @@ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { } /** - * Initializes {@link Lucene99AcceleratedHNSWVectorsFormat} with default values. + * Initializes {@link Lucene99AcceleratedHNSWVectorsFormat} with default parameter values. * * @throws LibraryException if the native library fails to load */ public Lucene99AcceleratedHNSWVectorsFormat() { - this( - DEFAULT_WRITER_THREADS, - DEFAULT_INTERMEDIATE_GRAPH_DEGREE, - DEFAULT_GRAPH_DEGREE, - DEFAULT_HNSW_GRAPH_LAYERS, - MAX_CONN, - BEAM_WIDTH); + this(new AcceleratedHNSWParams.Builder().build()); } /** * Initializes {@link Lucene99AcceleratedHNSWVectorsFormat} with the given threads, graph degree, etc. * - * @param cuvsWriterThreads number of cuVS threads to use while building the CAGRA index - * @param intGraphDegree the intermediate graph degree while building the CAGRA index - * @param graphDegree the graph degree to use while building the CAGRA index - * @param hnswLayers the number of HNSW layers to construct in the HNSW graph - * @param maxConn the maximum connections for the HNSW graph - * @param beamWidth the beam width to use while building the HNSW graph + * @param acceleratedHNSWParams An instance of {@link AcceleratedHNSWParams} */ - public Lucene99AcceleratedHNSWVectorsFormat( - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - int hnswLayers, - int maxConn, - int beamWidth) { + public Lucene99AcceleratedHNSWVectorsFormat(AcceleratedHNSWParams acceleratedHNSWParams) { super("Lucene99AcceleratedHNSWVectorsFormat"); - this.cuvsWriterThreads = cuvsWriterThreads; - this.intGraphDegree = intGraphDegree; - this.graphDegree = graphDegree; - this.hnswLayers = hnswLayers; - this.maxConn = maxConn; - this.beamWidth = beamWidth; + this.acceleratedHNSWParams = acceleratedHNSWParams; } /** @@ -118,14 +76,7 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); if (isSupported()) { log.log(Level.FINE, "cuVS is supported so using the Lucene99AcceleratedHNSWVectorsWriter"); - return new Lucene99AcceleratedHNSWVectorsWriter( - state, - cuvsWriterThreads, - intGraphDegree, - graphDegree, - cagraGraphBuildAlgo, - hnswLayers, - flatWriter); + return new Lucene99AcceleratedHNSWVectorsWriter(state, acceleratedHNSWParams, flatWriter); } else { log.log( Level.WARNING, @@ -133,7 +84,12 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException // TODO: Make num merge workers configurable. try { return LUCENE_PROVIDER.getLuceneHnswVectorsWriterInstance( - state, maxConn, beamWidth, flatWriter, NUM_MERGE_WORKERS, null); + state, + acceleratedHNSWParams.getMaxConn(), + acceleratedHNSWParams.getBeamWidth(), + flatWriter, + NUM_MERGE_WORKERS, + null); } catch (Exception e) { // maybe there is a better suited option to throwing RuntimeException? Need to explore. throw new RuntimeException(e.getMessage()); @@ -162,19 +118,4 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException public int getMaxDimensions(String fieldName) { return maxDimensions; } - - /** - * Returns a string containing the meta information like hnsw layers, graph degree etc. - */ - @Override - public String toString() { - StringBuilder sb = new StringBuilder(this.getClass().getSimpleName()); - sb.append("(cuvsWriterThreads=").append(cuvsWriterThreads); - sb.append("intGraphDegree=").append(intGraphDegree); - sb.append("graphDegree=").append(graphDegree); - sb.append("cagraGraphBuildAlgo=").append(cagraGraphBuildAlgo); - sb.append("hnswLayers=").append(hnswLayers); - sb.append(")"); - return sb.toString(); - } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index d5a944c184..c4d98d5c3e 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -72,18 +72,11 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { private static final Logger log = Logger.getLogger(Lucene99AcceleratedHNSWVectorsWriter.class.getName()); - /** The name of the CUVS component for the info-stream * */ private static final String CUVS_COMPONENT = "CUVS"; - private static final LuceneProvider LUCENE_PROVIDER; private static final Integer VERSION_CURRENT; private static final List VECTOR_SIMILARITY_FUNCTIONS; - private final int cuvsWriterThreads; - private final int intGraphDegree; - private final int graphDegree; - private final CagraGraphBuildAlgo cagraGraphBuildAlgo; - private final int hnswLayers; // Number of layers to create in CAGRA->HNSW conversion private final FlatVectorsWriter flatVectorsWriter; // for writing the raw vectors private final List fields = new ArrayList<>(); private final InfoStream infoStream; @@ -92,6 +85,7 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { private boolean finished; private String vemFileName; private String vexFileName; + private final AcceleratedHNSWParams acceleratedHNSWParams; static { try { @@ -107,31 +101,19 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { * Initializes {@link Lucene99AcceleratedHNSWVectorsWriter} * * @param state instance of the {@link org.apache.lucene.index.SegmentWriteState} - * @param cuvsWriterThreads number of cuVS threads to use while building the intermediate CAGRA index - * @param intGraphDegree the intermediate graph degree to use while building the CAGRA index - * @param graphDegree the graph degree to use while building the CAGRA index - * @param cagraGraphBuildAlgo the CAGRA graph build algorithm to use - * @param hnswLayers the number of hnsw layers to construct while building the HNSW graph + * @param acceleratedHNSWParams An instance of {@link AcceleratedHNSWParams} * @param flatVectorsWriter instance of the {@link org.apache.lucene.codecs.hnsw.FlatVectorsWriter} * @throws IOException IOException */ public Lucene99AcceleratedHNSWVectorsWriter( SegmentWriteState state, - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - CagraGraphBuildAlgo cagraGraphBuildAlgo, - int hnswLayers, + AcceleratedHNSWParams acceleratedHNSWParams, FlatVectorsWriter flatVectorsWriter) throws IOException { super(); - this.cuvsWriterThreads = cuvsWriterThreads; - this.intGraphDegree = intGraphDegree; - this.graphDegree = graphDegree; - this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; - this.hnswLayers = hnswLayers; this.flatVectorsWriter = flatVectorsWriter; this.infoStream = state.infoStream; + this.acceleratedHNSWParams = acceleratedHNSWParams; vemFileName = IndexFileNames.segmentFileName( @@ -184,34 +166,18 @@ public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException return writer; } - /** - * Utility method for building index metadata information string object. - * - * @param size index size - * @param args additional metadata information - * @return the string representation of the metadata information - */ - static String indexMsg(int size, int... args) { - StringBuilder sb = new StringBuilder("cagra index params"); - sb.append(": size=").append(size); - sb.append(", intGraphDegree=").append(args[0]); - sb.append(", actualIntGraphDegree=").append(args[1]); - sb.append(", graphDegree=").append(args[2]); - sb.append(", actualGraphDegree=").append(args[3]); - return sb.toString(); - } - /** * Builds an instance of CagraIndexParams. * * @return instance of CagraIndexParams */ private CagraIndexParams cagraIndexParams() { + // TODO: Make build algorithm configurable after fixing the related issue. return new CagraIndexParams.Builder() - .withNumWriterThreads(cuvsWriterThreads) - .withIntermediateGraphDegree(intGraphDegree) - .withGraphDegree(graphDegree) - .withCagraGraphBuildAlgo(cagraGraphBuildAlgo) + .withNumWriterThreads(acceleratedHNSWParams.getWriterThreads()) + .withIntermediateGraphDegree(acceleratedHNSWParams.getIntermediateGraphDegree()) + .withGraphDegree(acceleratedHNSWParams.getGraphdegree()) + .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) .build(); } @@ -269,7 +235,12 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro // Create multi-layer HNSW graph from CAGRA GPUBuiltHnswGraph hnswGraph = createMultiLayerHnswGraph( - fieldInfo, size, dimensions, adjacencyListMatrix, vectors, hnswLayers); + fieldInfo, + size, + dimensions, + adjacencyListMatrix, + vectors, + acceleratedHNSWParams.getHnswLayers()); long vectorIndexOffset = hnswVectorIndex.getFilePointer(); @@ -315,7 +286,7 @@ private GPUBuiltHnswGraph createMultiLayerHnswGraph( throws Throwable { // Calculate M as cagraGraphDegree/2 - int M = graphDegree / 2; + int M = acceleratedHNSWParams.getGraphdegree() / 2; // Store all layers data List layerNodes = new ArrayList<>(); @@ -450,7 +421,7 @@ private void writeMeta( meta.writeVLong(vectorIndexLength); meta.writeVInt(field.getVectorDimension()); meta.writeInt(count); - meta.writeVInt(graphDegree / 2); // M = cagraGraphDegree/2 + meta.writeVInt(acceleratedHNSWParams.getGraphdegree() / 2); // M = cagraGraphDegree/2 // write graph nodes on each level if (graph == null) { diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java index 17336e3220..f2c1aa3719 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java @@ -4,11 +4,6 @@ */ package com.nvidia.cuvs.lucene; -import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.DEFAULT_GRAPH_DEGREE; -import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.DEFAULT_HNSW_GRAPH_LAYERS; -import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.DEFAULT_INTERMEDIATE_GRAPH_DEGREE; -import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.DEFAULT_WRITER_THREADS; - import com.nvidia.cuvs.LibraryException; import java.util.logging.Level; import java.util.logging.Logger; @@ -26,65 +21,27 @@ public class LuceneAcceleratedHNSWBinaryQuantizedCodec extends FilterCodec { private static final Logger log = Logger.getLogger(LuceneAcceleratedHNSWBinaryQuantizedCodec.class.getName()); private static final String NAME = "Lucene101AcceleratedHNSWBinaryQuantizedCodec"; - private static final LuceneProvider LUCENE99_PROVIDER; - private static final Integer DEFAULT_MAX_CONN; - private static final Integer DEFAULT_BEAM_WIDTH; private KnnVectorsFormat format; - static { - try { - LUCENE99_PROVIDER = LuceneProvider.getInstance("99"); - DEFAULT_MAX_CONN = LUCENE99_PROVIDER.getStaticIntParam("DEFAULT_MAX_CONN"); - DEFAULT_BEAM_WIDTH = LUCENE99_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); - } catch (Exception e) { - throw new ExceptionInInitializerError(e.getMessage()); - } - } - public LuceneAcceleratedHNSWBinaryQuantizedCodec() throws Exception { this(NAME, LuceneProvider.getCodec("101")); } public LuceneAcceleratedHNSWBinaryQuantizedCodec(String name, Codec delegate) { super(name, delegate); - initializeFormatDefaultValues(); + initializeFormat(new AcceleratedHNSWParams.Builder().build()); } - public LuceneAcceleratedHNSWBinaryQuantizedCodec( - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - int hnswLayers, - int maxConn, - int beamWidth) + public LuceneAcceleratedHNSWBinaryQuantizedCodec(AcceleratedHNSWParams acceleratedHNSWParams) throws Exception { this(NAME, LuceneProvider.getCodec("101")); - initializeFormat( - cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); - } - - private void initializeFormatDefaultValues() { - initializeFormat( - DEFAULT_WRITER_THREADS, - DEFAULT_INTERMEDIATE_GRAPH_DEGREE, - DEFAULT_GRAPH_DEGREE, - DEFAULT_HNSW_GRAPH_LAYERS, - DEFAULT_MAX_CONN, - DEFAULT_BEAM_WIDTH); + initializeFormat(acceleratedHNSWParams); } - private void initializeFormat( - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - int hnswLayers, - int maxConn, - int beamWidth) { + private void initializeFormat(AcceleratedHNSWParams acceleratedHNSWParams) { try { - format = - new LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat( - cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); + format = new LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat(acceleratedHNSWParams); setKnnFormat(format); } catch (LibraryException ex) { log.log( diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java index fb74d97f78..0f8d960297 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java @@ -30,21 +30,9 @@ public class LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat extends KnnVector private static final LuceneProvider LUCENE102_PROVIDER; private static final LuceneProvider LUCENE99_PROVIDER; private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; - private static final Integer DEFAULT_MAX_CONN; - private static final Integer DEFAULT_BEAM_WIDTH; + private static final int MAX_DIMENSIONS = 4096; - public static final int DEFAULT_WRITER_THREADS = 32; - public static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; - public static final int DEFAULT_GRAPH_DEGREE = 64; - public static final int DEFAULT_HNSW_GRAPH_LAYERS = 2; - - private final int maxDimensions = 4096; - private final int cuvsWriterThreads; - private final int intGraphDegree; - private final int graphDegree; - private final int hnswLayers; - private final int maxConn; - private final int beamWidth; + private final AcceleratedHNSWParams acceleratedHNSWParams; static { try { @@ -52,8 +40,6 @@ public class LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat extends KnnVector LUCENE102_PROVIDER = LuceneProvider.getInstance("102"); FLAT_VECTORS_FORMAT = LUCENE102_PROVIDER.getLuceneFlatVectorsFormatInstance(DefaultFlatVectorScorer.INSTANCE); - DEFAULT_MAX_CONN = LUCENE99_PROVIDER.getStaticIntParam("DEFAULT_MAX_CONN"); - DEFAULT_BEAM_WIDTH = LUCENE99_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); } catch (Exception e) { throw new ExceptionInInitializerError(e.getMessage()); } @@ -65,49 +51,18 @@ public class LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat extends KnnVector * @throws LibraryException if the native library fails to load */ public LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat() { - this( - DEFAULT_WRITER_THREADS, - DEFAULT_INTERMEDIATE_GRAPH_DEGREE, - DEFAULT_GRAPH_DEGREE, - DEFAULT_HNSW_GRAPH_LAYERS, - DEFAULT_MAX_CONN, - DEFAULT_BEAM_WIDTH); + this(new AcceleratedHNSWParams.Builder().build()); } /** * Initializes {@link LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat} with the given threads, graph degree, etc. * - * @param cuvsWriterThreads number of cuVS threads to use while building the CAGRA index - * @param intGraphDegree the intermediate graph degree while building the CAGRA index - * @param graphDegree the graph degree to use while building the CAGRA index - * @param hnswLayers the number of HNSW layers to construct in the HNSW graph - * @param maxConn the maximum connections for the HNSW graph - * @param beamWidth the beam width to use while building the HNSW graph + * @param acceleratedHNSWParams An instance of {@link AcceleratedHNSWParams} */ public LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat( - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - int hnswLayers, - int maxConn, - int beamWidth) { + AcceleratedHNSWParams acceleratedHNSWParams) { super("Lucene99AcceleratedHNSWBinaryQuantizedVectorsFormat"); - - assert cuvsWriterThreads > 0 - : "cuvsWriterThreads must be greater than zero, but is: " + cuvsWriterThreads; - assert intGraphDegree > 0 - : "intGraphDegree must be greater than zero, but is: " + intGraphDegree; - assert graphDegree > 0 : "graphDegree must be greater than zero, but is: " + graphDegree; - assert hnswLayers > 0 : "hnswLayers must be greater than zero, but is: " + hnswLayers; - assert maxConn > 0 : "maxConn must be greater than zero, but is: " + maxConn; - assert beamWidth > 0 : "beamWidth must be greater than zero, but is: " + beamWidth; - - this.cuvsWriterThreads = cuvsWriterThreads; - this.intGraphDegree = intGraphDegree; - this.graphDegree = graphDegree; - this.hnswLayers = hnswLayers; - this.maxConn = maxConn; - this.beamWidth = beamWidth; + this.acceleratedHNSWParams = acceleratedHNSWParams; } /** @@ -121,7 +76,7 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException Level.FINE, "cuVS is supported so using the Lucene99AcceleratedHNSWBinaryQuantizedVectorsWriter"); return new LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter( - state, cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, flatWriter); + state, acceleratedHNSWParams, flatWriter); } else { try { // Fallback to Lucene's Lucene102HnswBinaryQuantizedVectorsFormat format @@ -131,7 +86,7 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException + " Lucene102HnswBinaryQuantizedVectorsFormat"); KnnVectorsFormat fallbackFormat = LUCENE102_PROVIDER.getLuceneHnswBinaryQuantizedVectorsFormatInstance( - maxConn, beamWidth); + acceleratedHNSWParams.getMaxConn(), acceleratedHNSWParams.getBeamWidth()); return fallbackFormat.fieldsWriter(state); } catch (Exception e) { throw new RuntimeException(e.getMessage()); @@ -157,20 +112,6 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException */ @Override public int getMaxDimensions(String fieldName) { - return maxDimensions; - } - - /** - * Returns a string containing the meta information like hnsw layers, graph degree etc. - */ - @Override - public String toString() { - StringBuilder sb = new StringBuilder(this.getClass().getSimpleName()); - sb.append("(cuvsWriterThreads=").append(cuvsWriterThreads); - sb.append("intGraphDegree=").append(intGraphDegree); - sb.append("graphDegree=").append(graphDegree); - sb.append("hnswLayers=").append(hnswLayers); - sb.append(")"); - return sb.toString(); + return MAX_DIMENSIONS; } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java index b55ed98248..80ba69193a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java @@ -61,13 +61,10 @@ public class LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter extends KnnVector shallowSizeOfInstance(LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.class); private static final String COMPONENT = "Lucene99AcceleratedHNSWBinaryQuantizedVectorsWriter"; - private final int cuvsWriterThreads; - private final int intGraphDegree; - private final int graphDegree; - private final int hnswLayers; private final FlatVectorsWriter flatVectorsWriter; private final List fields = new ArrayList<>(); private final InfoStream infoStream; + private final AcceleratedHNSWParams acceleratedHNSWParams; private IndexOutput hnswMeta = null, hnswVectorIndex = null; private boolean finished; private String vemFileName; @@ -77,26 +74,17 @@ public class LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter extends KnnVector * Initializes {@link LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter} * * @param state instance of the {@link org.apache.lucene.index.SegmentWriteState} - * @param cuvsWriterThreads number of cuVS threads to use while building the intermediate CAGRA index - * @param intGraphDegree the intermediate graph degree to use while building the CAGRA index - * @param graphDegree the graph degree to use while building the CAGRA index - * @param hnswLayers the number of hnsw layers to construct while building the HNSW graph + * @param acceleratedHNSWParams An instance of {@link AcceleratedHNSWParams} * @param flatVectorsWriter instance of the {@link org.apache.lucene.codecs.hnsw.FlatVectorsWriter} * @throws IOException IOException */ public LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter( SegmentWriteState state, - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - int hnswLayers, + AcceleratedHNSWParams acceleratedHNSWParams, FlatVectorsWriter flatVectorsWriter) throws IOException { super(); - this.cuvsWriterThreads = cuvsWriterThreads; - this.intGraphDegree = intGraphDegree; - this.graphDegree = graphDegree; - this.hnswLayers = hnswLayers; + this.acceleratedHNSWParams = acceleratedHNSWParams; this.flatVectorsWriter = flatVectorsWriter; this.infoStream = state.infoStream; @@ -150,23 +138,6 @@ public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException return writer; } - /** - * Utility method for building index metadata information string object. - * - * @param size index size - * @param args additional metadata information - * @return the string representation of the metadata information - */ - static String indexMsg(int size, int... args) { - StringBuilder sb = new StringBuilder("cagra index params"); - sb.append(": size=").append(size); - sb.append(", intGraphDegree=").append(args[0]); - sb.append(", actualIntGraphDegree=").append(args[1]); - sb.append(", graphDegree=").append(args[2]); - sb.append(", actualGraphDegree=").append(args[3]); - return sb.toString(); - } - /** * Builds the intermediate CAGRA index and builds and writes the HNSW index for binary quantized vectors. * Binary quantized vectors are stored as packed bits (1 bit per dimension, 8 dimensions per byte). @@ -193,7 +164,11 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw return; } - CagraIndexParams params = cagraIndexParams(cuvsWriterThreads, intGraphDegree, graphDegree); + CagraIndexParams params = + cagraIndexParams( + acceleratedHNSWParams.getWriterThreads(), + acceleratedHNSWParams.getIntermediateGraphDegree(), + acceleratedHNSWParams.getGraphdegree()); CagraIndex cagraIndex = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(dataset) @@ -211,8 +186,8 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw dimensions, adjacencyListMatrix, vectors, - hnswLayers, - graphDegree, + acceleratedHNSWParams.getHnswLayers(), + acceleratedHNSWParams.getGraphdegree(), params, QuantizationType.BINARY); @@ -231,7 +206,7 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw size, hnswGraph, graphLevelNodeOffsets, - graphDegree); + acceleratedHNSWParams.getGraphdegree()); cagraIndex.close(); @@ -322,7 +297,7 @@ private void writeSingleVectorGraph(FieldInfo fieldInfo, List vectors) size, hnswGraph, graphLevelNodeOffsets, - graphDegree); + acceleratedHNSWParams.getGraphdegree()); } catch (Throwable t) { Utils.handleThrowable(t); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java index 2b23a9c57c..0c7736a037 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java @@ -4,11 +4,6 @@ */ package com.nvidia.cuvs.lucene; -import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.DEFAULT_GRAPH_DEGREE; -import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.DEFAULT_HNSW_GRAPH_LAYERS; -import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.DEFAULT_INTERMEDIATE_GRAPH_DEGREE; -import static com.nvidia.cuvs.lucene.LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.DEFAULT_WRITER_THREADS; - import com.nvidia.cuvs.LibraryException; import java.util.logging.Level; import java.util.logging.Logger; @@ -26,22 +21,9 @@ public class LuceneAcceleratedHNSWScalarQuantizedCodec extends FilterCodec { private static final Logger log = Logger.getLogger(LuceneAcceleratedHNSWScalarQuantizedCodec.class.getName()); private static final String NAME = "Lucene101AcceleratedHNSWScalarQuantizedCodec"; - private static final LuceneProvider LUCENE99_PROVIDER; - private static final Integer DEFAULT_MAX_CONN; - private static final Integer DEFAULT_BEAM_WIDTH; private KnnVectorsFormat format; - static { - try { - LUCENE99_PROVIDER = LuceneProvider.getInstance("99"); - DEFAULT_MAX_CONN = LUCENE99_PROVIDER.getStaticIntParam("DEFAULT_MAX_CONN"); - DEFAULT_BEAM_WIDTH = LUCENE99_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); - } catch (Exception e) { - throw new ExceptionInInitializerError(e.getMessage()); - } - } - public LuceneAcceleratedHNSWScalarQuantizedCodec() throws Exception { this(NAME, LuceneProvider.getCodec("101")); } @@ -51,40 +33,19 @@ public LuceneAcceleratedHNSWScalarQuantizedCodec(String name, Codec delegate) { initializeFormatDefaultValues(); } - public LuceneAcceleratedHNSWScalarQuantizedCodec( - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - int hnswLayers, - int maxConn, - int beamWidth) + public LuceneAcceleratedHNSWScalarQuantizedCodec(AcceleratedHNSWParams acceleratedHNSWParams) throws Exception { this(NAME, LuceneProvider.getCodec("101")); - initializeFormat( - cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); + initializeFormat(acceleratedHNSWParams); } private void initializeFormatDefaultValues() { - initializeFormat( - DEFAULT_WRITER_THREADS, - DEFAULT_INTERMEDIATE_GRAPH_DEGREE, - DEFAULT_GRAPH_DEGREE, - DEFAULT_HNSW_GRAPH_LAYERS, - DEFAULT_MAX_CONN, - DEFAULT_BEAM_WIDTH); + initializeFormat(new AcceleratedHNSWParams.Builder().build()); } - private void initializeFormat( - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - int hnswLayers, - int maxConn, - int beamWidth) { + private void initializeFormat(AcceleratedHNSWParams acceleratedHNSWParams) { try { - format = - new LuceneAcceleratedHNSWScalarQuantizedVectorsFormat( - cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, maxConn, beamWidth); + format = new LuceneAcceleratedHNSWScalarQuantizedVectorsFormat(acceleratedHNSWParams); setKnnFormat(format); } catch (LibraryException ex) { log.log( diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java index e194e0a99b..534390b843 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java @@ -27,27 +27,13 @@ public class LuceneAcceleratedHNSWScalarQuantizedVectorsFormat extends KnnVector Logger.getLogger(LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.class.getName()); private static final LuceneProvider LUCENE_PROVIDER; private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; - private static final Integer DEFAULT_MAX_CONN; - private static final Integer DEFAULT_BEAM_WIDTH; + private static final int MAX_DIMENSIONS = 4096; - public static final int DEFAULT_WRITER_THREADS = 32; - public static final int DEFAULT_INTERMEDIATE_GRAPH_DEGREE = 128; - public static final int DEFAULT_GRAPH_DEGREE = 64; - public static final int DEFAULT_HNSW_GRAPH_LAYERS = 2; - - private final int maxDimensions = 4096; - private final int cuvsWriterThreads; - private final int intGraphDegree; - private final int graphDegree; - private final int hnswLayers; - private final int maxConn; - private final int beamWidth; + private final AcceleratedHNSWParams acceleratedHNSWParams; static { try { LUCENE_PROVIDER = LuceneProvider.getInstance("99"); - DEFAULT_MAX_CONN = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_MAX_CONN"); - DEFAULT_BEAM_WIDTH = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); FLAT_VECTORS_FORMAT = LUCENE_PROVIDER.getLuceneScalarQuantizedVectorsFormatInstance(); } catch (Exception e) { throw new ExceptionInInitializerError(e.getMessage()); @@ -60,49 +46,18 @@ public class LuceneAcceleratedHNSWScalarQuantizedVectorsFormat extends KnnVector * @throws LibraryException if the native library fails to load */ public LuceneAcceleratedHNSWScalarQuantizedVectorsFormat() { - this( - DEFAULT_WRITER_THREADS, - DEFAULT_INTERMEDIATE_GRAPH_DEGREE, - DEFAULT_GRAPH_DEGREE, - DEFAULT_HNSW_GRAPH_LAYERS, - DEFAULT_MAX_CONN, - DEFAULT_BEAM_WIDTH); + this(new AcceleratedHNSWParams.Builder().build()); } /** * Initializes {@link LuceneAcceleratedHNSWScalarQuantizedVectorsFormat} with the given threads, graph degree, etc. * - * @param cuvsWriterThreads number of cuVS threads to use while building the CAGRA index - * @param intGraphDegree the intermediate graph degree while building the CAGRA index - * @param graphDegree the graph degree to use while building the CAGRA index - * @param hnswLayers the number of HNSW layers to construct in the HNSW graph - * @param maxConn the maximum connections for the HNSW graph - * @param beamWidth the beam width to use while building the HNSW graph + * @param acceleratedHNSWParams An instance of {@link AcceleratedHNSWParams} */ public LuceneAcceleratedHNSWScalarQuantizedVectorsFormat( - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - int hnswLayers, - int maxConn, - int beamWidth) { + AcceleratedHNSWParams acceleratedHNSWParams) { super("Lucene99AcceleratedHNSWScalarQuantizedVectorsFormat"); - - assert cuvsWriterThreads > 0 - : "cuvsWriterThreads must be greater than zero, but is: " + cuvsWriterThreads; - assert intGraphDegree > 0 - : "intGraphDegree must be greater than zero, but is: " + intGraphDegree; - assert graphDegree > 0 : "graphDegree must be greater than zero, but is: " + graphDegree; - assert hnswLayers > 0 : "hnswLayers must be greater than zero, but is: " + hnswLayers; - assert maxConn > 0 : "maxConn must be greater than zero, but is: " + maxConn; - assert beamWidth > 0 : "beamWidth must be greater than zero, but is: " + beamWidth; - - this.cuvsWriterThreads = cuvsWriterThreads; - this.intGraphDegree = intGraphDegree; - this.graphDegree = graphDegree; - this.hnswLayers = hnswLayers; - this.maxConn = maxConn; - this.beamWidth = beamWidth; + this.acceleratedHNSWParams = acceleratedHNSWParams; } /** @@ -114,7 +69,7 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException if (isSupported()) { log.info("cuVS is supported so using the Lucene99AcceleratedHNSWQuantizedVectorsWriter"); return new LuceneAcceleratedHNSWScalarQuantizedVectorsWriter( - state, cuvsWriterThreads, intGraphDegree, graphDegree, hnswLayers, flatWriter); + state, acceleratedHNSWParams, flatWriter); } else { try { // Fallback to Lucene's Lucene99HnswScalarQuantizedVectorsFormat @@ -122,7 +77,8 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException "GPU based indexing not supported, falling back to using the" + " Lucene99HnswScalarQuantizedVectorsFormat"); KnnVectorsFormat fallbackFormat = - LUCENE_PROVIDER.getLuceneHnswScalarQuantizedVectorsFormatInstance(beamWidth, maxConn); + LUCENE_PROVIDER.getLuceneHnswScalarQuantizedVectorsFormatInstance( + acceleratedHNSWParams.getBeamWidth(), acceleratedHNSWParams.getMaxConn()); return fallbackFormat.fieldsWriter(state); } catch (Exception e) { throw new RuntimeException(e.getMessage()); @@ -148,20 +104,6 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException */ @Override public int getMaxDimensions(String fieldName) { - return maxDimensions; - } - - /** - * Returns a string containing the meta information like hnsw layers, graph degree etc. - */ - @Override - public String toString() { - StringBuilder sb = new StringBuilder(this.getClass().getSimpleName()); - sb.append("(cuvsWriterThreads=").append(cuvsWriterThreads); - sb.append("intGraphDegree=").append(intGraphDegree); - sb.append("graphDegree=").append(graphDegree); - sb.append("hnswLayers=").append(hnswLayers); - sb.append(")"); - return sb.toString(); + return MAX_DIMENSIONS; } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java index e13102b096..ae5b2686a0 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java @@ -62,13 +62,10 @@ public class LuceneAcceleratedHNSWScalarQuantizedVectorsWriter extends KnnVector private static final LuceneProvider LUCENE_PROVIDER; private static final Integer VERSION_CURRENT; - private final int cuvsWriterThreads; - private final int intGraphDegree; - private final int graphDegree; - private final int hnswLayers; private final FlatVectorsWriter flatVectorsWriter; private final List fields = new ArrayList<>(); private final InfoStream infoStream; + private final AcceleratedHNSWParams acceleratedHNSWParams; private IndexOutput hnswMeta = null, hnswVectorIndex = null; private boolean finished; private String vemFileName; @@ -87,26 +84,17 @@ public class LuceneAcceleratedHNSWScalarQuantizedVectorsWriter extends KnnVector * Initializes {@link LuceneAcceleratedHNSWScalarQuantizedVectorsWriter} * * @param state instance of the {@link org.apache.lucene.index.SegmentWriteState} - * @param cuvsWriterThreads number of cuVS threads to use while building the intermediate CAGRA index - * @param intGraphDegree the intermediate graph degree to use while building the CAGRA index - * @param graphDegree the graph degree to use while building the CAGRA index - * @param hnswLayers the number of hnsw layers to construct while building the HNSW graph + * @param acceleratedHNSWParams An instance of {@link AcceleratedHNSWParams} * @param flatVectorsWriter instance of the {@link org.apache.lucene.codecs.hnsw.FlatVectorsWriter} * @throws IOException IOException */ public LuceneAcceleratedHNSWScalarQuantizedVectorsWriter( SegmentWriteState state, - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - int hnswLayers, + AcceleratedHNSWParams acceleratedHNSWParams, FlatVectorsWriter flatVectorsWriter) throws IOException { super(); - this.cuvsWriterThreads = cuvsWriterThreads; - this.intGraphDegree = intGraphDegree; - this.graphDegree = graphDegree; - this.hnswLayers = hnswLayers; + this.acceleratedHNSWParams = acceleratedHNSWParams; this.flatVectorsWriter = flatVectorsWriter; this.infoStream = state.infoStream; @@ -119,7 +107,6 @@ public LuceneAcceleratedHNSWScalarQuantizedVectorsWriter( boolean success = false; try { - hnswMeta = state.directory.createOutput(vemFileName, state.context); hnswVectorIndex = state.directory.createOutput(vexFileName, state.context); @@ -160,23 +147,6 @@ public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException return writer; } - /** - * Utility method for building index metadata information string object. - * - * @param size index size - * @param args additional metadata information - * @return the string representation of the metadata information - */ - static String indexMsg(int size, int... args) { - StringBuilder sb = new StringBuilder("cagra index params"); - sb.append(": size=").append(size); - sb.append(", intGraphDegree=").append(args[0]); - sb.append(", actualIntGraphDegree=").append(args[1]); - sb.append(", graphDegree=").append(args[2]); - sb.append(", actualGraphDegree=").append(args[3]); - return sb.toString(); - } - private static byte signedToUnsignedByte(byte signedByte) { return (byte) (signedByte & 0xFF); } @@ -220,7 +190,11 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw return; } - CagraIndexParams params = cagraIndexParams(cuvsWriterThreads, intGraphDegree, graphDegree); + CagraIndexParams params = + cagraIndexParams( + acceleratedHNSWParams.getWriterThreads(), + acceleratedHNSWParams.getIntermediateGraphDegree(), + acceleratedHNSWParams.getGraphdegree()); CagraIndex cagraIndex = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(dataset) @@ -237,8 +211,8 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw dimensions, adjacencyListMatrix, unsignedVectors, - hnswLayers, - graphDegree, + acceleratedHNSWParams.getHnswLayers(), + acceleratedHNSWParams.getGraphdegree(), params, QuantizationType.SCALAR); @@ -259,7 +233,7 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw size, hnswGraph, graphLevelNodeOffsets, - graphDegree); + acceleratedHNSWParams.getGraphdegree()); cagraIndex.close(); } catch (Throwable t) { @@ -346,7 +320,7 @@ private void writeSingleVectorGraph(FieldInfo fieldInfo, List vectors) size, hnswGraph, graphLevelNodeOffsets, - graphDegree); + acceleratedHNSWParams.getGraphdegree()); } catch (Throwable t) { Utils.handleThrowable(t); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java new file mode 100644 index 0000000000..461c4b17f7 --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java @@ -0,0 +1,98 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nvidia.cuvs.lucene; + +import java.util.Random; +import java.util.logging.Logger; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestAcceleratedHNSWParams extends LuceneTestCase { + + @SuppressWarnings("unused") + private static final Logger log = Logger.getLogger(TestAcceleratedHNSWParams.class.getName()); + + private static Random random; + + @Test + public void testAcceleratedHNSWParamsDefaultValues() { + AcceleratedHNSWParams params = new AcceleratedHNSWParams.Builder().build(); + assertEquals(16, params.getBeamWidth()); + assertEquals(64, params.getGraphdegree()); + assertEquals(2, params.getHnswLayers()); + assertEquals(128, params.getIntermediateGraphDegree()); + assertEquals(8, params.getMaxConn()); + assertEquals(1, params.getWriterThreads()); + } + + @Test + public void testAcceleratedHNSWParamsInvalidBeamWidth() { + for (int v : + new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(17, Integer.MAX_VALUE)}) { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withBeamWidth(v).build()); + } + } + + @Test + public void testAcceleratedHNSWParamsInvalidGraphDegree() { + for (int v : + new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(65, Integer.MAX_VALUE)}) { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withGraphDegree(v).build()); + } + } + + @Test + public void testAcceleratedHNSWParamsInvalidHNSWLayers() { + for (int v : + new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(3, Integer.MAX_VALUE)}) { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withHNSWLayer(v).build()); + } + } + + @Test + public void testAcceleratedHNSWParamsInvalidIntGraphDegree() { + for (int v : + new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(129, Integer.MAX_VALUE)}) { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withIntermediateGraphDegree(v).build()); + } + } + + @Test + public void testAcceleratedHNSWParamsInvalidMaxConn() { + for (int v : + new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(9, Integer.MAX_VALUE)}) { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withMaxConn(v).build()); + } + } + + @Test + public void testAcceleratedHNSWParamsInvalidWriterThreads() { + for (int v : + new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(2, Integer.MAX_VALUE)}) { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withWriterThreads(v).build()); + } + } + + @BeforeClass + public static void beforeClass() { + random = random(); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java index 45cefeec7b..96be6c0112 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java @@ -59,7 +59,8 @@ public void beforeTest() throws Exception { @Test public void testCagraToHnswSerializationAndSearch() throws Exception { - Codec codec = new Lucene101AcceleratedHNSWCodec(1, 128, 64, 3, 16, 100); + AcceleratedHNSWParams params = new AcceleratedHNSWParams.Builder().build(); + Codec codec = new Lucene101AcceleratedHNSWCodec(params); IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); final int COMMIT_FREQ = 2000; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java index 81e2627b8d..04ad20a090 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java @@ -64,7 +64,14 @@ public static void beforeClass() throws Exception { @Test public void testCagraToHnswSerializationAndSearchWithFallbackWriter() throws Exception { - Codec codec = new Lucene101AcceleratedHNSWCodec(32, 128, 64, 3, 16, 100); + AcceleratedHNSWParams params = + new AcceleratedHNSWParams.Builder() + .withHNSWLayer(3) + .withMaxConn(16) + .withBeamWidth(100) + .build(); + Codec codec = new Lucene101AcceleratedHNSWCodec(params); + IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); final int COMMIT_FREQ = 2000; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java new file mode 100644 index 0000000000..41d660d62e --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java @@ -0,0 +1,83 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nvidia.cuvs.lucene; + +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; +import java.util.Random; +import java.util.logging.Logger; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.junit.BeforeClass; +import org.junit.Test; + +@SuppressSysoutChecks(bugUrl = "") +public class TestGPUSearchParams extends LuceneTestCase { + + @SuppressWarnings("unused") + private static final Logger log = Logger.getLogger(TestGPUSearchParams.class.getName()); + + private static Random random; + + @Test + public void testGPUSearchParamsDefaultValues() { + GPUSearchParams params = new GPUSearchParams.Builder().build(); + assertEquals(64, params.getGraphdegree()); + assertEquals(128, params.getIntermediateGraphDegree()); + assertEquals(1, params.getWriterThreads()); + assertEquals(CagraGraphBuildAlgo.NN_DESCENT, params.getCagraGraphBuildAlgo()); + assertEquals(IndexType.CAGRA, params.getIndexType()); + } + + @Test + public void testGPUSearchParamsInvalidGraphDegree() { + for (int v : + new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(65, Integer.MAX_VALUE)}) { + assertThrows( + IllegalArgumentException.class, + () -> new GPUSearchParams.Builder().withGraphDegree(v).build()); + } + } + + @Test + public void testGPUSearchParamsInvalidIntermediateGraphDegree() { + for (int v : + new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(129, Integer.MAX_VALUE)}) { + assertThrows( + IllegalArgumentException.class, + () -> new GPUSearchParams.Builder().withIntermediateGraphDegree(v).build()); + } + } + + @Test + public void testGPUSearchParamsInvalidWriterThreads() { + for (int v : + new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(2, Integer.MAX_VALUE)}) { + assertThrows( + IllegalArgumentException.class, + () -> new GPUSearchParams.Builder().withWriterThreads(v).build()); + } + } + + @Test + public void testGPUSearchParamsInvalidCagraGraphBuildAlgo() { + assertThrows( + IllegalArgumentException.class, + () -> new GPUSearchParams.Builder().withCagraGraphBuildAlgo(null).build()); + } + + @Test + public void testGPUSearchParamsInvalidIndexType() { + assertThrows( + IllegalArgumentException.class, + () -> new GPUSearchParams.Builder().withIndexType(null).build()); + } + + @BeforeClass + public static void beforeClass() { + random = random(); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java index 8e64e9fb1b..bcc29178b5 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java @@ -137,8 +137,10 @@ public void testMergeManyDocumentsMultipleSegments() throws IOException { + ", vectorProbability=" + vectorProbability); - CuVS2510GPUVectorsFormat format = - new CuVS2510GPUVectorsFormat(32, 128, 64, cagraGraphBuildAlgo, IndexType.CAGRA); + GPUSearchParams params = + new GPUSearchParams.Builder().withCagraGraphBuildAlgo(cagraGraphBuildAlgo).build(); + + CuVS2510GPUVectorsFormat format = new CuVS2510GPUVectorsFormat(params); IndexWriterConfig config = new IndexWriterConfig() @@ -271,8 +273,10 @@ public void testMergeWithIndexSorting() throws IOException { + ", vectorProbability=" + vectorProbability); - CuVS2510GPUVectorsFormat format = - new CuVS2510GPUVectorsFormat(32, 128, 64, cagraGraphBuildAlgo, IndexType.CAGRA); + GPUSearchParams params = + new GPUSearchParams.Builder().withCagraGraphBuildAlgo(cagraGraphBuildAlgo).build(); + + CuVS2510GPUVectorsFormat format = new CuVS2510GPUVectorsFormat(params); IndexWriterConfig config = new IndexWriterConfig() @@ -491,8 +495,10 @@ public void testMergeWithMissingVectors() throws IOException { int maxBufferedDocs = 10 + random().nextInt(11); // 10-20 docs per buffer int numSegments = 3 + random().nextInt(3); // 3-5 segments - CuVS2510GPUVectorsFormat format = - new CuVS2510GPUVectorsFormat(32, 128, 64, cagraGraphBuildAlgo, IndexType.CAGRA); + GPUSearchParams params = + new GPUSearchParams.Builder().withCagraGraphBuildAlgo(cagraGraphBuildAlgo).build(); + + CuVS2510GPUVectorsFormat format = new CuVS2510GPUVectorsFormat(params); IndexWriterConfig config = new IndexWriterConfig() @@ -640,8 +646,10 @@ public void testMergeWithDeletions() throws IOException { + ", deletionProbability=" + deletionProbability); - CuVS2510GPUVectorsFormat format = - new CuVS2510GPUVectorsFormat(32, 128, 64, cagraGraphBuildAlgo, IndexType.CAGRA); + GPUSearchParams params = + new GPUSearchParams.Builder().withCagraGraphBuildAlgo(cagraGraphBuildAlgo).build(); + + CuVS2510GPUVectorsFormat format = new CuVS2510GPUVectorsFormat(params); IndexWriterConfig config = new IndexWriterConfig() @@ -784,10 +792,15 @@ public void testMergeBruteForceIndex() throws IOException { + ", vectorProbability=" + vectorProbability); + GPUSearchParams params = + new GPUSearchParams.Builder() + .withCagraGraphBuildAlgo(cagraGraphBuildAlgo) + .withIndexType(IndexType.BRUTE_FORCE) + .build(); + // Configure with brute force index type CuVS2510GPUVectorsFormat bruteForceFormat = - new CuVS2510GPUVectorsFormat( - 32, 128, 64, cagraGraphBuildAlgo, IndexType.BRUTE_FORCE); // Use brute force index + new CuVS2510GPUVectorsFormat(params); // Use brute force index IndexWriterConfig config = new IndexWriterConfig() @@ -941,13 +954,14 @@ public void testMergeCagraAndBruteForceIndex() throws IOException { + vectorProbability); // Configure with CAGRA + brute force combined index type + GPUSearchParams params = + new GPUSearchParams.Builder() + .withCagraGraphBuildAlgo(cagraGraphBuildAlgo) + .withIndexType(IndexType.CAGRA_AND_BRUTE_FORCE) + .build(); + CuVS2510GPUVectorsFormat combinedFormat = - new CuVS2510GPUVectorsFormat( - 32, - 128, - 64, - cagraGraphBuildAlgo, - IndexType.CAGRA_AND_BRUTE_FORCE); // Use combined CAGRA + brute force + new CuVS2510GPUVectorsFormat(params); // Use combined CAGRA + brute force IndexWriterConfig config = new IndexWriterConfig() @@ -1123,8 +1137,10 @@ public void testLargeScaleMerge() throws IOException { + ", totalDocuments=" + totalDocuments); - CuVS2510GPUVectorsFormat format = - new CuVS2510GPUVectorsFormat(32, 128, 64, cagraGraphBuildAlgo, IndexType.CAGRA); + GPUSearchParams params = + new GPUSearchParams.Builder().withCagraGraphBuildAlgo(cagraGraphBuildAlgo).build(); + + CuVS2510GPUVectorsFormat format = new CuVS2510GPUVectorsFormat(params); IndexWriterConfig config = new IndexWriterConfig() From a1753c1e36608618753744a47ff80bbad6a758a3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 20 Feb 2026 13:06:08 -0600 Subject: [PATCH 071/105] check-nightly-ci: update to new version (NVIDIA/cuvs-lucene#119) Contributes to https://github.com/rapidsai/shared-actions/issues/94 Uses the updated `check_nightly_success` check from https://github.com/rapidsai/shared-actions/pull/96 . Now that CI check will only ever consider the branch a PR targets, which should prevent issues related to release timing in the future. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Kyle Edwards (https://github.com/KyleFromNVIDIA) URL: https://github.com/rapidsai/cuvs-lucene/pull/119 --- java/cuvs-lucene/.github/workflows/pr.yaml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index b1fdd1bfd0..a04f2d26c5 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -29,16 +29,21 @@ jobs: if: ${{ vars.TELEMETRY_ENABLED == 'true' }} uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main check-nightly-ci: - needs: telemetry-setup runs-on: ubuntu-latest + permissions: + actions: read + id-token: write env: - RAPIDS_GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ github.token }} steps: + - name: Get PR Info + id: get-pr-info + uses: nv-gha-runners/get-pr-info@main - name: Check if nightly CI is passing uses: rapidsai/shared-actions/check_nightly_success/dispatch@main with: - repo: cuvs - max_days_without_success: 30 + repo: ${{ github.repository }} + target-branch: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.ref }} changed-files: needs: telemetry-setup secrets: inherit From 53fb7e9bfd66f29cfb3de27d98cb5bc62ac67718 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:00:54 -0500 Subject: [PATCH 072/105] Make CAGRA build algo configurable for the AcceleratedHNSW (NVIDIA/cuvs-lucene#120) Fixes NVIDIA/cuvs-lucene#106 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/rapidsai/cuvs-lucene/pull/120 --- .../cuvs/lucene/AcceleratedHNSWParams.java | 181 ++++++++++++++++-- .../cuvs/lucene/AcceleratedHNSWUtils.java | 10 +- .../Lucene99AcceleratedHNSWVectorsWriter.java | 35 ++-- ...ratedHNSWBinaryQuantizedVectorsWriter.java | 4 +- ...ratedHNSWScalarQuantizedVectorsWriter.java | 4 +- .../lucene/TestAcceleratedHNSWParams.java | 40 +++- 6 files changed, 237 insertions(+), 37 deletions(-) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java index c32f63b90e..9e663d2cb7 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java @@ -5,6 +5,12 @@ package com.nvidia.cuvs.lucene; +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.CuVSIvfPqParams; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + public class AcceleratedHNSWParams { /* @@ -12,17 +18,32 @@ public class AcceleratedHNSWParams { * Issue: https://github.com/rapidsai/cuvs-lucene/issues/99 */ private static final int MIN_WRITER_THREADS = 1; - private static final int MAX_WRITER_THREADS = 32; + private static final int MAX_WRITER_THREADS = 512; private static final int MIN_INT_GRAPH_DEG = 2; - private static final int MAX_INT_GRAPH_DEG = 128; + private static final int MAX_INT_GRAPH_DEG = 512; private static final int MIN_GRAPH_DEG = 1; - private static final int MAX_GRAPH_DEG = 64; + private static final int MAX_GRAPH_DEG = 512; private static final int MIN_HNSW_LAYERS = 1; - private static final int MAX_HNSW_LAYERS = 5; + private static final int MAX_HNSW_LAYERS = 3; private static final int MIN_MAX_CONN = 1; private static final int MAX_MAX_CONN = 512; private static final int MIN_BEAM_WIDTH = 1; private static final int MAX_BEAM_WIDTH = 512; + private static final int MIN_NUM_MERGE_WORKERS = 1; + private static final int MAX_NUM_MERGE_WORKERS = 512; + + private static final int DEFAULT_WRITER_THREADS = 1; + private static final int DEFAULT_INT_GRAPH_DEGREE = 128; + private static final int DEFAULT_GRAPH_DEGREE = 64; + private static final int DEFAULT_HNSW_LAYERS = 1; + private static final int DEFAULT_MAX_CONN = 32; + private static final int DEFAULT_BEAM_WIDTH = 32; + private static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = + CagraGraphBuildAlgo.NN_DESCENT; + private static final CuVSIvfPqParams DEFAULT_IVF_PQ_PARAMS = + new CuVSIvfPqParams.Builder().build(); + private static final int DEFAULT_NUM_MERGE_WORKERS = 1; + private static final ExecutorService DEFAULT_MERGE_EXE_SRVC = Executors.newFixedThreadPool(1); private final int writerThreads; private final int intermediateGraphDegree; @@ -30,6 +51,10 @@ public class AcceleratedHNSWParams { private final int hnswLayers; private final int maxConn; private final int beamWidth; + private final CagraGraphBuildAlgo cagraGraphBuildAlgo; + private final CuVSIvfPqParams cuVSIvfPqParams; + private final int numMergeWorkers; + private final ExecutorService mergeExec; /** * Constructs an instance of {@link GPUSearchParams} with specific parameter values. @@ -41,6 +66,10 @@ public class AcceleratedHNSWParams { * @param hnswLayers The number of HNSW layers to build in the HNSW index. * @param maxConn The max connection parameter used when building HNSW index with the fallback mechanism. * @param beamWidth The beam width parameter used when building HNSW index with the fallback mechanism. + * @param cagraGraphBuildAlgo The CAGRA graph build algorithm to use [NN_DESCENT, IVF_PQ]. + * @param cagraGraphBuildAlgo An instance of CuVSIvfPqParams containing IVF_PQ specific parameters. + * @param numMergeWorkers The number of merge workers to use with the fallback mechanism. + * @param mergeExec The instance of {@link ExecutorService} to use with the fallback mechanism. */ private AcceleratedHNSWParams( int writerThreads, @@ -48,13 +77,22 @@ private AcceleratedHNSWParams( int graphdegree, int hnswLayers, int maxConn, - int beamWidth) { + int beamWidth, + CagraGraphBuildAlgo cagraGraphBuildAlgo, + CuVSIvfPqParams cuVSIvfPqParams, + int numMergeWorkers, + ExecutorService mergeExec) { + super(); this.writerThreads = writerThreads; this.intermediateGraphDegree = intermediateGraphDegree; this.graphdegree = graphdegree; this.hnswLayers = hnswLayers; this.maxConn = maxConn; this.beamWidth = beamWidth; + this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; + this.cuVSIvfPqParams = cuVSIvfPqParams; + this.numMergeWorkers = numMergeWorkers; + this.mergeExec = mergeExec; } /** @@ -111,6 +149,42 @@ public int getBeamWidth() { return beamWidth; } + /** + * Get the CAGRA graph build algorithm + * + * @return the CAGRA graph build algorithm + */ + public CagraGraphBuildAlgo getCagraGraphBuildAlgo() { + return cagraGraphBuildAlgo; + } + + /** + * Get the instance of {@link CuVSIvfPqParams} + * + * @return the instance of {@link CuVSIvfPqParams} + */ + public CuVSIvfPqParams getCuVSIvfPqParams() { + return cuVSIvfPqParams; + } + + /** + * Get the number of merge workers set to be used in the fallback mechanism + * + * @return the number of merge workers + */ + public int getNumMergeWorkers() { + return numMergeWorkers; + } + + /** + * Get the instance of the {@link ExecutorService} to be used in the fallback mechanism * + * + * @return the instance of the {@link ExecutorService} + */ + public ExecutorService getMergeExec() { + return mergeExec; + } + @Override public String toString() { return "AcceleratedHNSWParams [writerThreads=" @@ -125,6 +199,12 @@ public String toString() { + maxConn + ", beamWidth=" + beamWidth + + ", cagraGraphBuildAlgo=" + + cagraGraphBuildAlgo + + ", numMergeWorkers=" + + numMergeWorkers + + ", mergeExec=" + + mergeExec + "]"; } @@ -133,12 +213,16 @@ public String toString() { */ public static class Builder { - private int writerThreads = 1; - private int intermediateGraphDegree = 128; - private int graphdegree = 64; - private int hnswLayers = 2; - private int maxConn = 8; - private int beamWidth = 16; + private int writerThreads = DEFAULT_WRITER_THREADS; + private int intermediateGraphDegree = DEFAULT_INT_GRAPH_DEGREE; + private int graphdegree = DEFAULT_GRAPH_DEGREE; + private int hnswLayers = DEFAULT_HNSW_LAYERS; + private int maxConn = DEFAULT_MAX_CONN; + private int beamWidth = DEFAULT_BEAM_WIDTH; + private CagraGraphBuildAlgo cagraGraphBuildAlgo = DEFAULT_CAGRA_GRAPH_BUILD_ALGO; + private CuVSIvfPqParams cuVSIvfPqParams = DEFAULT_IVF_PQ_PARAMS; + private int numMergeWorkers = DEFAULT_NUM_MERGE_WORKERS; + private ExecutorService mergeExec = DEFAULT_MERGE_EXE_SRVC; /** * Set the number of cuVS writer threads while building the index @@ -218,6 +302,53 @@ public Builder withBeamWidth(int beamWidth) { return this; } + /** + * Set the CAGRA graph build algorithm to use + * Default NN_DESCENT + * + * @param cagraGraphBuildAlgo + * @return instance of {@link Builder} + */ + public Builder withCagraGraphBuildAlgo(CagraGraphBuildAlgo cagraGraphBuildAlgo) { + this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; + return this; + } + + /** + * Set the instance of {@link CuVSIvfPqParams} + * + * @param cuVSIvfPqParams + * @return instance of {@link Builder} + */ + public Builder withCuVSIvfPqParams(CuVSIvfPqParams cuVSIvfPqParams) { + this.cuVSIvfPqParams = cuVSIvfPqParams; + return this; + } + + /** + * Set the number of merge workers to be used with the fallback mechanism + * Default value - 1 + * + * @param numMergeWorkers number of merge workers to set + * @return instance of {@link Builder} + */ + public Builder withNumMergeWorkers(int numMergeWorkers) { + this.numMergeWorkers = numMergeWorkers; + return this; + } + + /** + * Set the merge executor service to be used in the fallback mechanism + * Default value an instance with one thread + * + * @param mergeExec an instance of {@link ExecutorService} + * @return instance of {@link Builder} + */ + public Builder withMergeExecutorService(ExecutorService mergeExec) { + this.mergeExec = mergeExec; + return this; + } + /** * Validates the input parameters. * @@ -273,6 +404,23 @@ private void validate() throws IllegalArgumentException { + MAX_BEAM_WIDTH + "]"); } + if (Objects.isNull(cagraGraphBuildAlgo)) { + throw new IllegalArgumentException("cagraGraphBuildAlgo cannot be null."); + } + if (Objects.isNull(cuVSIvfPqParams)) { + throw new IllegalArgumentException("cuVSIvfPqParams cannot be null."); + } + if (numMergeWorkers < MIN_NUM_MERGE_WORKERS || numMergeWorkers > MAX_NUM_MERGE_WORKERS) { + throw new IllegalArgumentException( + "numMergeWorkers not in valid range. Valid range: [" + + MIN_NUM_MERGE_WORKERS + + ", " + + MAX_NUM_MERGE_WORKERS + + "]"); + } + if (Objects.isNull(mergeExec)) { + throw new IllegalArgumentException("mergeExec cannot be null."); + } } /** @@ -283,7 +431,16 @@ private void validate() throws IllegalArgumentException { public AcceleratedHNSWParams build() { validate(); return new AcceleratedHNSWParams( - writerThreads, intermediateGraphDegree, graphdegree, hnswLayers, maxConn, beamWidth); + writerThreads, + intermediateGraphDegree, + graphdegree, + hnswLayers, + maxConn, + beamWidth, + cagraGraphBuildAlgo, + cuVSIvfPqParams, + numMergeWorkers, + mergeExec); } } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java index 9f9938d94d..457ad22d9d 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java @@ -11,6 +11,7 @@ import com.nvidia.cuvs.CagraIndex; import com.nvidia.cuvs.CagraIndexParams; import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.CuVSIvfPqParams; import com.nvidia.cuvs.CuVSMatrix; import com.nvidia.cuvs.RowView; import java.io.IOException; @@ -385,12 +386,17 @@ public static void writeEmpty(FieldInfo fieldInfo, IndexOutput op) throws IOExce * @return instance of CagraIndexParams */ public static CagraIndexParams cagraIndexParams( - int cuvsWriterThreads, int intGraphDegree, int graphDegree) { + int cuvsWriterThreads, + int intGraphDegree, + int graphDegree, + CagraGraphBuildAlgo cagraGraphBuildAlgo, + CuVSIvfPqParams cuVSIvfPqParams) { return new CagraIndexParams.Builder() .withNumWriterThreads(cuvsWriterThreads) .withIntermediateGraphDegree(intGraphDegree) .withGraphDegree(graphDegree) - .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) + .withCagraGraphBuildAlgo(cagraGraphBuildAlgo) + .withCuVSIvfPqParams(cuVSIvfPqParams) .build(); } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index c4d98d5c3e..5e74eba934 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -4,6 +4,7 @@ */ package com.nvidia.cuvs.lucene; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.cagraIndexParams; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_CODEC_NAME; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_EXT; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_EXT; @@ -16,7 +17,6 @@ import com.nvidia.cuvs.CagraIndex; import com.nvidia.cuvs.CagraIndexParams; -import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.CuVSMatrix; import com.nvidia.cuvs.RowView; import java.io.IOException; @@ -166,21 +166,6 @@ public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException return writer; } - /** - * Builds an instance of CagraIndexParams. - * - * @return instance of CagraIndexParams - */ - private CagraIndexParams cagraIndexParams() { - // TODO: Make build algorithm configurable after fixing the related issue. - return new CagraIndexParams.Builder() - .withNumWriterThreads(acceleratedHNSWParams.getWriterThreads()) - .withIntermediateGraphDegree(acceleratedHNSWParams.getIntermediateGraphDegree()) - .withGraphDegree(acceleratedHNSWParams.getGraphdegree()) - .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) - .build(); - } - /** * A utility method to print info/debugging messages using InfoStream. * @@ -219,7 +204,14 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro } long startTime = System.nanoTime(); - CagraIndexParams params = cagraIndexParams(); + + CagraIndexParams params = + cagraIndexParams( + acceleratedHNSWParams.getWriterThreads(), + acceleratedHNSWParams.getIntermediateGraphDegree(), + acceleratedHNSWParams.getGraphdegree(), + acceleratedHNSWParams.getCagraGraphBuildAlgo(), + acceleratedHNSWParams.getCuVSIvfPqParams()); CagraIndex cagraIndex = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(dataset) @@ -356,7 +348,14 @@ private CuVSMatrix buildCagraGraphForSubset(float[][] vectors, int[] selectedNod CuVSMatrix subsetDataset = CuVSMatrix.ofArray(vectors); // Build CAGRA index for the subset - CagraIndexParams params = cagraIndexParams(); + CagraIndexParams params = + cagraIndexParams( + acceleratedHNSWParams.getWriterThreads(), + acceleratedHNSWParams.getIntermediateGraphDegree(), + acceleratedHNSWParams.getGraphdegree(), + acceleratedHNSWParams.getCagraGraphBuildAlgo(), + acceleratedHNSWParams.getCuVSIvfPqParams()); + CagraIndex subsetIndex = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(subsetDataset) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java index 80ba69193a..1bdc62db98 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java @@ -168,7 +168,9 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw cagraIndexParams( acceleratedHNSWParams.getWriterThreads(), acceleratedHNSWParams.getIntermediateGraphDegree(), - acceleratedHNSWParams.getGraphdegree()); + acceleratedHNSWParams.getGraphdegree(), + acceleratedHNSWParams.getCagraGraphBuildAlgo(), + acceleratedHNSWParams.getCuVSIvfPqParams()); CagraIndex cagraIndex = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(dataset) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java index ae5b2686a0..11a085ee64 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java @@ -194,7 +194,9 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw cagraIndexParams( acceleratedHNSWParams.getWriterThreads(), acceleratedHNSWParams.getIntermediateGraphDegree(), - acceleratedHNSWParams.getGraphdegree()); + acceleratedHNSWParams.getGraphdegree(), + acceleratedHNSWParams.getCagraGraphBuildAlgo(), + acceleratedHNSWParams.getCuVSIvfPqParams()); CagraIndex cagraIndex = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(dataset) diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java index 461c4b17f7..1fc9b8f20a 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java @@ -5,6 +5,7 @@ package com.nvidia.cuvs.lucene; +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import java.util.Random; import java.util.logging.Logger; import org.apache.lucene.tests.util.LuceneTestCase; @@ -23,12 +24,14 @@ public class TestAcceleratedHNSWParams extends LuceneTestCase { @Test public void testAcceleratedHNSWParamsDefaultValues() { AcceleratedHNSWParams params = new AcceleratedHNSWParams.Builder().build(); - assertEquals(16, params.getBeamWidth()); + assertEquals(32, params.getBeamWidth()); assertEquals(64, params.getGraphdegree()); - assertEquals(2, params.getHnswLayers()); + assertEquals(1, params.getHnswLayers()); assertEquals(128, params.getIntermediateGraphDegree()); - assertEquals(8, params.getMaxConn()); + assertEquals(32, params.getMaxConn()); assertEquals(1, params.getWriterThreads()); + assertEquals(1, params.getNumMergeWorkers()); + assertEquals(CagraGraphBuildAlgo.NN_DESCENT, params.getCagraGraphBuildAlgo()); } @Test @@ -91,6 +94,37 @@ public void testAcceleratedHNSWParamsInvalidWriterThreads() { } } + @Test + public void testAcceleratedHNSWParamsInvalidNumMergeWorkers() { + for (int v : + new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(33, Integer.MAX_VALUE)}) { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withNumMergeWorkers(v).build()); + } + } + + @Test + public void testAcceleratedHNSWParamsInvalidMergeExecutorService() { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withMergeExecutorService(null).build()); + } + + @Test + public void testAcceleratedHNSWParamsInvalidCagraGraphBuildAlgo() { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withCagraGraphBuildAlgo(null).build()); + } + + @Test + public void testAcceleratedHNSWParamsInvalidCuVSIvfPqParams() { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withCuVSIvfPqParams(null).build()); + } + @BeforeClass public static void beforeClass() { random = random(); From 79c041ca9fc4aec063c89de7703632f7c04180c5 Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Thu, 12 Mar 2026 10:57:10 -0700 Subject: [PATCH 073/105] Revert "Prepare release/26.04" This reverts commit 86d8e84305e7649735529b39378dde3aa32391ba. --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 8 ++++---- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/RAPIDS_BRANCH | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 991ac62723..d32385a53a 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -35,7 +35,7 @@ concurrency: jobs: java-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index b4663094fe..a04f2d26c5 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.04 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main if: always() with: needs: ${{ toJSON(needs) }} @@ -47,7 +47,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@release/26.04 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main with: files_yaml: | test_java: @@ -57,14 +57,14 @@ jobs: checks: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@release/26.04 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" conda-java-tests: needs: [changed-files, checks] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 7fd050966a..c387f58d9a 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -25,7 +25,7 @@ on: jobs: conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.04 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main strategy: fail-fast: false matrix: diff --git a/java/cuvs-lucene/RAPIDS_BRANCH b/java/cuvs-lucene/RAPIDS_BRANCH index d5ea6ced53..ba2906d066 100644 --- a/java/cuvs-lucene/RAPIDS_BRANCH +++ b/java/cuvs-lucene/RAPIDS_BRANCH @@ -1 +1 @@ -release/26.04 +main From 86d8e84305e7649735529b39378dde3aa32391ba Mon Sep 17 00:00:00 2001 From: Jake Awe Date: Thu, 12 Mar 2026 10:57:10 -0700 Subject: [PATCH 074/105] Prepare release/26.04 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 8 ++++---- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/RAPIDS_BRANCH | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index d32385a53a..991ac62723 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -35,7 +35,7 @@ concurrency: jobs: java-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.04 # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index a04f2d26c5..b4663094fe 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.04 if: always() with: needs: ${{ toJSON(needs) }} @@ -47,7 +47,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@release/26.04 with: files_yaml: | test_java: @@ -57,14 +57,14 @@ jobs: checks: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@release/26.04 with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" conda-java-tests: needs: [changed-files, checks] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.04 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index c387f58d9a..7fd050966a 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -25,7 +25,7 @@ on: jobs: conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.04 strategy: fail-fast: false matrix: diff --git a/java/cuvs-lucene/RAPIDS_BRANCH b/java/cuvs-lucene/RAPIDS_BRANCH index ba2906d066..d5ea6ced53 100644 --- a/java/cuvs-lucene/RAPIDS_BRANCH +++ b/java/cuvs-lucene/RAPIDS_BRANCH @@ -1 +1 @@ -main +release/26.04 From d3700d8f0b8df214e8fd94f2c729bf62ba199047 Mon Sep 17 00:00:00 2001 From: Jake Awe <50372925+AyodeAwe@users.noreply.github.com> Date: Thu, 12 Mar 2026 12:58:18 -0500 Subject: [PATCH 075/105] Update to 26.06 (NVIDIA/cuvs-lucene#123) This PR updates the repository to version 26.06. This is part of the 26.04 release burndown process. --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 2 +- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/VERSION | 2 +- java/cuvs-lucene/bench/pom.xml | 2 +- java/cuvs-lucene/build.sh | 2 +- .../conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- .../conda/environments/all_cuda-131_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-131_arch-x86_64.yaml | 2 +- java/cuvs-lucene/dependencies.yaml | 2 +- java/cuvs-lucene/examples/README.md | 6 +++--- java/cuvs-lucene/examples/pom.xml | 4 ++-- java/cuvs-lucene/pom.xml | 4 ++-- 14 files changed, 18 insertions(+), 18 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index d32385a53a..7a02427333 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -49,7 +49,7 @@ jobs: branch: ${{ inputs.branch }} arch: "amd64" date: ${{ inputs.date }} - container_image: "rapidsai/ci-conda:26.04-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.06-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/build_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index a04f2d26c5..acfed0ce7d 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -78,7 +78,7 @@ jobs: build_type: pull-request node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:26.04-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.06-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index c387f58d9a..07914dfc88 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -39,5 +39,5 @@ jobs: sha: ${{ inputs.sha }} node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:26.04-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.06-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" diff --git a/java/cuvs-lucene/VERSION b/java/cuvs-lucene/VERSION index 0bd0e8a95b..cdb610a24d 100644 --- a/java/cuvs-lucene/VERSION +++ b/java/cuvs-lucene/VERSION @@ -1 +1 @@ -26.04.00 +26.06.00 diff --git a/java/cuvs-lucene/bench/pom.xml b/java/cuvs-lucene/bench/pom.xml index dbc1073b50..8f7262afe7 100644 --- a/java/cuvs-lucene/bench/pom.xml +++ b/java/cuvs-lucene/bench/pom.xml @@ -31,7 +31,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 26.04.0 + 26.06.0 commons-io diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index 92d9f321df..f3a7998142 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -8,7 +8,7 @@ set -e -u -o pipefail ARGS="$*" NUMARGS=$# -VERSION="26.04.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked +VERSION="26.06.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked GROUP_ID="com.nvidia.cuvs.lucene" function hasArg { diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index 8bd3b94861..e9953801d8 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -17,7 +17,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.04.* +- libcuvs==26.06.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index aca12acb97..12fad5b4f9 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -17,7 +17,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.04.* +- libcuvs==26.06.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml index 461c75751b..00444baa49 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml @@ -17,7 +17,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.04.* +- libcuvs==26.06.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml index 6f16b0c8ef..d1a66ae0e4 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml @@ -17,7 +17,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.04.* +- libcuvs==26.06.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index 99cec2a1ff..05922a300f 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -74,7 +74,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev - - libcuvs==26.04.* + - libcuvs==26.06.* java: common: - output_types: conda diff --git a/java/cuvs-lucene/examples/README.md b/java/cuvs-lucene/examples/README.md index a9917fb00c..ad4caf9617 100644 --- a/java/cuvs-lucene/examples/README.md +++ b/java/cuvs-lucene/examples/README.md @@ -16,7 +16,7 @@ cd .. Then do: ```sh -docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.04-cuda13.1.1-ubuntu24.04-py3.13 +docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.06-cuda13.1.1-ubuntu24.04-py3.13 ``` Inside the docker container (and in the `cuvs-lucene's` root directory) do: @@ -26,10 +26,10 @@ Inside the docker container (and in the `cuvs-lucene's` root directory) do: To run Accelerated HNSW example do: ```sh -mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.04.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.AcceleratedHnswExample +mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.06.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.AcceleratedHnswExample ``` To run the Index and Search on GPU example do: ```sh -mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.04.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.IndexAndSearchonGPUExample +mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.06.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.IndexAndSearchonGPUExample ``` diff --git a/java/cuvs-lucene/examples/pom.xml b/java/cuvs-lucene/examples/pom.xml index 768a9a8760..193374853b 100644 --- a/java/cuvs-lucene/examples/pom.xml +++ b/java/cuvs-lucene/examples/pom.xml @@ -11,7 +11,7 @@ com.nvidia.cuvs.lucene.examples examples - 26.04.0 + 26.06.0 examples @@ -51,7 +51,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 26.04.0 + 26.06.0 diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 0011497217..19479f10f3 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -12,7 +12,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 26.04.0 + 26.06.0 cuvs-lucene jar @@ -75,7 +75,7 @@ com.nvidia.cuvs cuvs-java - 26.04.0 + 26.06.0 From cf6d755bb3ad82eebc2a9d24ab853f91ee22393f Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Wed, 1 Apr 2026 13:40:51 -0400 Subject: [PATCH 076/105] Code improvements - Index and search on the GPU API (NVIDIA/cuvs-lucene#126) Fixes NVIDIA/cuvs-lucene#125 Summary of changes: - Update `CuVS2510GPUVectorsFormat` to update variable names and rearrange stuff for readability. - Update `CuVS2510GPUVectorsReader` to avoid loading of indexes on the GPU during merges to cut down on device memory usage, specifically during merges, fetch similarity functions via our provider instead of maintaining a list, implement `checkIntegrity` method, remove unused methods, update documentation, update search method to remove the existing workaround, and fix the prefiltering logic. - Update `CuVS2510GPUVectorsWriter` to remove some unused code, variable renaming, ordering stuff for better readability, update documentation, remove some unnecessary checks, plug in num threads for brute force correctly, and remove some obsolete logic from `mergeOneField`. - Update `GPUSearchParams` mainly to allow passing `CuVSIvfPqParams`, some other minor updates. - Update `Utils` to add a commonly used method. - Thanks to James Xia for suggesting that the queries can be built on the device directly [0664f7f](https://github.com/rapidsai/cuvs-lucene/pull/126/commits/0664f7fc09d395a9aad4824253accaee35bd713e) Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cuvs-lucene/pull/126 --- .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 21 +- .../cuvs/lucene/CuVS2510GPUVectorsFormat.java | 32 +- .../cuvs/lucene/CuVS2510GPUVectorsReader.java | 264 ++++++++------- .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 315 ++++++------------ .../nvidia/cuvs/lucene/GPUFieldWriter.java | 16 +- .../nvidia/cuvs/lucene/GPUSearchParams.java | 73 +++- .../java/com/nvidia/cuvs/lucene/Utils.java | 12 + .../cuvs/lucene/TestGPUSearchParams.java | 40 ++- 8 files changed, 388 insertions(+), 385 deletions(-) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index 321f1c1ca1..2de761a78a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -12,7 +12,7 @@ import org.apache.lucene.codecs.KnnVectorsFormat; /** - * cuVS based codec for GPU based vector search + * cuVS based codec for GPU based vector search that enables both - indexing and search on the GPU. * cuVS serialization formats are in experimental phase and hence backward compatibility cannot be guaranteed. * * @since 25.10 @@ -24,7 +24,7 @@ public class CuVS2510GPUSearchCodec extends FilterCodec { private KnnVectorsFormat format; /** - * Default constructor for {@link CuVS2510GPUSearchCodec} + * Default constructor for {@link CuVS2510GPUSearchCodec}. * * @throws Exception */ @@ -34,7 +34,8 @@ public CuVS2510GPUSearchCodec() throws Exception { } /** - * Initialize {@link CuVS2510GPUSearchCodec} with default parameter values. + * Initialize {@link CuVS2510GPUSearchCodec} with an instance of {@link GPUSearchParams} + * having default parameter values. * * @param name the name of the codec * @param delegate the delegate codec @@ -45,16 +46,22 @@ public CuVS2510GPUSearchCodec(String name, Codec delegate) { } /** - * Initialize the codec with custom parameter values. + * Initialize the codec with an instance of {@link GPUSearchParams} having either default + * or overridden parameter values. * * @param params An instance of {@link GPUSearchParams} - * @throws Exception Exception raised when initializing the codec. + * @throws Exception Exception raised when initializing the codec */ public CuVS2510GPUSearchCodec(GPUSearchParams params) throws Exception { this(NAME, LuceneProvider.getCodec("101")); initializeFormat(params); } + /** + * Initialize the {@link CuVS2510GPUVectorsFormat} instance using {@link GPUSearchParams}. + * + * @param params an instance of {@link GPUSearchParams} + */ private void initializeFormat(GPUSearchParams params) { try { format = new CuVS2510GPUVectorsFormat(params); @@ -67,7 +74,7 @@ private void initializeFormat(GPUSearchParams params) { } /** - * Get the configured {@link KnnVectorsFormat} + * Get the configured {@link KnnVectorsFormat}. * * @return the instance of the {@link KnnVectorsFormat} */ @@ -77,7 +84,7 @@ public KnnVectorsFormat knnVectorsFormat() { } /** - * Set the {@link KnnVectorsFormat} + * Set the {@link KnnVectorsFormat}. * * @param format the {@link KnnVectorsFormat} to set */ diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java index be7e6ab772..ccc61eaee1 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java @@ -8,9 +8,9 @@ import com.nvidia.cuvs.LibraryException; import java.io.IOException; -import java.util.logging.Logger; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.index.SegmentReadState; @@ -24,20 +24,18 @@ */ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { - @SuppressWarnings("unused") - private static final Logger log = Logger.getLogger(CuVS2510GPUVectorsFormat.class.getName()); - - private static final int maxDimensions = 4096; + private static final int MAX_DIMENSIONS = 4096; private static final LuceneProvider LUCENE_PROVIDER; private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; - private GPUSearchParams gpuSearchParams; - static final String CUVS_META_CODEC_NAME = "Lucene102CuVSVectorsFormatMeta"; - static final String CUVS_META_CODEC_EXT = "vemc"; - static final String CUVS_INDEX_CODEC_NAME = "Lucene102CuVSVectorsFormatIndex"; - static final String CUVS_INDEX_EXT = "vcag"; - static final int VERSION_START = 0; - static final int VERSION_CURRENT = VERSION_START; + public static final String CUVS_META_CODEC_NAME = "Lucene102CuVSVectorsFormatMeta"; + public static final String CUVS_META_CODEC_EXT = "vemc"; + public static final String CUVS_INDEX_CODEC_NAME = "Lucene102CuVSVectorsFormatIndex"; + public static final String CUVS_INDEX_EXT = "vcag"; + public static final int VERSION_START = 0; + public static final int VERSION_CURRENT = VERSION_START; + + private GPUSearchParams gpuSearchParams; static { try { @@ -59,7 +57,7 @@ public CuVS2510GPUVectorsFormat() { } /** - * Initializes the {@link CuVS2510GPUVectorsFormat} with the given threads, graph degree, etc. + * Initializes the {@link CuVS2510GPUVectorsFormat} with an instance of {@link GPUSearchParams}. * * @param gpuSearchParams An instance of {@link GPUSearchParams} * @throws LibraryException if the native library fails to load @@ -70,17 +68,17 @@ public CuVS2510GPUVectorsFormat(GPUSearchParams gpuSearchParams) { } /** - * Returns a {@link CuVS2510GPUVectorsWriter} to write the vectors to the index. + * Returns a KnnVectorsReader instance to write the vectors to the index. */ @Override - public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { assertIsSupported(); var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); return new CuVS2510GPUVectorsWriter(state, gpuSearchParams, flatWriter); } /** - * Returns a KnnVectorsReader to read the vectors from the index. + * Returns a KnnVectorsReader instance to read the vectors from the index. */ @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { @@ -93,6 +91,6 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException */ @Override public int getMaxDimensions(String fieldName) { - return maxDimensions; + return MAX_DIMENSIONS; } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java index 7bc29ab4b5..c783660d81 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java @@ -20,10 +20,11 @@ import com.nvidia.cuvs.CagraSearchParams; import com.nvidia.cuvs.CuVSMatrix; import java.io.IOException; +import java.util.BitSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.logging.Logger; +import java.util.Map.Entry; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.lucene.codecs.CodecUtil; @@ -43,10 +44,10 @@ import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IOContext.Context; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.ReadAdvice; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.hnsw.IntToIntFunction; @@ -57,15 +58,24 @@ */ public class CuVS2510GPUVectorsReader extends KnnVectorsReader { - @SuppressWarnings("unused") - private static final Logger log = Logger.getLogger(CuVS2510GPUVectorsReader.class.getName()); + private static final LuceneProvider LUCENE_PROVIDER; + private static final List VECTOR_SIMILARITY_FUNCTIONS; - private final FlatVectorsReader flatVectorsReader; // for reading the raw vectors + private final FlatVectorsReader flatVectorsReader; private final FieldInfos fieldInfos; private final IntObjectHashMap fields; private final IntObjectHashMap cuvsIndices; private final IndexInput cuvsIndexInput; + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + VECTOR_SIMILARITY_FUNCTIONS = LUCENE_PROVIDER.getSimilarityFunctions(); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + /** * Initializes the {@link CuVS2510GPUVectorsReader}, checks and loads the index. * @@ -79,7 +89,6 @@ public CuVS2510GPUVectorsReader(SegmentReadState state, FlatVectorsReader flatRe this.flatVectorsReader = flatReader; this.fieldInfos = state.fieldInfos; this.fields = new IntObjectHashMap<>(); - String metaFileName = IndexFileNames.segmentFileName( state.segmentInfo.name, state.segmentSuffix, CUVS_META_CODEC_EXT); @@ -104,7 +113,16 @@ public CuVS2510GPUVectorsReader(SegmentReadState state, FlatVectorsReader flatRe } var ioContext = state.context.withReadAdvice(ReadAdvice.SEQUENTIAL); cuvsIndexInput = openCuVSInput(state, versionMeta, ioContext); - cuvsIndices = loadCuVSIndices(); + /* + * Only load indexes on the GPU when this reader is opening for searches. + * Do not load indexes on the GPU when this reader is opening during merge calls. + * With this approach we reduce device memory usage by approximately 50% during merges. + */ + if (state.context.context().equals(Context.MERGE)) { + cuvsIndices = null; + } else { + cuvsIndices = loadCuVSIndices(); + } success = true; } finally { if (success == false) { @@ -185,17 +203,6 @@ private void readFields(ChecksumIndexInput meta) throws IOException { } } - // List of vector similarity functions. This list is defined here, in order - // to avoid an undesirable dependency on the declaration and order of values - // in VectorSimilarityFunction. The list values and order must be identical - // to that of {@link o.a.l.c.l.Lucene94FieldInfosFormat#SIMILARITY_FUNCTIONS}. - static final List SIMILARITY_FUNCTIONS = - List.of( - VectorSimilarityFunction.EUCLIDEAN, - VectorSimilarityFunction.DOT_PRODUCT, - VectorSimilarityFunction.COSINE, - VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT); - /** * Checks the distance function validity and returns it. * @@ -205,14 +212,14 @@ private void readFields(ChecksumIndexInput meta) throws IOException { */ static VectorSimilarityFunction readSimilarityFunction(DataInput input) throws IOException { int i = input.readInt(); - if (i < 0 || i >= SIMILARITY_FUNCTIONS.size()) { + if (i < 0 || i >= VECTOR_SIMILARITY_FUNCTIONS.size()) { throw new IllegalArgumentException("invalid distance function: " + i); } - return SIMILARITY_FUNCTIONS.get(i); + return VECTOR_SIMILARITY_FUNCTIONS.get(i); } /** - * Reads the vector encoding (The numeric datatype of the vector values) from the DataInput. + * Reads the vector encoding (The numeric data type of the vector values) from the DataInput. * * @param input instance of DataInput * @return the vector encoding @@ -282,9 +289,9 @@ private FieldEntry getFieldEntry(String field, VectorEncoding expectedEncoding) */ private IntObjectHashMap loadCuVSIndices() throws IOException { var indices = new IntObjectHashMap(); - for (var e : fields) { - var fieldEntry = e.value; - int fieldNumber = e.key; + for (var field : fields) { + var fieldEntry = field.value; + int fieldNumber = field.key; var cuvsIndex = loadCuVSIndex(fieldEntry); indices.put(fieldNumber, cuvsIndex); } @@ -292,7 +299,7 @@ private IntObjectHashMap loadCuVSIndices() throws IOException { } /** - * Loads the CAGRA and bruteforce index (if exists) onto the GPU. + * Loads the CAGRA and brute force index (if exists) onto the GPU. * * @param fieldEntry instance of {@link FieldEntry} * @return return the instance of {@link GPUIndex} @@ -301,7 +308,6 @@ private IntObjectHashMap loadCuVSIndices() throws IOException { private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { CagraIndex cagraIndex = null; BruteForceIndex bruteForceIndex = null; - try { long len = fieldEntry.cagraIndexLength(); if (len > 0) { @@ -311,7 +317,6 @@ private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { cagraIndex = CagraIndex.newBuilder(getCuVSResourcesInstance()).from(in).build(); } } - len = fieldEntry.bruteForceIndexLength(); if (len > 0) { long off = fieldEntry.bruteForceIndexOffset(); @@ -331,11 +336,12 @@ private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { */ @Override public void close() throws IOException { - var closeableStream = - Stream.concat( - Stream.of(flatVectorsReader, cuvsIndexInput), - stream(cuvsIndices.values().iterator()).map(cursor -> cursor.value)); + var closeableStream = Stream.of(flatVectorsReader, cuvsIndexInput); IOUtils.close(closeableStream::iterator); + if (cuvsIndices != null) { + var indexClosableStream = stream(cuvsIndices.values().iterator()).map(cursor -> cursor.value); + IOUtils.close(indexClosableStream::iterator); + } closeCuVSResourcesInstance(); } @@ -348,7 +354,8 @@ static Stream stream(Iterator iterator) { */ @Override public void checkIntegrity() throws IOException { - // TODO: Pending implementation + flatVectorsReader.checkIntegrity(); + CodecUtil.checksumEntireFile(cuvsIndexInput); } /** @@ -360,47 +367,33 @@ public FloatVectorValues getFloatVectorValues(String field) throws IOException { } /** - * Returns the FloatVectorValues for the given field. + * Returns the ByteVectorValues for the given field. * * This is not supported. */ @Override public ByteVectorValues getByteVectorValues(String field) { - throw new UnsupportedOperationException("byte vectors not supported"); + throw new UnsupportedOperationException("byte vectors are not currently supported"); } /** Native float to float function */ - public interface FloatToFloatFunction { + private interface FloatToFloatFunction { float apply(float v); } - /** - * Returns a long array from bits. - */ - static long[] bitsToLongArray(Bits bits) { - if (bits instanceof FixedBitSet fixedBitSet) { - return fixedBitSet.getBits(); - } else { - return FixedBitSet.copyOf(bits).getBits(); - } - } - /** * Get the score normalization function. * * @param sim instance of VectorSimilarityFunction * @return an instance of the FloatToFloatFunction */ - static FloatToFloatFunction getScoreNormalizationFunc(VectorSimilarityFunction sim) { + private static FloatToFloatFunction getScoreNormalizationFunc(VectorSimilarityFunction sim) { // TODO: check for different similarities return score -> (1f / (1f + score)); } - // This is a hack - https://github.com/rapidsai/cuvs/issues/696 - static final int FILTER_OVER_SAMPLE = 10; - /** - * Returns the k nearest neighbor documents using cuVS's CAGRA or Bruteforce algorithm for this field, to the given vector. + * Returns the k nearest neighbor documents using cuVS's CAGRA or brute force algorithm for this field, to the given vector. */ @Override public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) @@ -411,90 +404,121 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits } var fieldNumber = fieldInfos.fieldInfo(field).number; - - GPUIndex cuvsIndex = cuvsIndices.get(fieldNumber); + GPUIndex cuvsIndex = cuvsIndices != null ? cuvsIndices.get(fieldNumber) : null; if (cuvsIndex == null) { - throw new IllegalStateException("not index found for field:" + field); + throw new IllegalStateException("Index not found for field:" + field); } - int collectorTopK = knnCollector.k(); + final FloatVectorValues rawValues = flatVectorsReader.getFloatVectorValues(field); + final Bits acceptedOrds = rawValues.getAcceptOrds(acceptDocs); + BitSet[] mask = null; + int maskLength = 0; + int topK = knnCollector.k(); + if (acceptDocs != null) { - collectorTopK = knnCollector.k() * FILTER_OVER_SAMPLE; - } - final int topK = Math.min(collectorTopK, fieldEntry.count()); - assert topK > 0 : "Expected topK > 0, got:" + topK; - - Map result; - if (knnCollector.k() <= 1024 && cuvsIndex.getCagraIndex() != null) { - - CagraSearchParams searchParams; - if (knnCollector instanceof GPUPerLeafCuVSKnnCollector) { - GPUPerLeafCuVSKnnCollector collector = (GPUPerLeafCuVSKnnCollector) knnCollector; - searchParams = - new CagraSearchParams.Builder() - .withItopkSize(Math.max(collector.getiTopK(), topK)) - .withSearchWidth(collector.getSearchWidth()) - .build(); - } else { - // Setting itopK as topK because in any case iTopK should be ATLEAST equal to topK - searchParams = new CagraSearchParams.Builder().withItopkSize(topK).build(); + mask = new BitSet[1]; // As there is only one query "target" + mask[0] = new BitSet(acceptedOrds.length()); + /* + * Need to find if there is a better alternative for below loop + * in subsequent code improvement iterations. This is needed as + * there is a difference between Lucene's Bits "acceptDocs" and + * what our API accepts. + */ + for (int i = 0; i < acceptedOrds.length(); i++) { + if (acceptedOrds.get(i)) { + mask[0].set(i); + } } + topK = Math.min(knnCollector.k() + 10, mask[0].cardinality()); + maskLength = mask[0].length(); + } - var query = - new CagraQuery.Builder(getCuVSResourcesInstance()) - .withTopK(topK) - .withSearchParams(searchParams) - .withQueryVectors(CuVSMatrix.ofArray(new float[][] {target})) - .build(); - - CagraIndex cagraIndex = cuvsIndex.getCagraIndex(); + try { List> searchResult = null; - try { + if (knnCollector.k() <= 1024 && cuvsIndex.getCagraIndex() != null) { + CagraSearchParams searchParams; + if (knnCollector instanceof GPUPerLeafCuVSKnnCollector) { + GPUPerLeafCuVSKnnCollector collector = (GPUPerLeafCuVSKnnCollector) knnCollector; + searchParams = + new CagraSearchParams.Builder() + .withItopkSize(Math.max(collector.getiTopK(), topK)) + .withSearchWidth(collector.getSearchWidth()) + .build(); + } else { + // Setting itopK as topK because in any case iTopK should be ATLEAST equal to topK + searchParams = new CagraSearchParams.Builder().withItopkSize(topK).build(); + } + CagraIndex cagraIndex = cuvsIndex.getCagraIndex(); + assert cagraIndex != null; + CagraQuery query = null; + + CuVSMatrix.Builder builder = + CuVSMatrix.deviceBuilder( + getCuVSResourcesInstance(), 1, target.length, CuVSMatrix.DataType.FLOAT); + builder.addVector(target); + CuVSMatrix queryVector = builder.build(); + + if (acceptDocs != null) { + query = + new CagraQuery.Builder(getCuVSResourcesInstance()) + .withTopK(topK) + .withSearchParams(searchParams) + .withQueryVectors(queryVector) + .withPrefilter(mask[0], maskLength) + .build(); + } else { + query = + new CagraQuery.Builder(getCuVSResourcesInstance()) + .withTopK(topK) + .withSearchParams(searchParams) + .withQueryVectors(queryVector) + .build(); + } searchResult = cagraIndex.search(query).getResults(); - } catch (Throwable t) { - Utils.handleThrowable(t); - } - // List expected to have only one entry because of single query "target". - assert searchResult.size() == 1; - result = searchResult.getFirst(); - } else { - BruteForceIndex bruteforceIndex = cuvsIndex.getBruteforceIndex(); - assert bruteforceIndex != null; - var queryBuilder = - new BruteForceQuery.Builder(getCuVSResourcesInstance()) - .withQueryVectors(new float[][] {target}) - .withTopK(topK); - BruteForceQuery query = queryBuilder.build(); - - List> searchResult = null; - try { + } else { + BruteForceIndex bruteforceIndex = cuvsIndex.getBruteforceIndex(); + assert bruteforceIndex != null; + BruteForceQuery query = null; + float[][] queryVector = new float[][] {target}; + if (acceptDocs != null) { + query = + new BruteForceQuery.Builder(getCuVSResourcesInstance()) + .withQueryVectors(queryVector) + .withPrefilters(mask, maskLength) + .withTopK(topK) + .build(); + } else { + query = + new BruteForceQuery.Builder(getCuVSResourcesInstance()) + .withQueryVectors(queryVector) + .withTopK(topK) + .build(); + } searchResult = bruteforceIndex.search(query).getResults(); - } catch (Throwable t) { - Utils.handleThrowable(t); } - assert searchResult.size() == 1; - result = searchResult.getFirst(); - } - assert result != null; - final var rawValues = flatVectorsReader.getFloatVectorValues(field); - final Bits acceptedOrds = rawValues.getAcceptOrds(acceptDocs); - final var ordToDocFunction = (IntToIntFunction) rawValues::ordToDoc; - final var scoreCorrectionFunction = getScoreNormalizationFunc(fieldEntry.similarityFunction); + // List expected to have only one entry because of single query "target". + assert searchResult.size() == 1; + final IntToIntFunction ordToDocFunction = (IntToIntFunction) rawValues::ordToDoc; + final FloatToFloatFunction scoreCorrectionFunction = + getScoreNormalizationFunc(fieldEntry.similarityFunction); - for (var entry : result.entrySet()) { - int ord = entry.getKey(); - float score = entry.getValue(); - if (acceptedOrds == null || acceptedOrds.get(ord)) { + for (Entry entry : searchResult.getFirst().entrySet()) { + int ord = entry.getKey(); + float score = entry.getValue(); if (knnCollector.earlyTerminated()) { break; } - assert ord >= 0 : "unexpected ord: " + ord; - int doc = ordToDocFunction.apply(ord); + if (ord < 0) { + continue; + } float correctedScore = scoreCorrectionFunction.apply(score); + int doc = ordToDocFunction.apply(ord); knnCollector.incVisitedCount(1); knnCollector.collect(doc, correctedScore); } + } catch (Throwable t) { + Utils.handleThrowable(t); } } @@ -506,7 +530,7 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits @Override public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { - throw new UnsupportedOperationException("byte vectors not supported"); + throw new UnsupportedOperationException("Byte vectors are not currently supported"); } /** @@ -526,7 +550,7 @@ record FieldEntry( * Returns an instance of FieldEntry. * * @param input instance of IndexInput - * @param vectorEncoding The numeric datatype of the vector values + * @param vectorEncoding The numeric data type of the vector values * @param similarityFunction Vector similarity function; used in search to return top K most similar vectors to a target vector * @return an instance of FieldEntry * @throws IOException I/O Exceptions @@ -562,7 +586,7 @@ static FieldEntry readEntry( * @param in * @throws CorruptIndexException */ - static void checkVersion(int versionMeta, int versionVectorData, IndexInput in) + private static void checkVersion(int versionMeta, int versionVectorData, IndexInput in) throws CorruptIndexException { if (versionMeta != versionVectorData) { throw new CorruptIndexException( @@ -588,7 +612,7 @@ public FieldInfos getFieldInfos() { /** * Gets the map of {@link GPUIndex} objects. * - * @return the map of gpu index objects + * @return the map of GPU index objects */ public IntObjectHashMap getCuvsIndexes() { return cuvsIndices; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index cba32e21de..2bb6732cb9 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -11,6 +11,7 @@ import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.VERSION_CURRENT; import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.closeCuVSResourcesInstance; import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.getCuVSResourcesInstance; +import static com.nvidia.cuvs.lucene.Utils.info; import static org.apache.lucene.index.VectorEncoding.FLOAT32; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; @@ -27,8 +28,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.logging.Logger; -import java.util.stream.IntStream; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.KnnFieldVectorsWriter; import org.apache.lucene.codecs.KnnVectorsReader; @@ -60,26 +59,16 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { private static final long SHALLOW_RAM_BYTES_USED = shallowSizeOfInstance(CuVS2510GPUVectorsWriter.class); - - @SuppressWarnings("unused") - private static final Logger log = Logger.getLogger(CuVS2510GPUVectorsWriter.class.getName()); - - /** The name of the CUVS component for the info-stream * */ - private static final String CUVS_COMPONENT = "CUVS"; - + private static final String COMPONENT = "CuVS2510GPUVectorsWriter"; private static final LuceneProvider LUCENE_PROVIDER; private static final List VECTOR_SIMILARITY_FUNCTIONS; - - // The minimum number of vectors in the dataset required before - // we attempt to build a Cagra index - static final int MIN_CAGRA_INDEX_SIZE = 2; + private static final int MIN_CAGRA_INDEX_SIZE = 2; private final GPUSearchParams gpuSearchParams; - private final FlatVectorsWriter flatVectorsWriter; private final List fields = new ArrayList<>(); - private IndexOutput meta = null, cuvsIndex = null; private final InfoStream infoStream; + private IndexOutput meta = null, cuvsIndex = null; private boolean finished; static { @@ -96,13 +85,13 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { */ public enum IndexType { - /** Builds a Cagra index. */ + /** Builds a CAGRA index. */ CAGRA(true, false), /** Builds a Brute Force index. */ BRUTE_FORCE(false, true), - /** Builds a Cagra and a Brute Force index. */ + /** Builds both - CAGRA and Brute Force indexes. */ CAGRA_AND_BRUTE_FORCE(true, true); private final boolean cagra, bruteForce; @@ -111,21 +100,11 @@ public enum IndexType { this.bruteForce = bruteForce; } - /** - * Check if cagra is set - * - * @return is cagra set - */ - public boolean cagra() { + public boolean isCagra() { return cagra; } - /** - * Check if bruteforce is set - * - * @return is bruteforce set - */ - public boolean bruteForce() { + public boolean isBruteForce() { return bruteForce; } } @@ -146,16 +125,13 @@ public CuVS2510GPUVectorsWriter( this.gpuSearchParams = gpuSearchParams; this.flatVectorsWriter = flatVectorsWriter; this.infoStream = state.infoStream; - String metaFileName = IndexFileNames.segmentFileName( state.segmentInfo.name, state.segmentSuffix, CUVS_META_CODEC_EXT); String cagraFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, CUVS_INDEX_EXT); - boolean success = false; try { - meta = state.directory.createOutput(metaFileName, state.context); cuvsIndex = state.directory.createOutput(cagraFileName, state.context); CodecUtil.writeIndexHeader( @@ -170,7 +146,6 @@ public CuVS2510GPUVectorsWriter( VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); - success = true; } finally { if (success == false) { @@ -186,7 +161,7 @@ public CuVS2510GPUVectorsWriter( public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { var encoding = fieldInfo.getVectorEncoding(); if (encoding != FLOAT32) { - throw new IllegalArgumentException("expected float32, got:" + encoding); + throw new IllegalArgumentException("Expected float32, got:" + encoding); } var writer = Objects.requireNonNull(flatVectorsWriter.addField(fieldInfo)); @SuppressWarnings("unchecked") @@ -197,102 +172,55 @@ public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException } /** - * Returns a string containing meta information like graph degree etc. - * - * @param size index size - * @param args other parameters like graph degree, Intermediate graph degree, etc. - * @return the string containing the meta information - */ - static String indexMsg(int size, int... args) { - StringBuilder sb = new StringBuilder("cagra index params"); - sb.append(": size=").append(size); - sb.append(", intGraphDegree=").append(args[0]); - sb.append(", actualIntGraphDegree=").append(args[1]); - sb.append(", graphDegree=").append(args[2]); - sb.append(", actualGraphDegree=").append(args[3]); - return sb.toString(); - } - - /** - * Builds and returns an instance of CagraIndexParams. - * - * @param size the size of the index - * @return an instance of CagraIndexParams - */ - private CagraIndexParams cagraIndexParams(int size) { - if (size < 2) { - // https://github.com/rapidsai/cuvs/issues/666 - throw new IllegalArgumentException("cagra index must be greater than 2"); - } - - return new CagraIndexParams.Builder() - .withNumWriterThreads(gpuSearchParams.getWriterThreads()) - .withIntermediateGraphDegree(gpuSearchParams.getIntermediateGraphDegree()) - .withGraphDegree(gpuSearchParams.getGraphdegree()) - .withCagraGraphBuildAlgo(gpuSearchParams.getCagraGraphBuildAlgo()) - .build(); - } - - /** - * Utility to print info/debug messages via InfoStream. - * - * @param msg - */ - private void info(String msg) { - if (infoStream.isEnabled(CUVS_COMPONENT)) { - infoStream.message(CUVS_COMPONENT, msg); - } - } - - /** - * Creates CAGRA and/or Bruteforce indexes and writes them. + * Creates CAGRA and/or brute force indexes and writes them. * * @param fieldInfo Instance of the FieldInFo to use * @param vectors list of float vectors to index * @throws IOException */ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOException { - if (vectors.size() == 0) { + if (vectors != null && vectors.size() == 0) { writeEmpty(fieldInfo); return; } long cagraIndexOffset, cagraIndexLength = 0L; long bruteForceIndexOffset, bruteForceIndexLength = 0L; - // workaround for the minimum number of vectors for Cagra + /* + * CAGRA has an issue when asked to build an index with just one vector. + * Hence, we currently fallback to brute force in such a case. + */ IndexType indexType = - gpuSearchParams.getIndexType().cagra() && vectors.size() < MIN_CAGRA_INDEX_SIZE + gpuSearchParams.getIndexType().isCagra() && vectors.size() < MIN_CAGRA_INDEX_SIZE ? IndexType.BRUTE_FORCE : gpuSearchParams.getIndexType(); try { - cagraIndexOffset = cuvsIndex.getFilePointer(); - if (indexType.cagra()) { + if (indexType.isCagra()) { + var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); try { - var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - CuVSMatrix dataset = + CuVSMatrix cagraDataset = Utils.createFloatMatrix( vectors, fieldInfo.getVectorDimension(), getCuVSResourcesInstance()); - writeCagraIndex(cagraIndexOutputStream, dataset); + writeCagraIndex(cagraIndexOutputStream, cagraDataset); } catch (Throwable t) { - Utils.handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); - // workaround for cuVS issue + // Fallback to brute force in a few cases, for now. + Utils.handleThrowableWithIgnore(t, t.getMessage()); indexType = IndexType.BRUTE_FORCE; } cagraIndexLength = cuvsIndex.getFilePointer() - cagraIndexOffset; } - bruteForceIndexOffset = cuvsIndex.getFilePointer(); - if (indexType.bruteForce()) { + if (indexType.isBruteForce()) { var bruteForceIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - CuVSMatrix dataset = + CuVSMatrix bruteforceDataset = Utils.createFloatMatrix( vectors, fieldInfo.getVectorDimension(), getCuVSResourcesInstance()); - writeBruteForceIndex(bruteForceIndexOutputStream, dataset); + + writeBruteForceIndex(bruteForceIndexOutputStream, bruteforceDataset); bruteForceIndexLength = cuvsIndex.getFilePointer() - bruteForceIndexOffset; } - writeMeta( fieldInfo, vectors.size(), @@ -313,50 +241,46 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro * @throws Throwable */ private void writeCagraIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { - if (dataset.size() < 2) { - throw new IllegalArgumentException(dataset.size() + " vectors, less than min [2] required"); - } - CagraIndexParams params = cagraIndexParams((int) dataset.size()); - long startTime = System.nanoTime(); + CagraIndexParams params = + new CagraIndexParams.Builder() + .withNumWriterThreads(gpuSearchParams.getWriterThreads()) + .withIntermediateGraphDegree(gpuSearchParams.getIntermediateGraphDegree()) + .withGraphDegree(gpuSearchParams.getGraphdegree()) + .withCagraGraphBuildAlgo(gpuSearchParams.getCagraGraphBuildAlgo()) + .withCuVSIvfPqParams(gpuSearchParams.getCuVSIvfPqParams()) + .build(); CagraIndex index = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(dataset) .withIndexParams(params) .build(); - long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - startTime); - info("Cagra index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); - Path tmpFile = - Files.createTempFile(getCuVSResourcesInstance().tempDirectory(), "tmpindex", "cag"); - index.serialize(os, tmpFile); + index.serialize(os); index.close(); } /** - * Builds and writes the Bruteforce index. + * Builds and writes the brute force index. * - * @param os Instance of OutputStream to write the index to - * @param dataset Instance of CuVSMatrix that holds the dataset + * @param os Instance of OutputStream to write the index bytes to + * @param dataset Instance of CuVSMatrix that holds the data set * @throws Throwable */ private void writeBruteForceIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { BruteForceIndexParams params = new BruteForceIndexParams.Builder() - .withNumWriterThreads(32) // TODO: Make this configurable. + .withNumWriterThreads(gpuSearchParams.getWriterThreads()) .build(); - long startTime = System.nanoTime(); var index = BruteForceIndex.newBuilder(getCuVSResourcesInstance()) .withIndexParams(params) .withDataset(dataset) .build(); - long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - startTime); - info("bf index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); index.serialize(os); index.close(); } /** - * Creates the CAGRA and/or Bruteforce indexes and writes them to the disk. + * Creates the CAGRA and/or brute force indexes and writes them to the disk. */ @Override public void flush(int maxDoc, DocMap sortMap) throws IOException { @@ -389,16 +313,13 @@ private void writeField(GPUFieldWriter fieldData) throws IOException { */ private void writeSortingField(GPUFieldWriter fieldData, Sorter.DocMap sortMap) throws IOException { - DocsWithFieldSet oldDocsWithFieldSet = fieldData.getDocsWithFieldSet(); - final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; // new ord to old ord + final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; mapOldOrdToNewOrd(oldDocsWithFieldSet, sortMap, null, new2OldOrd, null); - List sortedVectors = new ArrayList(); for (int i = 0; i < fieldData.getVectors().size(); i++) { sortedVectors.add(fieldData.getVectors().get(new2OldOrd[i])); } - writeFieldInternal(fieldData.fieldInfo(), sortedVectors); } @@ -419,8 +340,8 @@ private void writeEmpty(FieldInfo fieldInfo) throws IOException { * @param count number of vectors * @param cagraIndexOffset CAGRA index offset * @param cagraIndexLength CAGRA index length - * @param bruteForceIndexOffset Bruteforce index offset - * @param bruteForceIndexLength Bruteforce index length + * @param bruteForceIndexOffset Brute force index offset + * @param bruteForceIndexLength Brute force index length * @throws IOException I/O Exceptions */ private void writeMeta( @@ -448,36 +369,27 @@ static int distFuncToOrd(VectorSimilarityFunction func) { return (byte) i; } } - throw new IllegalArgumentException("invalid distance function: " + func); + throw new IllegalArgumentException("Invalid distance function: " + func); } - // We currently ignore this, until cuVS supports tiered indices - private static final String CANNOT_GENERATE_CAGRA = - """ - Could not generate an intermediate CAGRA graph because the initial \ - kNN graph contains too many invalid or duplicated neighbor nodes. \ - This error can occur, for example, if too many overflows occur \ - during the norm computation between the dataset vectors\ - """; - /** * Uses the cuVS API to merge CAGRA indexes. * + * This is currently (and intentionally) marked as unused and will be plugged in later. + * * @param fieldInfo instance of the FieldInfo * @param mergeState instance of the MergeState * @throws IOException I/O Exceptions */ + @SuppressWarnings("unused") private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throws IOException { try { - List cagraIndexes = new ArrayList<>(); // We need this count so that the merged segment's meta information has the vector count. int totalVectorCount = 0; - for (int i = 0; i < mergeState.knnVectorsReaders.length; i++) { KnnVectorsReader knnReader = mergeState.knnVectorsReaders[i]; // Access the CAGRA index for this field from the reader - if (knnReader != null) { if (knnReader instanceof CuVS2510GPUVectorsReader cvr) { if (cvr != null) { @@ -495,46 +407,13 @@ private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throw } } assert cagraIndexes.size() > 1; - CagraIndex mergedIndex = CagraIndex.merge(cagraIndexes.toArray(new CagraIndex[cagraIndexes.size()])); writeMergedCagraIndex(fieldInfo, mergedIndex, totalVectorCount); - info("Successfully merged " + cagraIndexes.size() + " CAGRA indexes using native merge API"); - - } catch (Throwable t) { - Utils.handleThrowable(t); - } - } - - /** - * Creates List from merged vectors. - */ - private List createListFromMergedVectors(FloatVectorValues mergedVectorValues) - throws IOException { - List res = new ArrayList(); - KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); - for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { - int ordinal = iter.index(); - float[] vector = mergedVectorValues.vectorValue(ordinal); - res.add(vector.clone()); - } - return res; - } - - /** - * Fallback method that rebuilds indexes from merged vectors. - * Used when native CAGRA merge() is not possible. Also used - * when non-CAGRA index types are used (for e.g. Brute Force index). - */ - private void vectorBasedMerge(FieldInfo fieldInfo, MergeState mergeState) throws IOException { - if (fieldInfo.getVectorEncoding() != FLOAT32) { - throw new AssertionError("Only Float32 supported"); - } - try { - List dataset = - createListFromMergedVectors( - KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState)); - writeFieldInternal(fieldInfo, dataset); + info( + infoStream, + COMPONENT, + "Successfully merged " + cagraIndexes.size() + " CAGRA indexes using native merge API"); } catch (Throwable t) { Utils.handleThrowable(t); } @@ -547,9 +426,7 @@ private CagraIndex getCagraIndexFromReader(CuVS2510GPUVectorsReader reader, Stri try { IntObjectHashMap cuvsIndices = reader.getCuvsIndexes(); FieldInfos fieldInfos = reader.getFieldInfos(); - FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldName); - if (fieldInfo != null) { GPUIndex cuvsIndex = cuvsIndices.get(fieldInfo.number); if (cuvsIndex != null) { @@ -557,8 +434,11 @@ private CagraIndex getCagraIndexFromReader(CuVS2510GPUVectorsReader reader, Stri } } } catch (Exception e) { - e.printStackTrace(); - info("Failed to extract CAGRA index for field " + fieldName + ": " + e.getMessage()); + info( + infoStream, + COMPONENT, + "Failed to extract CAGRA index for field " + fieldName + ": " + e.getMessage()); + throw e; } return null; } @@ -571,53 +451,67 @@ private void writeMergedCagraIndex(FieldInfo fieldInfo, CagraIndex mergedIndex, try { long cagraIndexOffset = cuvsIndex.getFilePointer(); var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - - // Serialize the merged index Path tmpFile = Files.createTempFile(getCuVSResourcesInstance().tempDirectory(), "mergedindex", "cag"); mergedIndex.serialize(cagraIndexOutputStream, tmpFile); long cagraIndexLength = cuvsIndex.getFilePointer() - cagraIndexOffset; - writeMeta(fieldInfo, vectorCount, cagraIndexOffset, cagraIndexLength, 0L, 0L); - - // Clean up the merged index mergedIndex.close(); } catch (Throwable t) { Utils.handleThrowable(t); } } + /** + * Creates List from merged vectors. + */ + private List createListFromMergedVectors(FloatVectorValues mergedVectorValues) + throws IOException { + List res = new ArrayList(); + KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); + for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { + int ordinal = iter.index(); + float[] vector = mergedVectorValues.vectorValue(ordinal); + res.add(vector.clone()); + } + return res; + } + + /** + * Fallback method that rebuilds indexes from merged vectors. + * Used when native CAGRA merge() is not possible. Also used + * when non-CAGRA index types are used (for e.g. Brute Force index). + */ + private void vectorBasedMerge(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + try { + List dataset = + createListFromMergedVectors( + KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState)); + writeFieldInternal(fieldInfo, dataset); + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + /** * Write field for merging. */ @Override public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { flatVectorsWriter.mergeOneField(fieldInfo, mergeState); + vectorBasedMerge(fieldInfo, mergeState); + } - if (gpuSearchParams.getIndexType().cagra() && !gpuSearchParams.getIndexType().bruteForce()) { - // Since CAGRA merge does not support merging of indexes with purging of deletes, - // we fallback to vector-based re-indexing. Issue: - // https://github.com/rapidsai/cuvs/issues/1253 - boolean hasDeletions = - IntStream.range(0, mergeState.liveDocs.length) - .anyMatch( - i -> - mergeState.liveDocs[i] == null - || IntStream.range(0, mergeState.maxDocs[i]) - .anyMatch(j -> !mergeState.liveDocs[i].get(j))); - - if (mergeState.knnVectorsReaders.length > 1 && !hasDeletions) { - mergeCagraIndexes(fieldInfo, mergeState); - } else { - // CAGRA's merge API does not handle the trivial case of merging 1 index. - vectorBasedMerge(fieldInfo, mergeState); - } - - } else { - // If there is a Brute Force index then re-index using the vectors even if there is a CAGRA - // index. - vectorBasedMerge(fieldInfo, mergeState); + /** + * Returns the memory usage of this object in bytes. + */ + @Override + public long ramBytesUsed() { + long total = SHALLOW_RAM_BYTES_USED; + for (var field : fields) { + total += field.ramBytesUsed(); } + return total; } /** @@ -630,7 +524,6 @@ public void finish() throws IOException { } finished = true; flatVectorsWriter.finish(); - if (meta != null) { // write end of fields marker meta.writeInt(-1); @@ -649,16 +542,4 @@ public void close() throws IOException { IOUtils.close(meta, cuvsIndex, flatVectorsWriter); closeCuVSResourcesInstance(); } - - /** - * Returns the memory usage of this object in bytes. - */ - @Override - public long ramBytesUsed() { - long total = SHALLOW_RAM_BYTES_USED; - for (var field : fields) { - total += field.ramBytesUsed(); - } - return total; - } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java index 92531bb86e..45255da499 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; @@ -55,6 +55,20 @@ List getVectors() { return flatFieldVectorsWriter.getVectors(); } + /** + * Gets the vector dimension. + * + * @return the vector dimension + */ + int getVectorDimension() { + List vectors = flatFieldVectorsWriter.getVectors(); + if (vectors != null && vectors.size() > 0) { + return vectors.get(0).length; + } else { + return -1; + } + } + /** * Gets the field info that holds the description of the field. * diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java index dade15d2ee..cca0233097 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java @@ -6,8 +6,10 @@ package com.nvidia.cuvs.lucene; import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.CuVSIvfPqParams; import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; import java.util.Objects; +import java.util.function.Supplier; public class GPUSearchParams { @@ -15,18 +17,31 @@ public class GPUSearchParams { * TODO: Update boundaries for all parameters when a consensus is reached. * Issue: https://github.com/rapidsai/cuvs-lucene/issues/99 */ - private static final int MIN_WRITER_THREADS = 1; - private static final int MAX_WRITER_THREADS = 32; - private static final int MIN_INT_GRAPH_DEG = 2; - private static final int MAX_INT_GRAPH_DEG = 128; - private static final int MIN_GRAPH_DEG = 1; - private static final int MAX_GRAPH_DEG = 64; + public static final int MIN_WRITER_THREADS = 1; + public static final int MAX_WRITER_THREADS = 512; + public static final int MIN_INT_GRAPH_DEG = 2; + public static final int MAX_INT_GRAPH_DEG = 512; + public static final int MIN_GRAPH_DEG = 1; + public static final int MAX_GRAPH_DEG = 512; + + public static final int DEFAULT_INT_GRAPH_DEGREE = 128; + public static final int DEFAULT_GRAPH_DEGREE = 64; + public static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = + CagraGraphBuildAlgo.NN_DESCENT; + public static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; + public static final int DEFAULT_WRITER_THREADS = 1; + + public static final Supplier DEFAULT_IVF_PQ_PARAMS = + () -> { + return new CuVSIvfPqParams.Builder().build(); + }; private final int writerThreads; private final int intermediateGraphDegree; private final int graphdegree; private final CagraGraphBuildAlgo cagraGraphBuildAlgo; private final IndexType indexType; + private final CuVSIvfPqParams cuVSIvfPqParams; /** * Constructs an instance of {@link GPUSearchParams} with specific parameter values. @@ -36,19 +51,22 @@ public class GPUSearchParams { * @param graphdegree The graph degree to use while building the CAGRA index. * @param cagraGraphBuildAlgo The CAGRA build algorithm to use. * @param indexType The type of index to build - CAGRA, BRUTEFORCE, or both. + * @param cuVSIvfPqParams An instance of CuVSIvfPqParams containing IVF_PQ specific parameters. */ private GPUSearchParams( int writerThreads, int intermediateGraphDegree, int graphdegree, CagraGraphBuildAlgo cagraGraphBuildAlgo, - IndexType indexType) { + IndexType indexType, + CuVSIvfPqParams cuVSIvfPqParams) { super(); this.writerThreads = writerThreads; this.intermediateGraphDegree = intermediateGraphDegree; this.graphdegree = graphdegree; this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; this.indexType = indexType; + this.cuVSIvfPqParams = cuVSIvfPqParams; } /** @@ -96,6 +114,15 @@ public IndexType getIndexType() { return indexType; } + /** + * Get the instance of CuVSIvfPqParams + * + * @return an instance of CuVSIvfPqParams + */ + public CuVSIvfPqParams getCuVSIvfPqParams() { + return cuVSIvfPqParams; + } + @Override public String toString() { return "GPUSearchParams [writerThreads=" @@ -116,11 +143,12 @@ public String toString() { */ public static class Builder { - private int writerThreads = 1; - private int intermediateGraphDegree = 128; - private int graphdegree = 64; - private CagraGraphBuildAlgo cagraGraphBuildAlgo = CagraGraphBuildAlgo.NN_DESCENT; - private IndexType indexType = IndexType.CAGRA; + private int writerThreads = DEFAULT_WRITER_THREADS; + private int intermediateGraphDegree = DEFAULT_INT_GRAPH_DEGREE; + private int graphdegree = DEFAULT_GRAPH_DEGREE; + private CagraGraphBuildAlgo cagraGraphBuildAlgo = DEFAULT_CAGRA_GRAPH_BUILD_ALGO; + private IndexType indexType = DEFAULT_INDEX_TYPE; + private CuVSIvfPqParams cuVSIvfPqParams = null; /** * Set the number of cuVS writer threads while building the index @@ -185,6 +213,17 @@ public Builder withIndexType(IndexType indexType) { return this; } + /** + * Set the instance of {@link CuVSIvfPqParams} + * + * @param cuVSIvfPqParams + * @return instance of {@link Builder} + */ + public Builder withCuVSIvfPqParams(CuVSIvfPqParams cuVSIvfPqParams) { + this.cuVSIvfPqParams = cuVSIvfPqParams; + return this; + } + /** * Validates the input parameters. * @@ -230,9 +269,17 @@ private void validate() throws IllegalArgumentException { * @return instance of {@link GPUSearchParams} */ public GPUSearchParams build() { + if (Objects.isNull(cuVSIvfPqParams)) { + cuVSIvfPqParams = DEFAULT_IVF_PQ_PARAMS.get(); + } validate(); return new GPUSearchParams( - writerThreads, intermediateGraphDegree, graphdegree, cagraGraphBuildAlgo, indexType); + writerThreads, + intermediateGraphDegree, + graphdegree, + cagraGraphBuildAlgo, + indexType, + cuVSIvfPqParams); } } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java index 21898f8ee4..91d7f5a0b1 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.lucene.util.InfoStream; /** * This class provides common static utility methods. @@ -168,4 +169,15 @@ static void handleThrowableWithIgnore(Throwable t, String msg) throws IOExceptio } handleThrowable(t); } + + /** + * Utility to print info/debug messages via InfoStream. + * + * @param msg + */ + static void info(InfoStream infoStream, String component, String msg) { + if (infoStream.isEnabled(component)) { + infoStream.message(component, msg); + } + } } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java index 41d660d62e..495033172f 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java @@ -5,8 +5,20 @@ package com.nvidia.cuvs.lucene; -import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; -import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_CAGRA_GRAPH_BUILD_ALGO; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_GRAPH_DEGREE; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_INDEX_TYPE; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_INT_GRAPH_DEGREE; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_WRITER_THREADS; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MAX_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MAX_INT_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MAX_WRITER_THREADS; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MIN_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MIN_INT_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MIN_WRITER_THREADS; +import static java.lang.Integer.MAX_VALUE; +import static java.lang.Integer.MIN_VALUE; + import java.util.Random; import java.util.logging.Logger; import org.apache.lucene.tests.util.LuceneTestCase; @@ -25,17 +37,19 @@ public class TestGPUSearchParams extends LuceneTestCase { @Test public void testGPUSearchParamsDefaultValues() { GPUSearchParams params = new GPUSearchParams.Builder().build(); - assertEquals(64, params.getGraphdegree()); - assertEquals(128, params.getIntermediateGraphDegree()); - assertEquals(1, params.getWriterThreads()); - assertEquals(CagraGraphBuildAlgo.NN_DESCENT, params.getCagraGraphBuildAlgo()); - assertEquals(IndexType.CAGRA, params.getIndexType()); + assertEquals(DEFAULT_GRAPH_DEGREE, params.getGraphdegree()); + assertEquals(DEFAULT_INT_GRAPH_DEGREE, params.getIntermediateGraphDegree()); + assertEquals(DEFAULT_WRITER_THREADS, params.getWriterThreads()); + assertEquals(DEFAULT_CAGRA_GRAPH_BUILD_ALGO, params.getCagraGraphBuildAlgo()); + assertEquals(DEFAULT_INDEX_TYPE, params.getIndexType()); } @Test public void testGPUSearchParamsInvalidGraphDegree() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(65, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_GRAPH_DEG), random.nextInt(MAX_GRAPH_DEG + 1, MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new GPUSearchParams.Builder().withGraphDegree(v).build()); @@ -45,7 +59,10 @@ public void testGPUSearchParamsInvalidGraphDegree() { @Test public void testGPUSearchParamsInvalidIntermediateGraphDegree() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(129, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_INT_GRAPH_DEG), + random.nextInt(MAX_INT_GRAPH_DEG + 1, MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new GPUSearchParams.Builder().withIntermediateGraphDegree(v).build()); @@ -55,7 +72,10 @@ public void testGPUSearchParamsInvalidIntermediateGraphDegree() { @Test public void testGPUSearchParamsInvalidWriterThreads() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(2, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_WRITER_THREADS), + random.nextInt(MAX_WRITER_THREADS + 1, MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new GPUSearchParams.Builder().withWriterThreads(v).build()); From a3ad02e60d473bb69e257c6427047c821d4f76d1 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Thu, 2 Apr 2026 19:47:41 -0400 Subject: [PATCH 077/105] Code improvements - AcceleratedHNSW API (NVIDIA/cuvs-lucene#115) - Fixes NVIDIA/cuvs-lucene#114 - Mainly code deduplication by plugging in `AcceleratedHNSWUtils` and some rearrangement. - Summary of code changes: - Update `AcceleratedHNSWParams` to incorporate feedback suggestions from NVIDIA/cuvs-lucene#120 and update Javadocs. - Update `AcceleratedHNSWUtils` to be able to plug into `Lucene99AcceleratedHNSWVectorsWriter`. `AcceleratedHNSWUtils` is meant to contain common logic for `Lucene99AcceleratedHNSWVectorsWriter`, `LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter`, and `LuceneAcceleratedHNSWScalarQuantizedVectorsWriter`. - Update `QuantizedFieldWriter` to `FieldWriter` as it can be used for all three mentioned above. - Update `Lucene101AcceleratedHNSWCodec` to add some javadocs. - Update `Lucene99AcceleratedHNSWVectorsFormat` to plug in the number of merge workers and an instance of `TaskExecutor` used for the fallback mechanism, and some variable renaming. - Update `Lucene99AcceleratedHNSWVectorsWriter` to remove methods that are now in the `AcceleratedHNSWUtils`, remove some unused/obsolete code, remove some CAGRA merge API related methods that are not currently used and planned to be taken as a separate task, variable renaming, and update documentation. - Update `LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter` to use the now renamed `FieldWriter`. - Update `LuceneAcceleratedHNSWScalarQuantizedVectorsWriter` to use the now renamed `FieldWriter`. - Update `Utils` to move a common method `createListFromMergedVectors`. - Update `TestAcceleratedHNSWParams` to adapt to the feedback suggestion code changes from NVIDIA/cuvs-lucene#120. Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cuvs-lucene/pull/115 --- .../cuvs/lucene/AcceleratedHNSWParams.java | 98 +-- .../cuvs/lucene/AcceleratedHNSWUtils.java | 49 +- .../cuvs/lucene/CuVS2510GPUSearchCodec.java | 21 +- .../cuvs/lucene/CuVS2510GPUVectorsFormat.java | 32 +- .../cuvs/lucene/CuVS2510GPUVectorsReader.java | 264 ++++---- .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 315 +++------ ...tizedFieldWriter.java => FieldWriter.java} | 16 +- .../nvidia/cuvs/lucene/GPUFieldWriter.java | 16 +- .../nvidia/cuvs/lucene/GPUSearchParams.java | 73 ++- .../lucene/Lucene101AcceleratedHNSWCodec.java | 34 +- .../Lucene99AcceleratedHNSWVectorsFormat.java | 22 +- .../Lucene99AcceleratedHNSWVectorsWriter.java | 599 ++---------------- ...ratedHNSWBinaryQuantizedVectorsWriter.java | 13 +- ...ratedHNSWScalarQuantizedVectorsWriter.java | 22 +- .../java/com/nvidia/cuvs/lucene/Utils.java | 37 ++ .../lucene/TestAcceleratedHNSWParams.java | 87 ++- .../cuvs/lucene/TestGPUSearchParams.java | 40 +- 17 files changed, 670 insertions(+), 1068 deletions(-) rename java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/{QuantizedFieldWriter.java => FieldWriter.java} (84%) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java index 9e663d2cb7..ab387a6a14 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java @@ -10,6 +10,7 @@ import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.function.Supplier; public class AcceleratedHNSWParams { @@ -17,33 +18,40 @@ public class AcceleratedHNSWParams { * TODO: Update boundaries for all parameters when a consensus is reached. * Issue: https://github.com/rapidsai/cuvs-lucene/issues/99 */ - private static final int MIN_WRITER_THREADS = 1; - private static final int MAX_WRITER_THREADS = 512; - private static final int MIN_INT_GRAPH_DEG = 2; - private static final int MAX_INT_GRAPH_DEG = 512; - private static final int MIN_GRAPH_DEG = 1; - private static final int MAX_GRAPH_DEG = 512; - private static final int MIN_HNSW_LAYERS = 1; - private static final int MAX_HNSW_LAYERS = 3; - private static final int MIN_MAX_CONN = 1; - private static final int MAX_MAX_CONN = 512; - private static final int MIN_BEAM_WIDTH = 1; - private static final int MAX_BEAM_WIDTH = 512; - private static final int MIN_NUM_MERGE_WORKERS = 1; - private static final int MAX_NUM_MERGE_WORKERS = 512; - - private static final int DEFAULT_WRITER_THREADS = 1; - private static final int DEFAULT_INT_GRAPH_DEGREE = 128; - private static final int DEFAULT_GRAPH_DEGREE = 64; - private static final int DEFAULT_HNSW_LAYERS = 1; - private static final int DEFAULT_MAX_CONN = 32; - private static final int DEFAULT_BEAM_WIDTH = 32; - private static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = + public static final int MIN_WRITER_THREADS = 1; + public static final int MAX_WRITER_THREADS = 512; + public static final int MIN_INT_GRAPH_DEG = 2; + public static final int MAX_INT_GRAPH_DEG = 512; + public static final int MIN_GRAPH_DEG = 1; + public static final int MAX_GRAPH_DEG = 512; + public static final int MIN_HNSW_LAYERS = 1; + public static final int MAX_HNSW_LAYERS = 3; + public static final int MIN_MAX_CONN = 1; + public static final int MAX_MAX_CONN = 512; + public static final int MIN_BEAM_WIDTH = 1; + public static final int MAX_BEAM_WIDTH = 512; + public static final int MIN_NUM_MERGE_WORKERS = 1; + public static final int MAX_NUM_MERGE_WORKERS = 512; + + public static final int DEFAULT_WRITER_THREADS = 1; + public static final int DEFAULT_INT_GRAPH_DEGREE = 128; + public static final int DEFAULT_GRAPH_DEGREE = 64; + public static final int DEFAULT_HNSW_LAYERS = 1; + public static final int DEFAULT_MAX_CONN = 32; + public static final int DEFAULT_BEAM_WIDTH = 32; + public static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = CagraGraphBuildAlgo.NN_DESCENT; - private static final CuVSIvfPqParams DEFAULT_IVF_PQ_PARAMS = - new CuVSIvfPqParams.Builder().build(); - private static final int DEFAULT_NUM_MERGE_WORKERS = 1; - private static final ExecutorService DEFAULT_MERGE_EXE_SRVC = Executors.newFixedThreadPool(1); + public static final int DEFAULT_NUM_MERGE_WORKERS = 1; + + public static final Supplier DEFAULT_IVF_PQ_PARAMS = + () -> { + return new CuVSIvfPqParams.Builder().build(); + }; + + public static final Supplier DEFAULT_MERGE_EXE_SRVC = + () -> { + return Executors.newFixedThreadPool(DEFAULT_NUM_MERGE_WORKERS); + }; private final int writerThreads; private final int intermediateGraphDegree; @@ -57,7 +65,7 @@ public class AcceleratedHNSWParams { private final ExecutorService mergeExec; /** - * Constructs an instance of {@link GPUSearchParams} with specific parameter values. + * Constructs an instance of {@link AcceleratedHNSWParams} with specific parameter values. * * @param writerThreads Number of cuVS writer threads to use. * @param intermediateGraphDegree The intermediate graph degree while building the CAGRA index. @@ -67,7 +75,7 @@ public class AcceleratedHNSWParams { * @param maxConn The max connection parameter used when building HNSW index with the fallback mechanism. * @param beamWidth The beam width parameter used when building HNSW index with the fallback mechanism. * @param cagraGraphBuildAlgo The CAGRA graph build algorithm to use [NN_DESCENT, IVF_PQ]. - * @param cagraGraphBuildAlgo An instance of CuVSIvfPqParams containing IVF_PQ specific parameters. + * @param cuVSIvfPqParams An instance of CuVSIvfPqParams containing IVF_PQ specific parameters. * @param numMergeWorkers The number of merge workers to use with the fallback mechanism. * @param mergeExec The instance of {@link ExecutorService} to use with the fallback mechanism. */ @@ -177,7 +185,7 @@ public int getNumMergeWorkers() { } /** - * Get the instance of the {@link ExecutorService} to be used in the fallback mechanism * + * Get the instance of the {@link ExecutorService} to be used in the fallback mechanism * * @return the instance of the {@link ExecutorService} */ @@ -220,14 +228,14 @@ public static class Builder { private int maxConn = DEFAULT_MAX_CONN; private int beamWidth = DEFAULT_BEAM_WIDTH; private CagraGraphBuildAlgo cagraGraphBuildAlgo = DEFAULT_CAGRA_GRAPH_BUILD_ALGO; - private CuVSIvfPqParams cuVSIvfPqParams = DEFAULT_IVF_PQ_PARAMS; private int numMergeWorkers = DEFAULT_NUM_MERGE_WORKERS; - private ExecutorService mergeExec = DEFAULT_MERGE_EXE_SRVC; + private CuVSIvfPqParams cuVSIvfPqParams = null; + private ExecutorService mergeExec = null; /** * Set the number of cuVS writer threads while building the index * Valid range - Minimum: {@value MIN_WRITER_THREADS}, Maximum: {@value MAX_WRITER_THREADS} - * Default value - 64 + * Default value - {@value DEFAULT_WRITER_THREADS} * * @param writerThreads * @return instance of {@link Builder} @@ -240,7 +248,7 @@ public Builder withWriterThreads(int writerThreads) { /** * Set the intermediate graph degree to use while building CAGRA index * Valid range - Minimum: {@value MIN_INT_GRAPH_DEG}, Maximum: {@value MAX_INT_GRAPH_DEG} - * Default value - 128 + * Default value - {@value DEFAULT_INT_GRAPH_DEGREE} * * @param intermediateGraphDegree * @return instance of {@link Builder} @@ -253,7 +261,7 @@ public Builder withIntermediateGraphDegree(int intermediateGraphDegree) { /** * Set the graph degree to use while building CAGRA index * Valid range - Minimum: {@value MIN_GRAPH_DEG}, Maximum: {@value MAX_GRAPH_DEG} - * Default value - 64 + * Default value - {@value DEFAULT_GRAPH_DEGREE} * * @param graphDegree * @return instance of {@link Builder} @@ -266,7 +274,7 @@ public Builder withGraphDegree(int graphDegree) { /** * Set the number of HNSW layers to construct while building the HNSW index * Valid range - Minimum: {@value MIN_HNSW_LAYERS}, Maximum: {@value MAX_HNSW_LAYERS} - * Default value - 2 + * Default value - {@value DEFAULT_HNSW_LAYERS} * * @param hnswLayers the number of HNSW layers * @return instance of {@link Builder} @@ -279,7 +287,7 @@ public Builder withHNSWLayer(int hnswLayers) { /** * Set the max connections parameter while building HNSW index with fallback mechanism * Valid range - Minimum: {@value MIN_MAX_CONN}, Maximum: {@value MAX_MAX_CONN} - * Default value - 8 + * Default value - {@value DEFAULT_MAX_CONN} * * @param maxConn the max connections parameter * @return instance of {@link Builder} @@ -292,7 +300,7 @@ public Builder withMaxConn(int maxConn) { /** * Set the beam width parameter while building HNSW index with fallback mechanism * Valid range - Minimum: {@value MIN_BEAM_WIDTH}, Maximum: {@value MAX_BEAM_WIDTH} - * Default value - 16 + * Default value - {@value DEFAULT_BEAM_WIDTH} * * @param beamWidth the beam width parameter * @return instance of {@link Builder} @@ -304,7 +312,7 @@ public Builder withBeamWidth(int beamWidth) { /** * Set the CAGRA graph build algorithm to use - * Default NN_DESCENT + * Default value - NN_DESCENT * * @param cagraGraphBuildAlgo * @return instance of {@link Builder} @@ -327,7 +335,7 @@ public Builder withCuVSIvfPqParams(CuVSIvfPqParams cuVSIvfPqParams) { /** * Set the number of merge workers to be used with the fallback mechanism - * Default value - 1 + * Default value - {@value DEFAULT_NUM_MERGE_WORKERS} * * @param numMergeWorkers number of merge workers to set * @return instance of {@link Builder} @@ -407,9 +415,6 @@ private void validate() throws IllegalArgumentException { if (Objects.isNull(cagraGraphBuildAlgo)) { throw new IllegalArgumentException("cagraGraphBuildAlgo cannot be null."); } - if (Objects.isNull(cuVSIvfPqParams)) { - throw new IllegalArgumentException("cuVSIvfPqParams cannot be null."); - } if (numMergeWorkers < MIN_NUM_MERGE_WORKERS || numMergeWorkers > MAX_NUM_MERGE_WORKERS) { throw new IllegalArgumentException( "numMergeWorkers not in valid range. Valid range: [" @@ -418,9 +423,6 @@ private void validate() throws IllegalArgumentException { + MAX_NUM_MERGE_WORKERS + "]"); } - if (Objects.isNull(mergeExec)) { - throw new IllegalArgumentException("mergeExec cannot be null."); - } } /** @@ -429,6 +431,12 @@ private void validate() throws IllegalArgumentException { * @return instance of {@link AcceleratedHNSWParams} */ public AcceleratedHNSWParams build() { + if (Objects.isNull(cuVSIvfPqParams)) { + cuVSIvfPqParams = DEFAULT_IVF_PQ_PARAMS.get(); + } + if (Objects.isNull(mergeExec)) { + mergeExec = DEFAULT_MERGE_EXE_SRVC.get(); + } validate(); return new AcceleratedHNSWParams( writerThreads, diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java index 457ad22d9d..ac4c94266a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java @@ -34,7 +34,8 @@ public class AcceleratedHNSWUtils { public enum QuantizationType { BINARY, - SCALAR + SCALAR, + NONE } private static final LuceneProvider LUCENE_PROVIDER; @@ -84,7 +85,7 @@ public static GPUBuiltHnswGraph createMultiLayerHnswGraph( int size, int dimensions, CuVSMatrix adjacencyListMatrix, - List vectors, + List vectors, int hnswLayers, int graphDegree, CagraIndexParams params, @@ -132,17 +133,32 @@ public static GPUBuiltHnswGraph createMultiLayerHnswGraph( layerNodes.add(selectedNodes); - // Extract vectors for selected nodes - int bytesPerVector = (dimensions + 7) / 8; - byte[][] selectedVectors = new byte[nextLayerSize][]; - for (int i = 0; i < nextLayerSize; i++) { - selectedVectors[i] = vectors.get(selectedNodes[i]); - } + if (quantization == QuantizationType.NONE) { + // Extract vectors for selected nodes + float[][] selectedVectors = new float[nextLayerSize][]; + for (int i = 0; i < nextLayerSize; i++) { + selectedVectors[i] = (float[]) vectors.get(selectedNodes[i]); + } - // Build CAGRA graph for this layer - layerAdjacencies.add( - buildCagraGraphForSubset( - selectedVectors, selectedNodes, bytesPerVector, params, dimensions, quantization)); + // Build CAGRA graph for this layer + layerAdjacencies.add( + buildCagraGraphForSubset( + selectedVectors, selectedNodes, 0, params, dimensions, quantization)); + + } else { + + // Extract vectors for selected nodes + int bytesPerVector = (dimensions + 7) / 8; + byte[][] selectedVectors = new byte[nextLayerSize][]; + for (int i = 0; i < nextLayerSize; i++) { + selectedVectors[i] = (byte[]) vectors.get(selectedNodes[i]); + } + + // Build CAGRA graph for this layer + layerAdjacencies.add( + buildCagraGraphForSubset( + selectedVectors, selectedNodes, bytesPerVector, params, dimensions, quantization)); + } // Update for next iteration currentLayerSize = nextLayerSize; @@ -160,7 +176,7 @@ public static GPUBuiltHnswGraph createMultiLayerHnswGraph( * Builds a CAGRA graph for a subset of binary quantized vectors */ private static CuVSMatrix buildCagraGraphForSubset( - byte[][] vectors, + Object vectors, int[] selectedNodes, int bytesPerVector, CagraIndexParams params, @@ -172,9 +188,12 @@ private static CuVSMatrix buildCagraGraphForSubset( if (quantization == QuantizationType.BINARY) { subsetDataset = - createByteMatrixFromArray(vectors, bytesPerVector, getCuVSResourcesInstance()); + createByteMatrixFromArray((byte[][]) vectors, bytesPerVector, getCuVSResourcesInstance()); + } else if (quantization == QuantizationType.SCALAR) { + subsetDataset = + createByteMatrixFromArray((byte[][]) vectors, dimensions, getCuVSResourcesInstance()); } else { - subsetDataset = createByteMatrixFromArray(vectors, dimensions, getCuVSResourcesInstance()); + subsetDataset = CuVSMatrix.ofArray((float[][]) vectors); } // Build CAGRA index for the subset diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index 321f1c1ca1..2de761a78a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -12,7 +12,7 @@ import org.apache.lucene.codecs.KnnVectorsFormat; /** - * cuVS based codec for GPU based vector search + * cuVS based codec for GPU based vector search that enables both - indexing and search on the GPU. * cuVS serialization formats are in experimental phase and hence backward compatibility cannot be guaranteed. * * @since 25.10 @@ -24,7 +24,7 @@ public class CuVS2510GPUSearchCodec extends FilterCodec { private KnnVectorsFormat format; /** - * Default constructor for {@link CuVS2510GPUSearchCodec} + * Default constructor for {@link CuVS2510GPUSearchCodec}. * * @throws Exception */ @@ -34,7 +34,8 @@ public CuVS2510GPUSearchCodec() throws Exception { } /** - * Initialize {@link CuVS2510GPUSearchCodec} with default parameter values. + * Initialize {@link CuVS2510GPUSearchCodec} with an instance of {@link GPUSearchParams} + * having default parameter values. * * @param name the name of the codec * @param delegate the delegate codec @@ -45,16 +46,22 @@ public CuVS2510GPUSearchCodec(String name, Codec delegate) { } /** - * Initialize the codec with custom parameter values. + * Initialize the codec with an instance of {@link GPUSearchParams} having either default + * or overridden parameter values. * * @param params An instance of {@link GPUSearchParams} - * @throws Exception Exception raised when initializing the codec. + * @throws Exception Exception raised when initializing the codec */ public CuVS2510GPUSearchCodec(GPUSearchParams params) throws Exception { this(NAME, LuceneProvider.getCodec("101")); initializeFormat(params); } + /** + * Initialize the {@link CuVS2510GPUVectorsFormat} instance using {@link GPUSearchParams}. + * + * @param params an instance of {@link GPUSearchParams} + */ private void initializeFormat(GPUSearchParams params) { try { format = new CuVS2510GPUVectorsFormat(params); @@ -67,7 +74,7 @@ private void initializeFormat(GPUSearchParams params) { } /** - * Get the configured {@link KnnVectorsFormat} + * Get the configured {@link KnnVectorsFormat}. * * @return the instance of the {@link KnnVectorsFormat} */ @@ -77,7 +84,7 @@ public KnnVectorsFormat knnVectorsFormat() { } /** - * Set the {@link KnnVectorsFormat} + * Set the {@link KnnVectorsFormat}. * * @param format the {@link KnnVectorsFormat} to set */ diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java index be7e6ab772..ccc61eaee1 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java @@ -8,9 +8,9 @@ import com.nvidia.cuvs.LibraryException; import java.io.IOException; -import java.util.logging.Logger; import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.KnnVectorsReader; +import org.apache.lucene.codecs.KnnVectorsWriter; import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer; import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.index.SegmentReadState; @@ -24,20 +24,18 @@ */ public class CuVS2510GPUVectorsFormat extends KnnVectorsFormat { - @SuppressWarnings("unused") - private static final Logger log = Logger.getLogger(CuVS2510GPUVectorsFormat.class.getName()); - - private static final int maxDimensions = 4096; + private static final int MAX_DIMENSIONS = 4096; private static final LuceneProvider LUCENE_PROVIDER; private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; - private GPUSearchParams gpuSearchParams; - static final String CUVS_META_CODEC_NAME = "Lucene102CuVSVectorsFormatMeta"; - static final String CUVS_META_CODEC_EXT = "vemc"; - static final String CUVS_INDEX_CODEC_NAME = "Lucene102CuVSVectorsFormatIndex"; - static final String CUVS_INDEX_EXT = "vcag"; - static final int VERSION_START = 0; - static final int VERSION_CURRENT = VERSION_START; + public static final String CUVS_META_CODEC_NAME = "Lucene102CuVSVectorsFormatMeta"; + public static final String CUVS_META_CODEC_EXT = "vemc"; + public static final String CUVS_INDEX_CODEC_NAME = "Lucene102CuVSVectorsFormatIndex"; + public static final String CUVS_INDEX_EXT = "vcag"; + public static final int VERSION_START = 0; + public static final int VERSION_CURRENT = VERSION_START; + + private GPUSearchParams gpuSearchParams; static { try { @@ -59,7 +57,7 @@ public CuVS2510GPUVectorsFormat() { } /** - * Initializes the {@link CuVS2510GPUVectorsFormat} with the given threads, graph degree, etc. + * Initializes the {@link CuVS2510GPUVectorsFormat} with an instance of {@link GPUSearchParams}. * * @param gpuSearchParams An instance of {@link GPUSearchParams} * @throws LibraryException if the native library fails to load @@ -70,17 +68,17 @@ public CuVS2510GPUVectorsFormat(GPUSearchParams gpuSearchParams) { } /** - * Returns a {@link CuVS2510GPUVectorsWriter} to write the vectors to the index. + * Returns a KnnVectorsReader instance to write the vectors to the index. */ @Override - public CuVS2510GPUVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { + public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException { assertIsSupported(); var flatWriter = FLAT_VECTORS_FORMAT.fieldsWriter(state); return new CuVS2510GPUVectorsWriter(state, gpuSearchParams, flatWriter); } /** - * Returns a KnnVectorsReader to read the vectors from the index. + * Returns a KnnVectorsReader instance to read the vectors from the index. */ @Override public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException { @@ -93,6 +91,6 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException */ @Override public int getMaxDimensions(String fieldName) { - return maxDimensions; + return MAX_DIMENSIONS; } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java index 7bc29ab4b5..c783660d81 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java @@ -20,10 +20,11 @@ import com.nvidia.cuvs.CagraSearchParams; import com.nvidia.cuvs.CuVSMatrix; import java.io.IOException; +import java.util.BitSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.logging.Logger; +import java.util.Map.Entry; import java.util.stream.Stream; import java.util.stream.StreamSupport; import org.apache.lucene.codecs.CodecUtil; @@ -43,10 +44,10 @@ import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IOContext.Context; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.ReadAdvice; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.hnsw.IntToIntFunction; @@ -57,15 +58,24 @@ */ public class CuVS2510GPUVectorsReader extends KnnVectorsReader { - @SuppressWarnings("unused") - private static final Logger log = Logger.getLogger(CuVS2510GPUVectorsReader.class.getName()); + private static final LuceneProvider LUCENE_PROVIDER; + private static final List VECTOR_SIMILARITY_FUNCTIONS; - private final FlatVectorsReader flatVectorsReader; // for reading the raw vectors + private final FlatVectorsReader flatVectorsReader; private final FieldInfos fieldInfos; private final IntObjectHashMap fields; private final IntObjectHashMap cuvsIndices; private final IndexInput cuvsIndexInput; + static { + try { + LUCENE_PROVIDER = LuceneProvider.getInstance("99"); + VECTOR_SIMILARITY_FUNCTIONS = LUCENE_PROVIDER.getSimilarityFunctions(); + } catch (Exception e) { + throw new ExceptionInInitializerError(e.getMessage()); + } + } + /** * Initializes the {@link CuVS2510GPUVectorsReader}, checks and loads the index. * @@ -79,7 +89,6 @@ public CuVS2510GPUVectorsReader(SegmentReadState state, FlatVectorsReader flatRe this.flatVectorsReader = flatReader; this.fieldInfos = state.fieldInfos; this.fields = new IntObjectHashMap<>(); - String metaFileName = IndexFileNames.segmentFileName( state.segmentInfo.name, state.segmentSuffix, CUVS_META_CODEC_EXT); @@ -104,7 +113,16 @@ public CuVS2510GPUVectorsReader(SegmentReadState state, FlatVectorsReader flatRe } var ioContext = state.context.withReadAdvice(ReadAdvice.SEQUENTIAL); cuvsIndexInput = openCuVSInput(state, versionMeta, ioContext); - cuvsIndices = loadCuVSIndices(); + /* + * Only load indexes on the GPU when this reader is opening for searches. + * Do not load indexes on the GPU when this reader is opening during merge calls. + * With this approach we reduce device memory usage by approximately 50% during merges. + */ + if (state.context.context().equals(Context.MERGE)) { + cuvsIndices = null; + } else { + cuvsIndices = loadCuVSIndices(); + } success = true; } finally { if (success == false) { @@ -185,17 +203,6 @@ private void readFields(ChecksumIndexInput meta) throws IOException { } } - // List of vector similarity functions. This list is defined here, in order - // to avoid an undesirable dependency on the declaration and order of values - // in VectorSimilarityFunction. The list values and order must be identical - // to that of {@link o.a.l.c.l.Lucene94FieldInfosFormat#SIMILARITY_FUNCTIONS}. - static final List SIMILARITY_FUNCTIONS = - List.of( - VectorSimilarityFunction.EUCLIDEAN, - VectorSimilarityFunction.DOT_PRODUCT, - VectorSimilarityFunction.COSINE, - VectorSimilarityFunction.MAXIMUM_INNER_PRODUCT); - /** * Checks the distance function validity and returns it. * @@ -205,14 +212,14 @@ private void readFields(ChecksumIndexInput meta) throws IOException { */ static VectorSimilarityFunction readSimilarityFunction(DataInput input) throws IOException { int i = input.readInt(); - if (i < 0 || i >= SIMILARITY_FUNCTIONS.size()) { + if (i < 0 || i >= VECTOR_SIMILARITY_FUNCTIONS.size()) { throw new IllegalArgumentException("invalid distance function: " + i); } - return SIMILARITY_FUNCTIONS.get(i); + return VECTOR_SIMILARITY_FUNCTIONS.get(i); } /** - * Reads the vector encoding (The numeric datatype of the vector values) from the DataInput. + * Reads the vector encoding (The numeric data type of the vector values) from the DataInput. * * @param input instance of DataInput * @return the vector encoding @@ -282,9 +289,9 @@ private FieldEntry getFieldEntry(String field, VectorEncoding expectedEncoding) */ private IntObjectHashMap loadCuVSIndices() throws IOException { var indices = new IntObjectHashMap(); - for (var e : fields) { - var fieldEntry = e.value; - int fieldNumber = e.key; + for (var field : fields) { + var fieldEntry = field.value; + int fieldNumber = field.key; var cuvsIndex = loadCuVSIndex(fieldEntry); indices.put(fieldNumber, cuvsIndex); } @@ -292,7 +299,7 @@ private IntObjectHashMap loadCuVSIndices() throws IOException { } /** - * Loads the CAGRA and bruteforce index (if exists) onto the GPU. + * Loads the CAGRA and brute force index (if exists) onto the GPU. * * @param fieldEntry instance of {@link FieldEntry} * @return return the instance of {@link GPUIndex} @@ -301,7 +308,6 @@ private IntObjectHashMap loadCuVSIndices() throws IOException { private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { CagraIndex cagraIndex = null; BruteForceIndex bruteForceIndex = null; - try { long len = fieldEntry.cagraIndexLength(); if (len > 0) { @@ -311,7 +317,6 @@ private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { cagraIndex = CagraIndex.newBuilder(getCuVSResourcesInstance()).from(in).build(); } } - len = fieldEntry.bruteForceIndexLength(); if (len > 0) { long off = fieldEntry.bruteForceIndexOffset(); @@ -331,11 +336,12 @@ private GPUIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException { */ @Override public void close() throws IOException { - var closeableStream = - Stream.concat( - Stream.of(flatVectorsReader, cuvsIndexInput), - stream(cuvsIndices.values().iterator()).map(cursor -> cursor.value)); + var closeableStream = Stream.of(flatVectorsReader, cuvsIndexInput); IOUtils.close(closeableStream::iterator); + if (cuvsIndices != null) { + var indexClosableStream = stream(cuvsIndices.values().iterator()).map(cursor -> cursor.value); + IOUtils.close(indexClosableStream::iterator); + } closeCuVSResourcesInstance(); } @@ -348,7 +354,8 @@ static Stream stream(Iterator iterator) { */ @Override public void checkIntegrity() throws IOException { - // TODO: Pending implementation + flatVectorsReader.checkIntegrity(); + CodecUtil.checksumEntireFile(cuvsIndexInput); } /** @@ -360,47 +367,33 @@ public FloatVectorValues getFloatVectorValues(String field) throws IOException { } /** - * Returns the FloatVectorValues for the given field. + * Returns the ByteVectorValues for the given field. * * This is not supported. */ @Override public ByteVectorValues getByteVectorValues(String field) { - throw new UnsupportedOperationException("byte vectors not supported"); + throw new UnsupportedOperationException("byte vectors are not currently supported"); } /** Native float to float function */ - public interface FloatToFloatFunction { + private interface FloatToFloatFunction { float apply(float v); } - /** - * Returns a long array from bits. - */ - static long[] bitsToLongArray(Bits bits) { - if (bits instanceof FixedBitSet fixedBitSet) { - return fixedBitSet.getBits(); - } else { - return FixedBitSet.copyOf(bits).getBits(); - } - } - /** * Get the score normalization function. * * @param sim instance of VectorSimilarityFunction * @return an instance of the FloatToFloatFunction */ - static FloatToFloatFunction getScoreNormalizationFunc(VectorSimilarityFunction sim) { + private static FloatToFloatFunction getScoreNormalizationFunc(VectorSimilarityFunction sim) { // TODO: check for different similarities return score -> (1f / (1f + score)); } - // This is a hack - https://github.com/rapidsai/cuvs/issues/696 - static final int FILTER_OVER_SAMPLE = 10; - /** - * Returns the k nearest neighbor documents using cuVS's CAGRA or Bruteforce algorithm for this field, to the given vector. + * Returns the k nearest neighbor documents using cuVS's CAGRA or brute force algorithm for this field, to the given vector. */ @Override public void search(String field, float[] target, KnnCollector knnCollector, Bits acceptDocs) @@ -411,90 +404,121 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits } var fieldNumber = fieldInfos.fieldInfo(field).number; - - GPUIndex cuvsIndex = cuvsIndices.get(fieldNumber); + GPUIndex cuvsIndex = cuvsIndices != null ? cuvsIndices.get(fieldNumber) : null; if (cuvsIndex == null) { - throw new IllegalStateException("not index found for field:" + field); + throw new IllegalStateException("Index not found for field:" + field); } - int collectorTopK = knnCollector.k(); + final FloatVectorValues rawValues = flatVectorsReader.getFloatVectorValues(field); + final Bits acceptedOrds = rawValues.getAcceptOrds(acceptDocs); + BitSet[] mask = null; + int maskLength = 0; + int topK = knnCollector.k(); + if (acceptDocs != null) { - collectorTopK = knnCollector.k() * FILTER_OVER_SAMPLE; - } - final int topK = Math.min(collectorTopK, fieldEntry.count()); - assert topK > 0 : "Expected topK > 0, got:" + topK; - - Map result; - if (knnCollector.k() <= 1024 && cuvsIndex.getCagraIndex() != null) { - - CagraSearchParams searchParams; - if (knnCollector instanceof GPUPerLeafCuVSKnnCollector) { - GPUPerLeafCuVSKnnCollector collector = (GPUPerLeafCuVSKnnCollector) knnCollector; - searchParams = - new CagraSearchParams.Builder() - .withItopkSize(Math.max(collector.getiTopK(), topK)) - .withSearchWidth(collector.getSearchWidth()) - .build(); - } else { - // Setting itopK as topK because in any case iTopK should be ATLEAST equal to topK - searchParams = new CagraSearchParams.Builder().withItopkSize(topK).build(); + mask = new BitSet[1]; // As there is only one query "target" + mask[0] = new BitSet(acceptedOrds.length()); + /* + * Need to find if there is a better alternative for below loop + * in subsequent code improvement iterations. This is needed as + * there is a difference between Lucene's Bits "acceptDocs" and + * what our API accepts. + */ + for (int i = 0; i < acceptedOrds.length(); i++) { + if (acceptedOrds.get(i)) { + mask[0].set(i); + } } + topK = Math.min(knnCollector.k() + 10, mask[0].cardinality()); + maskLength = mask[0].length(); + } - var query = - new CagraQuery.Builder(getCuVSResourcesInstance()) - .withTopK(topK) - .withSearchParams(searchParams) - .withQueryVectors(CuVSMatrix.ofArray(new float[][] {target})) - .build(); - - CagraIndex cagraIndex = cuvsIndex.getCagraIndex(); + try { List> searchResult = null; - try { + if (knnCollector.k() <= 1024 && cuvsIndex.getCagraIndex() != null) { + CagraSearchParams searchParams; + if (knnCollector instanceof GPUPerLeafCuVSKnnCollector) { + GPUPerLeafCuVSKnnCollector collector = (GPUPerLeafCuVSKnnCollector) knnCollector; + searchParams = + new CagraSearchParams.Builder() + .withItopkSize(Math.max(collector.getiTopK(), topK)) + .withSearchWidth(collector.getSearchWidth()) + .build(); + } else { + // Setting itopK as topK because in any case iTopK should be ATLEAST equal to topK + searchParams = new CagraSearchParams.Builder().withItopkSize(topK).build(); + } + CagraIndex cagraIndex = cuvsIndex.getCagraIndex(); + assert cagraIndex != null; + CagraQuery query = null; + + CuVSMatrix.Builder builder = + CuVSMatrix.deviceBuilder( + getCuVSResourcesInstance(), 1, target.length, CuVSMatrix.DataType.FLOAT); + builder.addVector(target); + CuVSMatrix queryVector = builder.build(); + + if (acceptDocs != null) { + query = + new CagraQuery.Builder(getCuVSResourcesInstance()) + .withTopK(topK) + .withSearchParams(searchParams) + .withQueryVectors(queryVector) + .withPrefilter(mask[0], maskLength) + .build(); + } else { + query = + new CagraQuery.Builder(getCuVSResourcesInstance()) + .withTopK(topK) + .withSearchParams(searchParams) + .withQueryVectors(queryVector) + .build(); + } searchResult = cagraIndex.search(query).getResults(); - } catch (Throwable t) { - Utils.handleThrowable(t); - } - // List expected to have only one entry because of single query "target". - assert searchResult.size() == 1; - result = searchResult.getFirst(); - } else { - BruteForceIndex bruteforceIndex = cuvsIndex.getBruteforceIndex(); - assert bruteforceIndex != null; - var queryBuilder = - new BruteForceQuery.Builder(getCuVSResourcesInstance()) - .withQueryVectors(new float[][] {target}) - .withTopK(topK); - BruteForceQuery query = queryBuilder.build(); - - List> searchResult = null; - try { + } else { + BruteForceIndex bruteforceIndex = cuvsIndex.getBruteforceIndex(); + assert bruteforceIndex != null; + BruteForceQuery query = null; + float[][] queryVector = new float[][] {target}; + if (acceptDocs != null) { + query = + new BruteForceQuery.Builder(getCuVSResourcesInstance()) + .withQueryVectors(queryVector) + .withPrefilters(mask, maskLength) + .withTopK(topK) + .build(); + } else { + query = + new BruteForceQuery.Builder(getCuVSResourcesInstance()) + .withQueryVectors(queryVector) + .withTopK(topK) + .build(); + } searchResult = bruteforceIndex.search(query).getResults(); - } catch (Throwable t) { - Utils.handleThrowable(t); } - assert searchResult.size() == 1; - result = searchResult.getFirst(); - } - assert result != null; - final var rawValues = flatVectorsReader.getFloatVectorValues(field); - final Bits acceptedOrds = rawValues.getAcceptOrds(acceptDocs); - final var ordToDocFunction = (IntToIntFunction) rawValues::ordToDoc; - final var scoreCorrectionFunction = getScoreNormalizationFunc(fieldEntry.similarityFunction); + // List expected to have only one entry because of single query "target". + assert searchResult.size() == 1; + final IntToIntFunction ordToDocFunction = (IntToIntFunction) rawValues::ordToDoc; + final FloatToFloatFunction scoreCorrectionFunction = + getScoreNormalizationFunc(fieldEntry.similarityFunction); - for (var entry : result.entrySet()) { - int ord = entry.getKey(); - float score = entry.getValue(); - if (acceptedOrds == null || acceptedOrds.get(ord)) { + for (Entry entry : searchResult.getFirst().entrySet()) { + int ord = entry.getKey(); + float score = entry.getValue(); if (knnCollector.earlyTerminated()) { break; } - assert ord >= 0 : "unexpected ord: " + ord; - int doc = ordToDocFunction.apply(ord); + if (ord < 0) { + continue; + } float correctedScore = scoreCorrectionFunction.apply(score); + int doc = ordToDocFunction.apply(ord); knnCollector.incVisitedCount(1); knnCollector.collect(doc, correctedScore); } + } catch (Throwable t) { + Utils.handleThrowable(t); } } @@ -506,7 +530,7 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits @Override public void search(String field, byte[] target, KnnCollector knnCollector, Bits acceptDocs) throws IOException { - throw new UnsupportedOperationException("byte vectors not supported"); + throw new UnsupportedOperationException("Byte vectors are not currently supported"); } /** @@ -526,7 +550,7 @@ record FieldEntry( * Returns an instance of FieldEntry. * * @param input instance of IndexInput - * @param vectorEncoding The numeric datatype of the vector values + * @param vectorEncoding The numeric data type of the vector values * @param similarityFunction Vector similarity function; used in search to return top K most similar vectors to a target vector * @return an instance of FieldEntry * @throws IOException I/O Exceptions @@ -562,7 +586,7 @@ static FieldEntry readEntry( * @param in * @throws CorruptIndexException */ - static void checkVersion(int versionMeta, int versionVectorData, IndexInput in) + private static void checkVersion(int versionMeta, int versionVectorData, IndexInput in) throws CorruptIndexException { if (versionMeta != versionVectorData) { throw new CorruptIndexException( @@ -588,7 +612,7 @@ public FieldInfos getFieldInfos() { /** * Gets the map of {@link GPUIndex} objects. * - * @return the map of gpu index objects + * @return the map of GPU index objects */ public IntObjectHashMap getCuvsIndexes() { return cuvsIndices; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index cba32e21de..2bb6732cb9 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -11,6 +11,7 @@ import static com.nvidia.cuvs.lucene.CuVS2510GPUVectorsFormat.VERSION_CURRENT; import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.closeCuVSResourcesInstance; import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.getCuVSResourcesInstance; +import static com.nvidia.cuvs.lucene.Utils.info; import static org.apache.lucene.index.VectorEncoding.FLOAT32; import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; @@ -27,8 +28,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.logging.Logger; -import java.util.stream.IntStream; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.KnnFieldVectorsWriter; import org.apache.lucene.codecs.KnnVectorsReader; @@ -60,26 +59,16 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { private static final long SHALLOW_RAM_BYTES_USED = shallowSizeOfInstance(CuVS2510GPUVectorsWriter.class); - - @SuppressWarnings("unused") - private static final Logger log = Logger.getLogger(CuVS2510GPUVectorsWriter.class.getName()); - - /** The name of the CUVS component for the info-stream * */ - private static final String CUVS_COMPONENT = "CUVS"; - + private static final String COMPONENT = "CuVS2510GPUVectorsWriter"; private static final LuceneProvider LUCENE_PROVIDER; private static final List VECTOR_SIMILARITY_FUNCTIONS; - - // The minimum number of vectors in the dataset required before - // we attempt to build a Cagra index - static final int MIN_CAGRA_INDEX_SIZE = 2; + private static final int MIN_CAGRA_INDEX_SIZE = 2; private final GPUSearchParams gpuSearchParams; - private final FlatVectorsWriter flatVectorsWriter; private final List fields = new ArrayList<>(); - private IndexOutput meta = null, cuvsIndex = null; private final InfoStream infoStream; + private IndexOutput meta = null, cuvsIndex = null; private boolean finished; static { @@ -96,13 +85,13 @@ public class CuVS2510GPUVectorsWriter extends KnnVectorsWriter { */ public enum IndexType { - /** Builds a Cagra index. */ + /** Builds a CAGRA index. */ CAGRA(true, false), /** Builds a Brute Force index. */ BRUTE_FORCE(false, true), - /** Builds a Cagra and a Brute Force index. */ + /** Builds both - CAGRA and Brute Force indexes. */ CAGRA_AND_BRUTE_FORCE(true, true); private final boolean cagra, bruteForce; @@ -111,21 +100,11 @@ public enum IndexType { this.bruteForce = bruteForce; } - /** - * Check if cagra is set - * - * @return is cagra set - */ - public boolean cagra() { + public boolean isCagra() { return cagra; } - /** - * Check if bruteforce is set - * - * @return is bruteforce set - */ - public boolean bruteForce() { + public boolean isBruteForce() { return bruteForce; } } @@ -146,16 +125,13 @@ public CuVS2510GPUVectorsWriter( this.gpuSearchParams = gpuSearchParams; this.flatVectorsWriter = flatVectorsWriter; this.infoStream = state.infoStream; - String metaFileName = IndexFileNames.segmentFileName( state.segmentInfo.name, state.segmentSuffix, CUVS_META_CODEC_EXT); String cagraFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, CUVS_INDEX_EXT); - boolean success = false; try { - meta = state.directory.createOutput(metaFileName, state.context); cuvsIndex = state.directory.createOutput(cagraFileName, state.context); CodecUtil.writeIndexHeader( @@ -170,7 +146,6 @@ public CuVS2510GPUVectorsWriter( VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); - success = true; } finally { if (success == false) { @@ -186,7 +161,7 @@ public CuVS2510GPUVectorsWriter( public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { var encoding = fieldInfo.getVectorEncoding(); if (encoding != FLOAT32) { - throw new IllegalArgumentException("expected float32, got:" + encoding); + throw new IllegalArgumentException("Expected float32, got:" + encoding); } var writer = Objects.requireNonNull(flatVectorsWriter.addField(fieldInfo)); @SuppressWarnings("unchecked") @@ -197,102 +172,55 @@ public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException } /** - * Returns a string containing meta information like graph degree etc. - * - * @param size index size - * @param args other parameters like graph degree, Intermediate graph degree, etc. - * @return the string containing the meta information - */ - static String indexMsg(int size, int... args) { - StringBuilder sb = new StringBuilder("cagra index params"); - sb.append(": size=").append(size); - sb.append(", intGraphDegree=").append(args[0]); - sb.append(", actualIntGraphDegree=").append(args[1]); - sb.append(", graphDegree=").append(args[2]); - sb.append(", actualGraphDegree=").append(args[3]); - return sb.toString(); - } - - /** - * Builds and returns an instance of CagraIndexParams. - * - * @param size the size of the index - * @return an instance of CagraIndexParams - */ - private CagraIndexParams cagraIndexParams(int size) { - if (size < 2) { - // https://github.com/rapidsai/cuvs/issues/666 - throw new IllegalArgumentException("cagra index must be greater than 2"); - } - - return new CagraIndexParams.Builder() - .withNumWriterThreads(gpuSearchParams.getWriterThreads()) - .withIntermediateGraphDegree(gpuSearchParams.getIntermediateGraphDegree()) - .withGraphDegree(gpuSearchParams.getGraphdegree()) - .withCagraGraphBuildAlgo(gpuSearchParams.getCagraGraphBuildAlgo()) - .build(); - } - - /** - * Utility to print info/debug messages via InfoStream. - * - * @param msg - */ - private void info(String msg) { - if (infoStream.isEnabled(CUVS_COMPONENT)) { - infoStream.message(CUVS_COMPONENT, msg); - } - } - - /** - * Creates CAGRA and/or Bruteforce indexes and writes them. + * Creates CAGRA and/or brute force indexes and writes them. * * @param fieldInfo Instance of the FieldInFo to use * @param vectors list of float vectors to index * @throws IOException */ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOException { - if (vectors.size() == 0) { + if (vectors != null && vectors.size() == 0) { writeEmpty(fieldInfo); return; } long cagraIndexOffset, cagraIndexLength = 0L; long bruteForceIndexOffset, bruteForceIndexLength = 0L; - // workaround for the minimum number of vectors for Cagra + /* + * CAGRA has an issue when asked to build an index with just one vector. + * Hence, we currently fallback to brute force in such a case. + */ IndexType indexType = - gpuSearchParams.getIndexType().cagra() && vectors.size() < MIN_CAGRA_INDEX_SIZE + gpuSearchParams.getIndexType().isCagra() && vectors.size() < MIN_CAGRA_INDEX_SIZE ? IndexType.BRUTE_FORCE : gpuSearchParams.getIndexType(); try { - cagraIndexOffset = cuvsIndex.getFilePointer(); - if (indexType.cagra()) { + if (indexType.isCagra()) { + var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); try { - var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - CuVSMatrix dataset = + CuVSMatrix cagraDataset = Utils.createFloatMatrix( vectors, fieldInfo.getVectorDimension(), getCuVSResourcesInstance()); - writeCagraIndex(cagraIndexOutputStream, dataset); + writeCagraIndex(cagraIndexOutputStream, cagraDataset); } catch (Throwable t) { - Utils.handleThrowableWithIgnore(t, CANNOT_GENERATE_CAGRA); - // workaround for cuVS issue + // Fallback to brute force in a few cases, for now. + Utils.handleThrowableWithIgnore(t, t.getMessage()); indexType = IndexType.BRUTE_FORCE; } cagraIndexLength = cuvsIndex.getFilePointer() - cagraIndexOffset; } - bruteForceIndexOffset = cuvsIndex.getFilePointer(); - if (indexType.bruteForce()) { + if (indexType.isBruteForce()) { var bruteForceIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - CuVSMatrix dataset = + CuVSMatrix bruteforceDataset = Utils.createFloatMatrix( vectors, fieldInfo.getVectorDimension(), getCuVSResourcesInstance()); - writeBruteForceIndex(bruteForceIndexOutputStream, dataset); + + writeBruteForceIndex(bruteForceIndexOutputStream, bruteforceDataset); bruteForceIndexLength = cuvsIndex.getFilePointer() - bruteForceIndexOffset; } - writeMeta( fieldInfo, vectors.size(), @@ -313,50 +241,46 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro * @throws Throwable */ private void writeCagraIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { - if (dataset.size() < 2) { - throw new IllegalArgumentException(dataset.size() + " vectors, less than min [2] required"); - } - CagraIndexParams params = cagraIndexParams((int) dataset.size()); - long startTime = System.nanoTime(); + CagraIndexParams params = + new CagraIndexParams.Builder() + .withNumWriterThreads(gpuSearchParams.getWriterThreads()) + .withIntermediateGraphDegree(gpuSearchParams.getIntermediateGraphDegree()) + .withGraphDegree(gpuSearchParams.getGraphdegree()) + .withCagraGraphBuildAlgo(gpuSearchParams.getCagraGraphBuildAlgo()) + .withCuVSIvfPqParams(gpuSearchParams.getCuVSIvfPqParams()) + .build(); CagraIndex index = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(dataset) .withIndexParams(params) .build(); - long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - startTime); - info("Cagra index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); - Path tmpFile = - Files.createTempFile(getCuVSResourcesInstance().tempDirectory(), "tmpindex", "cag"); - index.serialize(os, tmpFile); + index.serialize(os); index.close(); } /** - * Builds and writes the Bruteforce index. + * Builds and writes the brute force index. * - * @param os Instance of OutputStream to write the index to - * @param dataset Instance of CuVSMatrix that holds the dataset + * @param os Instance of OutputStream to write the index bytes to + * @param dataset Instance of CuVSMatrix that holds the data set * @throws Throwable */ private void writeBruteForceIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { BruteForceIndexParams params = new BruteForceIndexParams.Builder() - .withNumWriterThreads(32) // TODO: Make this configurable. + .withNumWriterThreads(gpuSearchParams.getWriterThreads()) .build(); - long startTime = System.nanoTime(); var index = BruteForceIndex.newBuilder(getCuVSResourcesInstance()) .withIndexParams(params) .withDataset(dataset) .build(); - long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - startTime); - info("bf index created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); index.serialize(os); index.close(); } /** - * Creates the CAGRA and/or Bruteforce indexes and writes them to the disk. + * Creates the CAGRA and/or brute force indexes and writes them to the disk. */ @Override public void flush(int maxDoc, DocMap sortMap) throws IOException { @@ -389,16 +313,13 @@ private void writeField(GPUFieldWriter fieldData) throws IOException { */ private void writeSortingField(GPUFieldWriter fieldData, Sorter.DocMap sortMap) throws IOException { - DocsWithFieldSet oldDocsWithFieldSet = fieldData.getDocsWithFieldSet(); - final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; // new ord to old ord + final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; mapOldOrdToNewOrd(oldDocsWithFieldSet, sortMap, null, new2OldOrd, null); - List sortedVectors = new ArrayList(); for (int i = 0; i < fieldData.getVectors().size(); i++) { sortedVectors.add(fieldData.getVectors().get(new2OldOrd[i])); } - writeFieldInternal(fieldData.fieldInfo(), sortedVectors); } @@ -419,8 +340,8 @@ private void writeEmpty(FieldInfo fieldInfo) throws IOException { * @param count number of vectors * @param cagraIndexOffset CAGRA index offset * @param cagraIndexLength CAGRA index length - * @param bruteForceIndexOffset Bruteforce index offset - * @param bruteForceIndexLength Bruteforce index length + * @param bruteForceIndexOffset Brute force index offset + * @param bruteForceIndexLength Brute force index length * @throws IOException I/O Exceptions */ private void writeMeta( @@ -448,36 +369,27 @@ static int distFuncToOrd(VectorSimilarityFunction func) { return (byte) i; } } - throw new IllegalArgumentException("invalid distance function: " + func); + throw new IllegalArgumentException("Invalid distance function: " + func); } - // We currently ignore this, until cuVS supports tiered indices - private static final String CANNOT_GENERATE_CAGRA = - """ - Could not generate an intermediate CAGRA graph because the initial \ - kNN graph contains too many invalid or duplicated neighbor nodes. \ - This error can occur, for example, if too many overflows occur \ - during the norm computation between the dataset vectors\ - """; - /** * Uses the cuVS API to merge CAGRA indexes. * + * This is currently (and intentionally) marked as unused and will be plugged in later. + * * @param fieldInfo instance of the FieldInfo * @param mergeState instance of the MergeState * @throws IOException I/O Exceptions */ + @SuppressWarnings("unused") private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throws IOException { try { - List cagraIndexes = new ArrayList<>(); // We need this count so that the merged segment's meta information has the vector count. int totalVectorCount = 0; - for (int i = 0; i < mergeState.knnVectorsReaders.length; i++) { KnnVectorsReader knnReader = mergeState.knnVectorsReaders[i]; // Access the CAGRA index for this field from the reader - if (knnReader != null) { if (knnReader instanceof CuVS2510GPUVectorsReader cvr) { if (cvr != null) { @@ -495,46 +407,13 @@ private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throw } } assert cagraIndexes.size() > 1; - CagraIndex mergedIndex = CagraIndex.merge(cagraIndexes.toArray(new CagraIndex[cagraIndexes.size()])); writeMergedCagraIndex(fieldInfo, mergedIndex, totalVectorCount); - info("Successfully merged " + cagraIndexes.size() + " CAGRA indexes using native merge API"); - - } catch (Throwable t) { - Utils.handleThrowable(t); - } - } - - /** - * Creates List from merged vectors. - */ - private List createListFromMergedVectors(FloatVectorValues mergedVectorValues) - throws IOException { - List res = new ArrayList(); - KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); - for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { - int ordinal = iter.index(); - float[] vector = mergedVectorValues.vectorValue(ordinal); - res.add(vector.clone()); - } - return res; - } - - /** - * Fallback method that rebuilds indexes from merged vectors. - * Used when native CAGRA merge() is not possible. Also used - * when non-CAGRA index types are used (for e.g. Brute Force index). - */ - private void vectorBasedMerge(FieldInfo fieldInfo, MergeState mergeState) throws IOException { - if (fieldInfo.getVectorEncoding() != FLOAT32) { - throw new AssertionError("Only Float32 supported"); - } - try { - List dataset = - createListFromMergedVectors( - KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState)); - writeFieldInternal(fieldInfo, dataset); + info( + infoStream, + COMPONENT, + "Successfully merged " + cagraIndexes.size() + " CAGRA indexes using native merge API"); } catch (Throwable t) { Utils.handleThrowable(t); } @@ -547,9 +426,7 @@ private CagraIndex getCagraIndexFromReader(CuVS2510GPUVectorsReader reader, Stri try { IntObjectHashMap cuvsIndices = reader.getCuvsIndexes(); FieldInfos fieldInfos = reader.getFieldInfos(); - FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldName); - if (fieldInfo != null) { GPUIndex cuvsIndex = cuvsIndices.get(fieldInfo.number); if (cuvsIndex != null) { @@ -557,8 +434,11 @@ private CagraIndex getCagraIndexFromReader(CuVS2510GPUVectorsReader reader, Stri } } } catch (Exception e) { - e.printStackTrace(); - info("Failed to extract CAGRA index for field " + fieldName + ": " + e.getMessage()); + info( + infoStream, + COMPONENT, + "Failed to extract CAGRA index for field " + fieldName + ": " + e.getMessage()); + throw e; } return null; } @@ -571,53 +451,67 @@ private void writeMergedCagraIndex(FieldInfo fieldInfo, CagraIndex mergedIndex, try { long cagraIndexOffset = cuvsIndex.getFilePointer(); var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - - // Serialize the merged index Path tmpFile = Files.createTempFile(getCuVSResourcesInstance().tempDirectory(), "mergedindex", "cag"); mergedIndex.serialize(cagraIndexOutputStream, tmpFile); long cagraIndexLength = cuvsIndex.getFilePointer() - cagraIndexOffset; - writeMeta(fieldInfo, vectorCount, cagraIndexOffset, cagraIndexLength, 0L, 0L); - - // Clean up the merged index mergedIndex.close(); } catch (Throwable t) { Utils.handleThrowable(t); } } + /** + * Creates List from merged vectors. + */ + private List createListFromMergedVectors(FloatVectorValues mergedVectorValues) + throws IOException { + List res = new ArrayList(); + KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); + for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { + int ordinal = iter.index(); + float[] vector = mergedVectorValues.vectorValue(ordinal); + res.add(vector.clone()); + } + return res; + } + + /** + * Fallback method that rebuilds indexes from merged vectors. + * Used when native CAGRA merge() is not possible. Also used + * when non-CAGRA index types are used (for e.g. Brute Force index). + */ + private void vectorBasedMerge(FieldInfo fieldInfo, MergeState mergeState) throws IOException { + try { + List dataset = + createListFromMergedVectors( + KnnVectorsWriter.MergedVectorValues.mergeFloatVectorValues(fieldInfo, mergeState)); + writeFieldInternal(fieldInfo, dataset); + } catch (Throwable t) { + Utils.handleThrowable(t); + } + } + /** * Write field for merging. */ @Override public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { flatVectorsWriter.mergeOneField(fieldInfo, mergeState); + vectorBasedMerge(fieldInfo, mergeState); + } - if (gpuSearchParams.getIndexType().cagra() && !gpuSearchParams.getIndexType().bruteForce()) { - // Since CAGRA merge does not support merging of indexes with purging of deletes, - // we fallback to vector-based re-indexing. Issue: - // https://github.com/rapidsai/cuvs/issues/1253 - boolean hasDeletions = - IntStream.range(0, mergeState.liveDocs.length) - .anyMatch( - i -> - mergeState.liveDocs[i] == null - || IntStream.range(0, mergeState.maxDocs[i]) - .anyMatch(j -> !mergeState.liveDocs[i].get(j))); - - if (mergeState.knnVectorsReaders.length > 1 && !hasDeletions) { - mergeCagraIndexes(fieldInfo, mergeState); - } else { - // CAGRA's merge API does not handle the trivial case of merging 1 index. - vectorBasedMerge(fieldInfo, mergeState); - } - - } else { - // If there is a Brute Force index then re-index using the vectors even if there is a CAGRA - // index. - vectorBasedMerge(fieldInfo, mergeState); + /** + * Returns the memory usage of this object in bytes. + */ + @Override + public long ramBytesUsed() { + long total = SHALLOW_RAM_BYTES_USED; + for (var field : fields) { + total += field.ramBytesUsed(); } + return total; } /** @@ -630,7 +524,6 @@ public void finish() throws IOException { } finished = true; flatVectorsWriter.finish(); - if (meta != null) { // write end of fields marker meta.writeInt(-1); @@ -649,16 +542,4 @@ public void close() throws IOException { IOUtils.close(meta, cuvsIndex, flatVectorsWriter); closeCuVSResourcesInstance(); } - - /** - * Returns the memory usage of this object in bytes. - */ - @Override - public long ramBytesUsed() { - long total = SHALLOW_RAM_BYTES_USED; - for (var field : fields) { - total += field.ramBytesUsed(); - } - return total; - } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/QuantizedFieldWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FieldWriter.java similarity index 84% rename from java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/QuantizedFieldWriter.java rename to java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FieldWriter.java index 8a6bbfed27..627d931daa 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/QuantizedFieldWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FieldWriter.java @@ -17,10 +17,10 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.util.RamUsageEstimator; -public class QuantizedFieldWriter extends KnnFieldVectorsWriter { +public class FieldWriter extends KnnFieldVectorsWriter { private static final long SHALLOW_SIZE = - RamUsageEstimator.shallowSizeOfInstance(QuantizedFieldWriter.class); + RamUsageEstimator.shallowSizeOfInstance(FieldWriter.class); private final FieldInfo fieldInfo; private final FlatFieldVectorsWriter flatFieldVectorsWriter; @@ -28,7 +28,7 @@ public class QuantizedFieldWriter extends KnnFieldVectorsWriter { private QuantizationType quantizationType; @SuppressWarnings("unchecked") - public QuantizedFieldWriter( + public FieldWriter( QuantizationType quantizationType, FieldInfo fieldInfo, FlatFieldVectorsWriter flatFieldVectorsWriter) { @@ -49,14 +49,20 @@ public void addValue(int docID, Object vectorValue) throws IOException { flatFieldVectorsWriter.addValue(docID, (float[]) vectorValue); } - List getVectors() { + List getByteVectors() { if (quantizationType == QuantizationType.BINARY) { return quantizeFloatVectorsToBinary(flatFieldVectorsWriter.getVectors()); - } else { + } else if (quantizationType == QuantizationType.SCALAR) { return quantizeFloatVectorsToScalar(flatFieldVectorsWriter.getVectors()); + } else { + throw new UnsupportedOperationException("Not applicable for QuantizationType.NONE"); } } + List getFloatVectors() { + return flatFieldVectorsWriter.getVectors(); + } + FieldInfo fieldInfo() { return fieldInfo; } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java index 92531bb86e..45255da499 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; @@ -55,6 +55,20 @@ List getVectors() { return flatFieldVectorsWriter.getVectors(); } + /** + * Gets the vector dimension. + * + * @return the vector dimension + */ + int getVectorDimension() { + List vectors = flatFieldVectorsWriter.getVectors(); + if (vectors != null && vectors.size() > 0) { + return vectors.get(0).length; + } else { + return -1; + } + } + /** * Gets the field info that holds the description of the field. * diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java index dade15d2ee..cca0233097 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java @@ -6,8 +6,10 @@ package com.nvidia.cuvs.lucene; import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.CuVSIvfPqParams; import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; import java.util.Objects; +import java.util.function.Supplier; public class GPUSearchParams { @@ -15,18 +17,31 @@ public class GPUSearchParams { * TODO: Update boundaries for all parameters when a consensus is reached. * Issue: https://github.com/rapidsai/cuvs-lucene/issues/99 */ - private static final int MIN_WRITER_THREADS = 1; - private static final int MAX_WRITER_THREADS = 32; - private static final int MIN_INT_GRAPH_DEG = 2; - private static final int MAX_INT_GRAPH_DEG = 128; - private static final int MIN_GRAPH_DEG = 1; - private static final int MAX_GRAPH_DEG = 64; + public static final int MIN_WRITER_THREADS = 1; + public static final int MAX_WRITER_THREADS = 512; + public static final int MIN_INT_GRAPH_DEG = 2; + public static final int MAX_INT_GRAPH_DEG = 512; + public static final int MIN_GRAPH_DEG = 1; + public static final int MAX_GRAPH_DEG = 512; + + public static final int DEFAULT_INT_GRAPH_DEGREE = 128; + public static final int DEFAULT_GRAPH_DEGREE = 64; + public static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = + CagraGraphBuildAlgo.NN_DESCENT; + public static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; + public static final int DEFAULT_WRITER_THREADS = 1; + + public static final Supplier DEFAULT_IVF_PQ_PARAMS = + () -> { + return new CuVSIvfPqParams.Builder().build(); + }; private final int writerThreads; private final int intermediateGraphDegree; private final int graphdegree; private final CagraGraphBuildAlgo cagraGraphBuildAlgo; private final IndexType indexType; + private final CuVSIvfPqParams cuVSIvfPqParams; /** * Constructs an instance of {@link GPUSearchParams} with specific parameter values. @@ -36,19 +51,22 @@ public class GPUSearchParams { * @param graphdegree The graph degree to use while building the CAGRA index. * @param cagraGraphBuildAlgo The CAGRA build algorithm to use. * @param indexType The type of index to build - CAGRA, BRUTEFORCE, or both. + * @param cuVSIvfPqParams An instance of CuVSIvfPqParams containing IVF_PQ specific parameters. */ private GPUSearchParams( int writerThreads, int intermediateGraphDegree, int graphdegree, CagraGraphBuildAlgo cagraGraphBuildAlgo, - IndexType indexType) { + IndexType indexType, + CuVSIvfPqParams cuVSIvfPqParams) { super(); this.writerThreads = writerThreads; this.intermediateGraphDegree = intermediateGraphDegree; this.graphdegree = graphdegree; this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; this.indexType = indexType; + this.cuVSIvfPqParams = cuVSIvfPqParams; } /** @@ -96,6 +114,15 @@ public IndexType getIndexType() { return indexType; } + /** + * Get the instance of CuVSIvfPqParams + * + * @return an instance of CuVSIvfPqParams + */ + public CuVSIvfPqParams getCuVSIvfPqParams() { + return cuVSIvfPqParams; + } + @Override public String toString() { return "GPUSearchParams [writerThreads=" @@ -116,11 +143,12 @@ public String toString() { */ public static class Builder { - private int writerThreads = 1; - private int intermediateGraphDegree = 128; - private int graphdegree = 64; - private CagraGraphBuildAlgo cagraGraphBuildAlgo = CagraGraphBuildAlgo.NN_DESCENT; - private IndexType indexType = IndexType.CAGRA; + private int writerThreads = DEFAULT_WRITER_THREADS; + private int intermediateGraphDegree = DEFAULT_INT_GRAPH_DEGREE; + private int graphdegree = DEFAULT_GRAPH_DEGREE; + private CagraGraphBuildAlgo cagraGraphBuildAlgo = DEFAULT_CAGRA_GRAPH_BUILD_ALGO; + private IndexType indexType = DEFAULT_INDEX_TYPE; + private CuVSIvfPqParams cuVSIvfPqParams = null; /** * Set the number of cuVS writer threads while building the index @@ -185,6 +213,17 @@ public Builder withIndexType(IndexType indexType) { return this; } + /** + * Set the instance of {@link CuVSIvfPqParams} + * + * @param cuVSIvfPqParams + * @return instance of {@link Builder} + */ + public Builder withCuVSIvfPqParams(CuVSIvfPqParams cuVSIvfPqParams) { + this.cuVSIvfPqParams = cuVSIvfPqParams; + return this; + } + /** * Validates the input parameters. * @@ -230,9 +269,17 @@ private void validate() throws IllegalArgumentException { * @return instance of {@link GPUSearchParams} */ public GPUSearchParams build() { + if (Objects.isNull(cuVSIvfPqParams)) { + cuVSIvfPqParams = DEFAULT_IVF_PQ_PARAMS.get(); + } validate(); return new GPUSearchParams( - writerThreads, intermediateGraphDegree, graphdegree, cagraGraphBuildAlgo, indexType); + writerThreads, + intermediateGraphDegree, + graphdegree, + cagraGraphBuildAlgo, + indexType, + cuVSIvfPqParams); } } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java index 86004141ec..b4c5a33d90 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java @@ -12,7 +12,9 @@ import org.apache.lucene.codecs.KnnVectorsFormat; /** - * CuVS based codec for GPU based vector search + * A codec that enables GPU-based accelerated HNSW capability and can be used + * to accelerated indexing using GPUs and search using CPUs. Fallbacks to CPU + * based indexing when used on a machine without a GPU and/or cuVS. * * @since 25.10 */ @@ -22,25 +24,51 @@ public class Lucene101AcceleratedHNSWCodec extends FilterCodec { private static final String NAME = "Lucene101AcceleratedHNSWCodec"; private KnnVectorsFormat format; + /** + * Default constructor for {@link Lucene101AcceleratedHNSWCodec}. + * + * @throws Exception + */ public Lucene101AcceleratedHNSWCodec() throws Exception { this(NAME, LuceneProvider.getCodec("101")); } + /** + * Constructor for {@link Lucene101AcceleratedHNSWCodec}. + * + * @param name the codec's name + * @param delegate the delegate codec to filter + */ public Lucene101AcceleratedHNSWCodec(String name, Codec delegate) { super(name, delegate); initializeFormatDefaultValues(); } + /** + * Constructor for {@link Lucene101AcceleratedHNSWCodec}. + * + * @param acceleratedHNSWParams instance of {@link AcceleratedHNSWParams} + * @throws Exception exception + */ public Lucene101AcceleratedHNSWCodec(AcceleratedHNSWParams acceleratedHNSWParams) throws Exception { this(NAME, LuceneProvider.getCodec("101")); initializeFormat(acceleratedHNSWParams); } + /** + * Initialize an instance of {@link Lucene99AcceleratedHNSWVectorsFormat} + * with an instance of {@link AcceleratedHNSWParams} with default parameter values. + */ private void initializeFormatDefaultValues() { initializeFormat(new AcceleratedHNSWParams.Builder().build()); } + /** + * Initialize an instance of {@link Lucene99AcceleratedHNSWVectorsFormat}. + * + * @param acceleratedHNSWParams instance of {@link AcceleratedHNSWParams} to use + */ private void initializeFormat(AcceleratedHNSWParams acceleratedHNSWParams) { try { format = new Lucene99AcceleratedHNSWVectorsFormat(acceleratedHNSWParams); @@ -53,7 +81,7 @@ private void initializeFormat(AcceleratedHNSWParams acceleratedHNSWParams) { } /** - * Get the configured {@link KnnVectorsFormat} + * Get the configured {@link KnnVectorsFormat}. * * @return the instance of the {@link KnnVectorsFormat} */ @@ -63,7 +91,7 @@ public KnnVectorsFormat knnVectorsFormat() { } /** - * Set the {@link KnnVectorsFormat} + * Set the {@link KnnVectorsFormat}. * * @param format the {@link KnnVectorsFormat} to set */ diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java index 233313ec86..3c42707c45 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java @@ -17,6 +17,7 @@ import org.apache.lucene.codecs.hnsw.FlatVectorsFormat; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.search.TaskExecutor; /** * cuVS based KnnVectorsFormat for indexing on GPU and searching on the CPU. @@ -28,9 +29,8 @@ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { private static final Logger log = Logger.getLogger(Lucene99AcceleratedHNSWVectorsFormat.class.getName()); private static final FlatVectorsFormat FLAT_VECTORS_FORMAT; - private static final Integer NUM_MERGE_WORKERS; - private static final int maxDimensions = 4096; - private AcceleratedHNSWParams acceleratedHNSWParams; + private static final int MAX_DIMENSIONS = 4096; + private final AcceleratedHNSWParams acceleratedHNSWParams; static final String HNSW_META_CODEC_NAME = "Lucene99HnswVectorsFormatMeta"; static final String HNSW_META_CODEC_EXT = "vem"; @@ -41,7 +41,6 @@ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { static { try { LUCENE_PROVIDER = LuceneProvider.getInstance("99"); - NUM_MERGE_WORKERS = LUCENE_PROVIDER.getStaticIntParam("DEFAULT_BEAM_WIDTH"); FLAT_VECTORS_FORMAT = LUCENE_PROVIDER.getLuceneFlatVectorsFormatInstance(DefaultFlatVectorScorer.INSTANCE); } catch (Exception e) { @@ -50,7 +49,8 @@ public class Lucene99AcceleratedHNSWVectorsFormat extends KnnVectorsFormat { } /** - * Initializes {@link Lucene99AcceleratedHNSWVectorsFormat} with default parameter values. + * Initializes {@link Lucene99AcceleratedHNSWVectorsFormat} with an instance + * of {@link AcceleratedHNSWParams} with default parameter values. * * @throws LibraryException if the native library fails to load */ @@ -59,7 +59,8 @@ public Lucene99AcceleratedHNSWVectorsFormat() { } /** - * Initializes {@link Lucene99AcceleratedHNSWVectorsFormat} with the given threads, graph degree, etc. + * Initializes {@link Lucene99AcceleratedHNSWVectorsFormat} with an instance + * of {@link AcceleratedHNSWParams}. * * @param acceleratedHNSWParams An instance of {@link AcceleratedHNSWParams} */ @@ -81,17 +82,15 @@ public KnnVectorsWriter fieldsWriter(SegmentWriteState state) throws IOException log.log( Level.WARNING, "GPU based indexing not supported, falling back to using the Lucene99HnswVectorsWriter"); - // TODO: Make num merge workers configurable. try { return LUCENE_PROVIDER.getLuceneHnswVectorsWriterInstance( state, acceleratedHNSWParams.getMaxConn(), acceleratedHNSWParams.getBeamWidth(), flatWriter, - NUM_MERGE_WORKERS, - null); + acceleratedHNSWParams.getNumMergeWorkers(), + new TaskExecutor(acceleratedHNSWParams.getMergeExec())); } catch (Exception e) { - // maybe there is a better suited option to throwing RuntimeException? Need to explore. throw new RuntimeException(e.getMessage()); } } @@ -106,7 +105,6 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException return LUCENE_PROVIDER.getLuceneHnswVectorsReaderInstance( state, FLAT_VECTORS_FORMAT.fieldsReader(state)); } catch (Exception e) { - // maybe there is a better suited option to throwing RuntimeException? Need to explore. throw new RuntimeException(e.getMessage()); } } @@ -116,6 +114,6 @@ public KnnVectorsReader fieldsReader(SegmentReadState state) throws IOException */ @Override public int getMaxDimensions(String fieldName) { - return maxDimensions; + return MAX_DIMENSIONS; } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index 5e74eba934..bc79dfd3c3 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -5,61 +5,48 @@ package com.nvidia.cuvs.lucene; import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.cagraIndexParams; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.createMultiLayerHnswGraph; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.createSingleVectorHnswGraph; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.printInfoStream; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.writeEmpty; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.writeGraph; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.writeMeta; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_CODEC_NAME; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_INDEX_EXT; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_EXT; import static com.nvidia.cuvs.lucene.Lucene99AcceleratedHNSWVectorsFormat.HNSW_META_CODEC_NAME; import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.closeCuVSResourcesInstance; import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.getCuVSResourcesInstance; +import static com.nvidia.cuvs.lucene.Utils.createListFromMergedVectors; import static org.apache.lucene.index.VectorEncoding.FLOAT32; -import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; import static org.apache.lucene.util.RamUsageEstimator.shallowSizeOfInstance; import com.nvidia.cuvs.CagraIndex; import com.nvidia.cuvs.CagraIndexParams; import com.nvidia.cuvs.CuVSMatrix; -import com.nvidia.cuvs.RowView; +import com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.QuantizationType; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.Random; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.logging.Logger; -import java.util.stream.IntStream; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.KnnFieldVectorsWriter; -import org.apache.lucene.codecs.KnnVectorsReader; import org.apache.lucene.codecs.KnnVectorsWriter; -import org.apache.lucene.codecs.hnsw.FlatFieldVectorsWriter; import org.apache.lucene.codecs.hnsw.FlatVectorsWriter; import org.apache.lucene.index.DocsWithFieldSet; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.FloatVectorValues; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.KnnVectorValues; import org.apache.lucene.index.MergeState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Sorter; import org.apache.lucene.index.Sorter.DocMap; -import org.apache.lucene.index.VectorSimilarityFunction; -import org.apache.lucene.internal.hppc.IntObjectHashMap; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.InfoStream; -import org.apache.lucene.util.hnsw.HnswGraph; -import org.apache.lucene.util.hnsw.HnswGraph.NodesIterator; -import org.apache.lucene.util.hnsw.NeighborArray; -import org.apache.lucene.util.packed.DirectMonotonicWriter; /** * This class extends upon the KnnVectorsWriter to - * enable the creation of GPU-based accelerated vector search indexes. + * enable the creation of GPU-based accelerated HNSW based vector search. * * @since 25.10 */ @@ -67,31 +54,24 @@ public class Lucene99AcceleratedHNSWVectorsWriter extends KnnVectorsWriter { private static final long SHALLOW_RAM_BYTES_USED = shallowSizeOfInstance(Lucene99AcceleratedHNSWVectorsWriter.class); - - @SuppressWarnings("unused") - private static final Logger log = - Logger.getLogger(Lucene99AcceleratedHNSWVectorsWriter.class.getName()); - - private static final String CUVS_COMPONENT = "CUVS"; + private static final String COMPONENT = "Lucene99AcceleratedHNSWVectorsWriter"; private static final LuceneProvider LUCENE_PROVIDER; private static final Integer VERSION_CURRENT; - private static final List VECTOR_SIMILARITY_FUNCTIONS; - private final FlatVectorsWriter flatVectorsWriter; // for writing the raw vectors - private final List fields = new ArrayList<>(); + private final AcceleratedHNSWParams acceleratedHNSWParams; + private final FlatVectorsWriter flatVectorsWriter; + private final List fields = new ArrayList<>(); private final InfoStream infoStream; - private IndexOutput cuvsIndex = null; - private IndexOutput hnswMeta = null, hnswVectorIndex = null; - private boolean finished; + private IndexOutput hnswMeta = null; + private IndexOutput hnswVectorIndex = null; private String vemFileName; private String vexFileName; - private final AcceleratedHNSWParams acceleratedHNSWParams; + private boolean finished; static { try { LUCENE_PROVIDER = LuceneProvider.getInstance("99"); VERSION_CURRENT = LUCENE_PROVIDER.getStaticIntParam("VERSION_CURRENT"); - VECTOR_SIMILARITY_FUNCTIONS = LUCENE_PROVIDER.getSimilarityFunctions(); } catch (Exception e) { throw new ExceptionInInitializerError(e.getMessage()); } @@ -114,20 +94,15 @@ public Lucene99AcceleratedHNSWVectorsWriter( this.flatVectorsWriter = flatVectorsWriter; this.infoStream = state.infoStream; this.acceleratedHNSWParams = acceleratedHNSWParams; - vemFileName = IndexFileNames.segmentFileName( state.segmentInfo.name, state.segmentSuffix, HNSW_META_CODEC_EXT); - vexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, HNSW_INDEX_EXT); - boolean success = false; try { - hnswMeta = state.directory.createOutput(vemFileName, state.context); hnswVectorIndex = state.directory.createOutput(vexFileName, state.context); - CodecUtil.writeIndexHeader( hnswMeta, HNSW_META_CODEC_NAME, @@ -140,8 +115,8 @@ public Lucene99AcceleratedHNSWVectorsWriter( VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); - success = true; + printInfoStream(infoStream, COMPONENT, "Lucene99AcceleratedHNSWVectorsWriter is initialized"); } finally { if (success == false) { IOUtils.closeWhileHandlingException(this); @@ -156,55 +131,34 @@ public Lucene99AcceleratedHNSWVectorsWriter( public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException { var encoding = fieldInfo.getVectorEncoding(); if (encoding != FLOAT32) { - throw new IllegalArgumentException("expected float32, got:" + encoding); + throw new IllegalArgumentException("Expected float32, got:" + encoding); } var writer = Objects.requireNonNull(flatVectorsWriter.addField(fieldInfo)); - @SuppressWarnings("unchecked") - var flatWriter = (FlatFieldVectorsWriter) writer; - var cuvsFieldWriter = new GPUFieldWriter(fieldInfo, flatWriter); + var cuvsFieldWriter = new FieldWriter(QuantizationType.NONE, fieldInfo, writer); fields.add(cuvsFieldWriter); return writer; } /** - * A utility method to print info/debugging messages using InfoStream. - * - * @param msg the debugging message to print - */ - private void info(String msg) { - if (infoStream.isEnabled(CUVS_COMPONENT)) { - infoStream.message(CUVS_COMPONENT, msg); - } - } - - /** - * Builds the intermediate CAGRA index and builds and writes the HNSW index + * Builds the intermediate CAGRA index and builds and writes the HNSW index. * * @param fieldInfo instance of FieldInfo that has the field description * @param vectors vectors to index * @throws IOException */ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOException { - if (vectors.size() == 0) { - writeEmpty(fieldInfo); + writeEmpty(fieldInfo, hnswMeta); + return; + } + if (vectors.size() < 2) { + writeSingleVectorGraph(fieldInfo, vectors); return; } - try { CuVSMatrix dataset = Utils.createFloatMatrix( vectors, fieldInfo.getVectorDimension(), getCuVSResourcesInstance()); - - if (dataset.size() < 2) { - // Handle single vector case by creating a dummy HNSW graph - // TODO: Remove this workaround once https://github.com/rapidsai/cuvs/pull/1256 is merged - writeSingleVectorGraph(fieldInfo, vectors); - return; - } - - long startTime = System.nanoTime(); - CagraIndexParams params = cagraIndexParams( acceleratedHNSWParams.getWriterThreads(), @@ -217,14 +171,9 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro .withDataset(dataset) .withIndexParams(params) .build(); - - // Get the adjacency list from CAGRA index CuVSMatrix adjacencyListMatrix = cagraIndex.getGraph(); - int size = (int) dataset.size(); int dimensions = fieldInfo.getVectorDimension(); - - // Create multi-layer HNSW graph from CAGRA GPUBuiltHnswGraph hnswGraph = createMultiLayerHnswGraph( fieldInfo, @@ -232,16 +181,13 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro dimensions, adjacencyListMatrix, vectors, - acceleratedHNSWParams.getHnswLayers()); - + acceleratedHNSWParams.getHnswLayers(), + acceleratedHNSWParams.getGraphdegree(), + params, + QuantizationType.NONE); long vectorIndexOffset = hnswVectorIndex.getFilePointer(); - - // Write the graph to the vector index int[][] graphLevelNodeOffsets = writeGraph(hnswGraph, hnswVectorIndex); - long vectorIndexLength = hnswVectorIndex.getFilePointer() - vectorIndexOffset; - - // Write metadata writeMeta( hnswVectorIndex, hnswMeta, @@ -250,283 +196,14 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro vectorIndexLength, size, hnswGraph, - graphLevelNodeOffsets); - - long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - startTime); - info("HNSW graph created in " + elapsedMillis + "ms, with " + dataset.size() + " vectors"); - + graphLevelNodeOffsets, + acceleratedHNSWParams.getGraphdegree()); cagraIndex.close(); - } catch (Throwable t) { Utils.handleThrowable(t); } } - /** - * Creates a multi-layer HNSW graph with dynamic number of layers. - * M = cagraGraphDegree/2 - * Each layer contains 1/M nodes from the previous layer - * Creates layers until the highest layer has ≤ M nodes - */ - private GPUBuiltHnswGraph createMultiLayerHnswGraph( - FieldInfo fieldInfo, - int size, - int dimensions, - CuVSMatrix adjacencyListMatrix, - List vectors, - int hnswLayers) - throws Throwable { - - // Calculate M as cagraGraphDegree/2 - int M = acceleratedHNSWParams.getGraphdegree() / 2; - - // Store all layers data - List layerNodes = new ArrayList<>(); - List layerAdjacencies = new ArrayList<>(); - - // Layer 0: Use full CAGRA adjacency list - layerNodes.add(null); // Layer 0 contains all nodes, so we don't need to store node list - layerAdjacencies.add(adjacencyListMatrix); - - int currentLayerSize = size; - int layerIndex = 1; - Random random = new Random(); - - while (layerIndex < hnswLayers && currentLayerSize > 1) { - // Calculate size for next layer (1/M of current layer) - int nextLayerSize = Math.max(2, currentLayerSize / M); - // Select nodes for this layer - SortedSet selectedNodesSet = new TreeSet<>(); - - if (layerIndex == 1) { - // Select from all nodes (Layer 0) - while (selectedNodesSet.size() < nextLayerSize) { - selectedNodesSet.add(random.nextInt(size)); - } - } else { - // Select from previous layer nodes - int[] prevLayerNodes = layerNodes.get(layerNodes.size() - 1); - while (selectedNodesSet.size() < nextLayerSize) { - int idx = random.nextInt(prevLayerNodes.length); - selectedNodesSet.add(prevLayerNodes[idx]); - } - } - - // Convert to sorted array - int[] selectedNodes = - selectedNodesSet.stream().mapToInt(Integer::intValue).sorted().toArray(); - - layerNodes.add(selectedNodes); - - // Extract vectors for selected nodes - float[][] selectedVectors = new float[nextLayerSize][]; - for (int i = 0; i < nextLayerSize; i++) { - selectedVectors[i] = vectors.get(selectedNodes[i]); - } - - // Build CAGRA graph for this layer - layerAdjacencies.add(buildCagraGraphForSubset(selectedVectors, selectedNodes)); - - // Update for next iteration - currentLayerSize = nextLayerSize; - layerIndex++; - - // Use different seed for each layer - random = new Random(new Random().nextLong()); - } - - // Create the multi-layer graph with all layers - return new GPUBuiltHnswGraph(size, dimensions, layerNodes, layerAdjacencies); - } - - /** - * Builds a CAGRA graph for a subset of vectors - */ - private CuVSMatrix buildCagraGraphForSubset(float[][] vectors, int[] selectedNodes) - throws Throwable { - // Create CuVSMatrix from the subset vectors - CuVSMatrix subsetDataset = CuVSMatrix.ofArray(vectors); - - // Build CAGRA index for the subset - CagraIndexParams params = - cagraIndexParams( - acceleratedHNSWParams.getWriterThreads(), - acceleratedHNSWParams.getIntermediateGraphDegree(), - acceleratedHNSWParams.getGraphdegree(), - acceleratedHNSWParams.getCagraGraphBuildAlgo(), - acceleratedHNSWParams.getCuVSIvfPqParams()); - - CagraIndex subsetIndex = - CagraIndex.newBuilder(getCuVSResourcesInstance()) - .withDataset(subsetDataset) - .withIndexParams(params) - .build(); - - // Get adjacency list from subset CAGRA index - CuVSMatrix cagraGraph = subsetIndex.getGraph(); - - long numNodes = cagraGraph.size(); - long degree = cagraGraph.columns(); - - // Create a re-mapped adjacency list - int[][] remappedAdjacency = new int[(int) numNodes][(int) degree]; - - for (int i = 0; i < numNodes; i++) { - RowView rv = cagraGraph.getRow(i); - for (int j = 0; j < degree && j < rv.size(); j++) { - int subsetIndex1 = rv.getAsInt(j); - // Map subset index to original node ID - if (subsetIndex1 >= 0 && subsetIndex1 < selectedNodes.length) { - remappedAdjacency[i][j] = selectedNodes[subsetIndex1]; - } else { - // Invalid index, use self-reference - remappedAdjacency[i][j] = selectedNodes[i]; - } - } - } - - subsetIndex.close(); - return CuVSMatrix.ofArray(remappedAdjacency); - } - - /** - * Writes the meta information for the index. - * - * @param vectorIndex instance of IndexOutput - * @param meta instance of IndexOutput - * @param field instance of FieldInfo - * @param vectorIndexOffset vector index offset - * @param vectorIndexLength vector index length - * @param count the count of vectors - * @param graph instance of HnswGraph - * @param graphLevelNodeOffsets graph level node offsets - * @throws IOException I/O Exceptions - */ - private void writeMeta( - IndexOutput vectorIndex, - IndexOutput meta, - FieldInfo field, - long vectorIndexOffset, - long vectorIndexLength, - int count, - HnswGraph graph, - int[][] graphLevelNodeOffsets) - throws IOException { - - meta.writeInt(field.number); - meta.writeInt(field.getVectorEncoding().ordinal()); - meta.writeInt(distFuncToOrd(field.getVectorSimilarityFunction())); - meta.writeVLong(vectorIndexOffset); - meta.writeVLong(vectorIndexLength); - meta.writeVInt(field.getVectorDimension()); - meta.writeInt(count); - meta.writeVInt(acceleratedHNSWParams.getGraphdegree() / 2); // M = cagraGraphDegree/2 - - // write graph nodes on each level - if (graph == null) { - meta.writeVInt(0); - } else { - meta.writeVInt(graph.numLevels()); - long valueCount = 0; - for (int level = 0; level < graph.numLevels(); level++) { - NodesIterator nodesOnLevel = graph.getNodesOnLevel(level); - valueCount += nodesOnLevel.size(); - if (level > 0) { - int[] nol = new int[nodesOnLevel.size()]; - int numberConsumed = nodesOnLevel.consume(nol); - Arrays.sort(nol); - assert numberConsumed == nodesOnLevel.size(); - meta.writeVInt(nol.length); // number of nodes on a level - for (int i = nodesOnLevel.size() - 1; i > 0; --i) { - nol[i] -= nol[i - 1]; - } - for (int n : nol) { - meta.writeVInt(n); - } - } else { - assert nodesOnLevel.size() == count : "Level 0 expects to have all nodes"; - } - } - - long start = vectorIndex.getFilePointer(); - meta.writeLong(start); - meta.writeVInt(16); // DIRECT_MONOTONIC_BLOCK_SHIFT); - - final DirectMonotonicWriter memoryOffsetsWriter = - DirectMonotonicWriter.getInstance(meta, vectorIndex, valueCount, 16); - long cumulativeOffsetSum = 0; - for (int[] levelOffsets : graphLevelNodeOffsets) { - for (int v : levelOffsets) { - memoryOffsetsWriter.add(cumulativeOffsetSum); - cumulativeOffsetSum += v; - } - } - - memoryOffsetsWriter.finish(); - - meta.writeLong(vectorIndex.getFilePointer() - start); - } - } - - /** - * Returns a 2D array of offsets (information written while writing the meta info) - * - * @param graph instance of GPUBuiltHnswGraph - * @param vectorIndex instance of IndexOutput - * @return a 2D array of offsets - * @throws IOException I/O Exceptions - */ - private int[][] writeGraph(GPUBuiltHnswGraph graph, IndexOutput vectorIndex) throws IOException { - // write vectors' neighbors on each level into the vectorIndex file - int countOnLevel0 = graph.size(); - int[][] offsets = new int[graph.numLevels()][]; - int[] scratch = new int[graph.maxConn() * 2]; - for (int level = 0; level < graph.numLevels(); level++) { - int[] sortedNodes = NodesIterator.getSortedNodes(graph.getNodesOnLevel(level)); - offsets[level] = new int[sortedNodes.length]; - int nodeOffsetId = 0; - - for (int node : sortedNodes) { - // Get node neighbors - NeighborArray neighbors = graph.getNeighbors(level, node); - // Get the size of the neighbor array - int size = neighbors.size(); - // Write size in VInt as the neighbors list is typically small - long offsetStart = vectorIndex.getFilePointer(); - // Get neighbors - int[] nnodes = neighbors.nodes(); - // Sort them - Arrays.sort(nnodes, 0, size); - // Now that we have sorted, do delta encoding to minimize the required bits to store the - // information - int actualSize = 0; - if (size > 0) { - scratch[0] = nnodes[0]; - actualSize = 1; - } - // De-duplication - for (int i = 1; i < size; i++) { - assert nnodes[i] < countOnLevel0 : "node too large: " + nnodes[i] + ">=" + countOnLevel0; - // Sorting step helps here - if (nnodes[i - 1] == nnodes[i]) { - continue; - } - scratch[actualSize++] = nnodes[i] - nnodes[i - 1]; - } - // Write the size after duplicates are removed - vectorIndex.writeVInt(actualSize); - // Write de-duplicated neighbors - for (int i = 0; i < actualSize; i++) { - vectorIndex.writeVInt(scratch[i]); - } - offsets[level][nodeOffsetId++] = - Math.toIntExact(vectorIndex.getFilePointer() - offsetStart); - } - } - // Return offsets (information written while writing the meta info) - return offsets; - } - /** * Build the indexes and writes it to the disk. */ @@ -548,8 +225,8 @@ public void flush(int maxDoc, DocMap sortMap) throws IOException { * @param fieldData * @throws IOException */ - private void writeField(GPUFieldWriter fieldData) throws IOException { - writeFieldInternal(fieldData.fieldInfo(), fieldData.getVectors()); + private void writeField(FieldWriter fieldData) throws IOException { + writeFieldInternal(fieldData.fieldInfo(), fieldData.getFloatVectors()); } /** @@ -559,18 +236,15 @@ private void writeField(GPUFieldWriter fieldData) throws IOException { * @param sortMap instance of the DocMap * @throws IOException */ - private void writeSortingField(GPUFieldWriter fieldData, Sorter.DocMap sortMap) - throws IOException { - + private void writeSortingField(FieldWriter fieldData, Sorter.DocMap sortMap) throws IOException { DocsWithFieldSet oldDocsWithFieldSet = fieldData.getDocsWithFieldSet(); - final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; // new ord to old ord + final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; mapOldOrdToNewOrd(oldDocsWithFieldSet, sortMap, null, new2OldOrd, null); - List sortedVectors = new ArrayList(); - for (int i = 0; i < fieldData.getVectors().size(); i++) { - sortedVectors.add(fieldData.getVectors().get(new2OldOrd[i])); + List floatVectors = fieldData.getFloatVectors(); + for (int i = 0; i < floatVectors.size(); i++) { + sortedVectors.add(floatVectors.get(new2OldOrd[i])); } - writeFieldInternal(fieldData.fieldInfo(), sortedVectors); } @@ -583,23 +257,13 @@ private void writeSortingField(GPUFieldWriter fieldData, Sorter.DocMap sortMap) */ private void writeSingleVectorGraph(FieldInfo fieldInfo, List vectors) throws IOException { - // Workaround for CAGRA not supporting single vector indexes - // TODO: Remove this method once https://github.com/rapidsai/cuvs/pull/1256 is merged try { int size = 1; int dimensions = fieldInfo.getVectorDimension(); - - // Create a dummy HNSW graph for a single vector GPUBuiltHnswGraph hnswGraph = createSingleVectorHnswGraph(size, dimensions); - long vectorIndexOffset = hnswVectorIndex.getFilePointer(); - - // Write the graph to the vector index int[][] graphLevelNodeOffsets = writeGraph(hnswGraph, hnswVectorIndex); - long vectorIndexLength = hnswVectorIndex.getFilePointer() - vectorIndexOffset; - - // Write metadata writeMeta( hnswVectorIndex, hnswMeta, @@ -608,127 +272,17 @@ private void writeSingleVectorGraph(FieldInfo fieldInfo, List vectors) vectorIndexLength, size, hnswGraph, - graphLevelNodeOffsets); - - long elapsedMillis = Utils.nanosToMillis(System.nanoTime() - System.nanoTime()); - info("Single vector HNSW graph created in " + elapsedMillis + "ms, with " + size + " vector"); - - } catch (Throwable t) { - Utils.handleThrowable(t); - } - } - - /** - * Creates a dummy HNSW graph for a single vector. - * The graph will have 1 level with 1 node and no neighbors. - */ - private GPUBuiltHnswGraph createSingleVectorHnswGraph(int size, int dimensions) throws Throwable { - // Create adjacency list for single node with no neighbors - int[][] singleNodeAdjacency = new int[][] {{-1}}; // -1 indicates no neighbors - - // Create CuVSMatrix from the adjacency list - CuVSMatrix adjacencyMatrix = CuVSMatrix.ofArray(singleNodeAdjacency); - - // Create layer data for single-level graph - List layerNodes = new ArrayList<>(); - List layerAdjacencies = new ArrayList<>(); - - // Layer 0: contains all nodes (just the single node) - layerNodes.add(null); // Layer 0 contains all nodes, so we don't need to store node list - layerAdjacencies.add(adjacencyMatrix); - - // Create the single-layer graph - return new GPUBuiltHnswGraph(size, dimensions, layerNodes, layerAdjacencies); - } - - /** - * Writes an empty meta information for the field. - * - * @param fieldInfo instance of FieldInfo - * @throws IOException I/O Exceptions - */ - private void writeEmpty(FieldInfo fieldInfo) throws IOException { - writeMeta(null, hnswMeta, fieldInfo, 0, 0, 0, null, null); - } - - static int distFuncToOrd(VectorSimilarityFunction func) { - for (int i = 0; i < VECTOR_SIMILARITY_FUNCTIONS.size(); i++) { - if (VECTOR_SIMILARITY_FUNCTIONS.get(i).equals(func)) { - return (byte) i; - } - } - throw new IllegalArgumentException("invalid distance function: " + func); - } - - /** - * Uses the CAGRA API to merge the CAGRA indexes. - * - * @param fieldInfo instance of FieldInfo - * @param mergeState instance of MergeState - * @throws IOException I/O Exceptions - */ - private void mergeCagraIndexes(FieldInfo fieldInfo, MergeState mergeState) throws IOException { - try { - - List cagraIndexes = new ArrayList<>(); - // We need this count so that the merged segment's meta information has the vector count. - int totalVectorCount = 0; - - for (int i = 0; i < mergeState.knnVectorsReaders.length; i++) { - KnnVectorsReader knnReader = mergeState.knnVectorsReaders[i]; - // Access the CAGRA index for this field from the reader - - if (knnReader != null) { - if (knnReader instanceof CuVS2510GPUVectorsReader cvr) { - if (cvr != null) { - totalVectorCount += cvr.getFieldEntries().get(fieldInfo.number).count(); - CagraIndex cagraIndex = getCagraIndexFromReader(cvr, fieldInfo.name); - if (cagraIndex != null) { - cagraIndexes.add(cagraIndex); - } - } - } else { - // This should never happen - throw new RuntimeException( - "Reader is not of CuVSVectorsReader type. Instead it is: " + knnReader.getClass()); - } - } - } - assert cagraIndexes.size() > 1; - - CagraIndex mergedIndex = - CagraIndex.merge(cagraIndexes.toArray(new CagraIndex[cagraIndexes.size()])); - writeMergedCagraIndex(fieldInfo, mergedIndex, totalVectorCount); - info("Successfully merged " + cagraIndexes.size() + " CAGRA indexes using native merge API"); - + graphLevelNodeOffsets, + acceleratedHNSWParams.getGraphdegree()); } catch (Throwable t) { Utils.handleThrowable(t); } } /** - * Creates List from merged vectors - */ - private List createListFromMergedVectors(FloatVectorValues mergedVectorValues) - throws IOException { - List vectors = new ArrayList(); - KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); - for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { - float[] vector = mergedVectorValues.vectorValue(iter.index()); - vectors.add(vector.clone()); - } - return vectors; - } - - /** - * Fallback method that rebuilds indexes from merged vectors. - * Used when native CAGRA merge() is not possible. Also used - * when non-CAGRA index types are used (for e.g. Brute Force index). + * Create combined data set for the merged segment and call writeFieldInternal. */ private void vectorBasedMerge(FieldInfo fieldInfo, MergeState mergeState) throws IOException { - if (fieldInfo.getVectorEncoding() != FLOAT32) { - throw new AssertionError("Only Float32 supported"); - } try { List dataset = createListFromMergedVectors( @@ -739,74 +293,13 @@ private void vectorBasedMerge(FieldInfo fieldInfo, MergeState mergeState) throws } } - /** - * Extracts the CAGRA index for a specific field from a CuVSVectorsReader. - */ - private CagraIndex getCagraIndexFromReader(CuVS2510GPUVectorsReader reader, String fieldName) { - try { - IntObjectHashMap cuvsIndices = reader.getCuvsIndexes(); - FieldInfos fieldInfos = reader.getFieldInfos(); - FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldName); - - if (fieldInfo != null) { - GPUIndex cuvsIndex = cuvsIndices.get(fieldInfo.number); - if (cuvsIndex != null) { - return cuvsIndex.getCagraIndex(); - } - } - } catch (Exception e) { - e.printStackTrace(); - info("Failed to extract CAGRA index for field " + fieldName + ": " + e.getMessage()); - } - return null; - } - - /** - * Writes a pre-built merged CAGRA index to the output. - */ - private void writeMergedCagraIndex(FieldInfo fieldInfo, CagraIndex mergedIndex, int vectorCount) - throws IOException { - try { - var cagraIndexOutputStream = new IndexOutputOutputStream(cuvsIndex); - - // Serialize the merged index - Path tmpFile = - Files.createTempFile(getCuVSResourcesInstance().tempDirectory(), "mergedindex", "cag"); - mergedIndex.serialize(cagraIndexOutputStream, tmpFile); - - // TODO: Path to writeFieldInternal missing. Fix this. - - // Clean up the merged index - mergedIndex.close(); - } catch (Throwable t) { - Utils.handleThrowable(t); - } - } - /** * Write field for merging. */ @Override public void mergeOneField(FieldInfo fieldInfo, MergeState mergeState) throws IOException { flatVectorsWriter.mergeOneField(fieldInfo, mergeState); - - // Since CAGRA merge does not support merging of indexes with purging of deletes, - // we fallback to vector-based re-indexing. Issue: - // https://github.com/rapidsai/cuvs/issues/1253 - boolean hasDeletions = - IntStream.range(0, mergeState.liveDocs.length) - .anyMatch( - i -> - mergeState.liveDocs[i] == null - || IntStream.range(0, mergeState.maxDocs[i]) - .anyMatch(j -> !mergeState.liveDocs[i].get(j))); - - if (mergeState.knnVectorsReaders.length > 1 && !hasDeletions) { - mergeCagraIndexes(fieldInfo, mergeState); - } else { - // CAGRA's merge API does not handle the trivial case of merging 1 index. - vectorBasedMerge(fieldInfo, mergeState); - } + vectorBasedMerge(fieldInfo, mergeState); } /** @@ -819,11 +312,6 @@ public void finish() throws IOException { } finished = true; flatVectorsWriter.finish(); - - if (cuvsIndex != null) { - CodecUtil.writeFooter(cuvsIndex); - } - if (hnswMeta != null) { // write end of fields marker hnswMeta.writeInt(-1); @@ -839,7 +327,8 @@ public void finish() throws IOException { */ @Override public void close() throws IOException { - IOUtils.close(cuvsIndex, hnswMeta, hnswVectorIndex, flatVectorsWriter); + printInfoStream(infoStream, COMPONENT, "Closing resources"); + IOUtils.close(hnswMeta, hnswVectorIndex, flatVectorsWriter); closeCuVSResourcesInstance(); } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java index 1bdc62db98..ade0e70270 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java @@ -62,7 +62,7 @@ public class LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter extends KnnVector private static final String COMPONENT = "Lucene99AcceleratedHNSWBinaryQuantizedVectorsWriter"; private final FlatVectorsWriter flatVectorsWriter; - private final List fields = new ArrayList<>(); + private final List fields = new ArrayList<>(); private final InfoStream infoStream; private final AcceleratedHNSWParams acceleratedHNSWParams; private IndexOutput hnswMeta = null, hnswVectorIndex = null; @@ -133,7 +133,7 @@ public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException throw new IllegalArgumentException("expected float32, got:" + encoding); } var writer = Objects.requireNonNull(flatVectorsWriter.addField(fieldInfo)); - var cuvsFieldWriter = new QuantizedFieldWriter(QuantizationType.BINARY, fieldInfo, writer); + var cuvsFieldWriter = new FieldWriter(QuantizationType.BINARY, fieldInfo, writer); fields.add(cuvsFieldWriter); return writer; } @@ -238,8 +238,8 @@ public void flush(int maxDoc, DocMap sortMap) throws IOException { * @param fieldData * @throws IOException */ - private void writeField(QuantizedFieldWriter fieldData) throws IOException { - writeFieldInternal(fieldData.fieldInfo(), fieldData.getVectors()); + private void writeField(FieldWriter fieldData) throws IOException { + writeFieldInternal(fieldData.fieldInfo(), fieldData.getByteVectors()); } /** @@ -249,15 +249,14 @@ private void writeField(QuantizedFieldWriter fieldData) throws IOException { * @param sortMap instance of the DocMap * @throws IOException */ - private void writeSortingField(QuantizedFieldWriter fieldData, Sorter.DocMap sortMap) - throws IOException { + private void writeSortingField(FieldWriter fieldData, Sorter.DocMap sortMap) throws IOException { DocsWithFieldSet oldDocsWithFieldSet = fieldData.getDocsWithFieldSet(); final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; // new ord to old ord mapOldOrdToNewOrd(oldDocsWithFieldSet, sortMap, null, new2OldOrd, null); List sortedVectors = new ArrayList(); - List vectors = fieldData.getVectors(); + List vectors = fieldData.getByteVectors(); for (int i = 0; i < vectors.size(); i++) { sortedVectors.add(vectors.get(new2OldOrd[i])); } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java index 11a085ee64..3e3713df91 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java @@ -63,7 +63,7 @@ public class LuceneAcceleratedHNSWScalarQuantizedVectorsWriter extends KnnVector private static final Integer VERSION_CURRENT; private final FlatVectorsWriter flatVectorsWriter; - private final List fields = new ArrayList<>(); + private final List fields = new ArrayList<>(); private final InfoStream infoStream; private final AcceleratedHNSWParams acceleratedHNSWParams; private IndexOutput hnswMeta = null, hnswVectorIndex = null; @@ -142,7 +142,7 @@ public KnnFieldVectorsWriter addField(FieldInfo fieldInfo) throws IOException throw new IllegalArgumentException("expected float32, got:" + encoding); } var writer = Objects.requireNonNull(flatVectorsWriter.addField(fieldInfo)); - var cuvsFieldWriter = new QuantizedFieldWriter(QuantizationType.SCALAR, fieldInfo, writer); + var cuvsFieldWriter = new FieldWriter(QuantizationType.SCALAR, fieldInfo, writer); fields.add(cuvsFieldWriter); return writer; } @@ -166,7 +166,7 @@ private static byte[] convertSignedToUnsigned(byte[] signedVector) { * @param vectors quantized vectors * @throws IOException */ - private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOException { + private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOException { if (vectors.size() == 0) { writeEmpty(fieldInfo, hnswMeta); return; @@ -177,8 +177,8 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw // Convert 7-bit signed bytes to 8-bit unsigned bytes for cuVS compatibility List unsignedVectors = new ArrayList<>(vectors.size()); - for (byte[] signedVector : vectors) { - unsignedVectors.add(convertSignedToUnsigned(signedVector)); + for (Object signedVector : vectors) { + unsignedVectors.add(convertSignedToUnsigned((byte[]) signedVector)); } // Create CuVSMatrix with BYTE data type (unsigned bytes) @@ -264,8 +264,8 @@ public void flush(int maxDoc, DocMap sortMap) throws IOException { * @param fieldData * @throws IOException */ - private void writeField(QuantizedFieldWriter fieldData) throws IOException { - writeFieldInternal(fieldData.fieldInfo(), fieldData.getVectors()); + private void writeField(FieldWriter fieldData) throws IOException { + writeFieldInternal(fieldData.fieldInfo(), fieldData.getByteVectors()); } /** @@ -275,16 +275,16 @@ private void writeField(QuantizedFieldWriter fieldData) throws IOException { * @param sortMap instance of the DocMap * @throws IOException */ - private void writeSortingField(QuantizedFieldWriter fieldData, Sorter.DocMap sortMap) - throws IOException { + private void writeSortingField(FieldWriter fieldData, Sorter.DocMap sortMap) throws IOException { DocsWithFieldSet oldDocsWithFieldSet = fieldData.getDocsWithFieldSet(); final int[] new2OldOrd = new int[oldDocsWithFieldSet.cardinality()]; // new ord to old ord mapOldOrdToNewOrd(oldDocsWithFieldSet, sortMap, null, new2OldOrd, null); List sortedVectors = new ArrayList(); - for (int i = 0; i < fieldData.getVectors().size(); i++) { - sortedVectors.add(fieldData.getVectors().get(new2OldOrd[i])); + List byteVectors = fieldData.getByteVectors(); + for (int i = 0; i < byteVectors.size(); i++) { + sortedVectors.add(byteVectors.get(new2OldOrd[i])); } writeFieldInternal(fieldData.fieldInfo(), sortedVectors); diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java index 21898f8ee4..c55aa7c3e3 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java @@ -4,13 +4,19 @@ */ package com.nvidia.cuvs.lucene; +import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS; + import com.nvidia.cuvs.CuVSMatrix; import com.nvidia.cuvs.CuVSResources; import java.io.IOException; import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.lucene.index.FloatVectorValues; +import org.apache.lucene.index.KnnVectorValues; +import org.apache.lucene.util.InfoStream; /** * This class provides common static utility methods. @@ -168,4 +174,35 @@ static void handleThrowableWithIgnore(Throwable t, String msg) throws IOExceptio } handleThrowable(t); } + + /** + * Creates a list of float vectors from the input + * + * @param mergedVectorValues instance of {@link FloatVectorValues} + * @return a list of float arrays + * @throws IOException I/O Exception + */ + static List createListFromMergedVectors(FloatVectorValues mergedVectorValues) + throws IOException { + List vectors = new ArrayList(); + KnnVectorValues.DocIndexIterator iter = mergedVectorValues.iterator(); + for (int docV = iter.nextDoc(); docV != NO_MORE_DOCS; docV = iter.nextDoc()) { + float[] vector = mergedVectorValues.vectorValue(iter.index()); + vectors.add(vector.clone()); + } + return vectors; + } + + /** + * Utility to print info/debug messages via InfoStream. + * + * @param infoStream the writer's infostream + * @param component the name of the index writer + * @param msg the log message to push via the InfoStream + */ + static void info(InfoStream infoStream, String component, String msg) { + if (infoStream.isEnabled(component)) { + infoStream.message(component, msg); + } + } } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java index 1fc9b8f20a..66a657e6e3 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java @@ -5,7 +5,31 @@ package com.nvidia.cuvs.lucene; -import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_BEAM_WIDTH; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_CAGRA_GRAPH_BUILD_ALGO; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_GRAPH_DEGREE; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_HNSW_LAYERS; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_INT_GRAPH_DEGREE; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_MAX_CONN; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_NUM_MERGE_WORKERS; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_WRITER_THREADS; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_BEAM_WIDTH; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_HNSW_LAYERS; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_INT_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_MAX_CONN; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_NUM_MERGE_WORKERS; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_WRITER_THREADS; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_BEAM_WIDTH; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_HNSW_LAYERS; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_INT_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_MAX_CONN; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_NUM_MERGE_WORKERS; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_WRITER_THREADS; +import static java.lang.Integer.MAX_VALUE; +import static java.lang.Integer.MIN_VALUE; + import java.util.Random; import java.util.logging.Logger; import org.apache.lucene.tests.util.LuceneTestCase; @@ -24,20 +48,22 @@ public class TestAcceleratedHNSWParams extends LuceneTestCase { @Test public void testAcceleratedHNSWParamsDefaultValues() { AcceleratedHNSWParams params = new AcceleratedHNSWParams.Builder().build(); - assertEquals(32, params.getBeamWidth()); - assertEquals(64, params.getGraphdegree()); - assertEquals(1, params.getHnswLayers()); - assertEquals(128, params.getIntermediateGraphDegree()); - assertEquals(32, params.getMaxConn()); - assertEquals(1, params.getWriterThreads()); - assertEquals(1, params.getNumMergeWorkers()); - assertEquals(CagraGraphBuildAlgo.NN_DESCENT, params.getCagraGraphBuildAlgo()); + assertEquals(DEFAULT_BEAM_WIDTH, params.getBeamWidth()); + assertEquals(DEFAULT_GRAPH_DEGREE, params.getGraphdegree()); + assertEquals(DEFAULT_HNSW_LAYERS, params.getHnswLayers()); + assertEquals(DEFAULT_INT_GRAPH_DEGREE, params.getIntermediateGraphDegree()); + assertEquals(DEFAULT_MAX_CONN, params.getMaxConn()); + assertEquals(DEFAULT_WRITER_THREADS, params.getWriterThreads()); + assertEquals(DEFAULT_NUM_MERGE_WORKERS, params.getNumMergeWorkers()); + assertEquals(DEFAULT_CAGRA_GRAPH_BUILD_ALGO, params.getCagraGraphBuildAlgo()); } @Test public void testAcceleratedHNSWParamsInvalidBeamWidth() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(17, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_BEAM_WIDTH), random.nextInt(MAX_BEAM_WIDTH + 1, MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new AcceleratedHNSWParams.Builder().withBeamWidth(v).build()); @@ -47,7 +73,9 @@ public void testAcceleratedHNSWParamsInvalidBeamWidth() { @Test public void testAcceleratedHNSWParamsInvalidGraphDegree() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(65, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_GRAPH_DEG), random.nextInt(MAX_GRAPH_DEG + 1, MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new AcceleratedHNSWParams.Builder().withGraphDegree(v).build()); @@ -57,7 +85,9 @@ public void testAcceleratedHNSWParamsInvalidGraphDegree() { @Test public void testAcceleratedHNSWParamsInvalidHNSWLayers() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(3, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_HNSW_LAYERS), random.nextInt(MAX_HNSW_LAYERS + 1, MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new AcceleratedHNSWParams.Builder().withHNSWLayer(v).build()); @@ -67,7 +97,10 @@ public void testAcceleratedHNSWParamsInvalidHNSWLayers() { @Test public void testAcceleratedHNSWParamsInvalidIntGraphDegree() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(129, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_INT_GRAPH_DEG), + random.nextInt(MAX_INT_GRAPH_DEG + 1, MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new AcceleratedHNSWParams.Builder().withIntermediateGraphDegree(v).build()); @@ -77,7 +110,9 @@ public void testAcceleratedHNSWParamsInvalidIntGraphDegree() { @Test public void testAcceleratedHNSWParamsInvalidMaxConn() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(9, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_MAX_CONN), random.nextInt(MAX_MAX_CONN + 1, MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new AcceleratedHNSWParams.Builder().withMaxConn(v).build()); @@ -87,7 +122,10 @@ public void testAcceleratedHNSWParamsInvalidMaxConn() { @Test public void testAcceleratedHNSWParamsInvalidWriterThreads() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(2, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_WRITER_THREADS), + random.nextInt(MAX_WRITER_THREADS + 1, Integer.MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new AcceleratedHNSWParams.Builder().withWriterThreads(v).build()); @@ -97,20 +135,16 @@ public void testAcceleratedHNSWParamsInvalidWriterThreads() { @Test public void testAcceleratedHNSWParamsInvalidNumMergeWorkers() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(33, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_NUM_MERGE_WORKERS), + random.nextInt(MAX_NUM_MERGE_WORKERS + 1, MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new AcceleratedHNSWParams.Builder().withNumMergeWorkers(v).build()); } } - @Test - public void testAcceleratedHNSWParamsInvalidMergeExecutorService() { - assertThrows( - IllegalArgumentException.class, - () -> new AcceleratedHNSWParams.Builder().withMergeExecutorService(null).build()); - } - @Test public void testAcceleratedHNSWParamsInvalidCagraGraphBuildAlgo() { assertThrows( @@ -118,13 +152,6 @@ public void testAcceleratedHNSWParamsInvalidCagraGraphBuildAlgo() { () -> new AcceleratedHNSWParams.Builder().withCagraGraphBuildAlgo(null).build()); } - @Test - public void testAcceleratedHNSWParamsInvalidCuVSIvfPqParams() { - assertThrows( - IllegalArgumentException.class, - () -> new AcceleratedHNSWParams.Builder().withCuVSIvfPqParams(null).build()); - } - @BeforeClass public static void beforeClass() { random = random(); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java index 41d660d62e..495033172f 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java @@ -5,8 +5,20 @@ package com.nvidia.cuvs.lucene; -import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; -import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_CAGRA_GRAPH_BUILD_ALGO; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_GRAPH_DEGREE; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_INDEX_TYPE; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_INT_GRAPH_DEGREE; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_WRITER_THREADS; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MAX_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MAX_INT_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MAX_WRITER_THREADS; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MIN_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MIN_INT_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MIN_WRITER_THREADS; +import static java.lang.Integer.MAX_VALUE; +import static java.lang.Integer.MIN_VALUE; + import java.util.Random; import java.util.logging.Logger; import org.apache.lucene.tests.util.LuceneTestCase; @@ -25,17 +37,19 @@ public class TestGPUSearchParams extends LuceneTestCase { @Test public void testGPUSearchParamsDefaultValues() { GPUSearchParams params = new GPUSearchParams.Builder().build(); - assertEquals(64, params.getGraphdegree()); - assertEquals(128, params.getIntermediateGraphDegree()); - assertEquals(1, params.getWriterThreads()); - assertEquals(CagraGraphBuildAlgo.NN_DESCENT, params.getCagraGraphBuildAlgo()); - assertEquals(IndexType.CAGRA, params.getIndexType()); + assertEquals(DEFAULT_GRAPH_DEGREE, params.getGraphdegree()); + assertEquals(DEFAULT_INT_GRAPH_DEGREE, params.getIntermediateGraphDegree()); + assertEquals(DEFAULT_WRITER_THREADS, params.getWriterThreads()); + assertEquals(DEFAULT_CAGRA_GRAPH_BUILD_ALGO, params.getCagraGraphBuildAlgo()); + assertEquals(DEFAULT_INDEX_TYPE, params.getIndexType()); } @Test public void testGPUSearchParamsInvalidGraphDegree() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(65, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_GRAPH_DEG), random.nextInt(MAX_GRAPH_DEG + 1, MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new GPUSearchParams.Builder().withGraphDegree(v).build()); @@ -45,7 +59,10 @@ public void testGPUSearchParamsInvalidGraphDegree() { @Test public void testGPUSearchParamsInvalidIntermediateGraphDegree() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(129, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_INT_GRAPH_DEG), + random.nextInt(MAX_INT_GRAPH_DEG + 1, MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new GPUSearchParams.Builder().withIntermediateGraphDegree(v).build()); @@ -55,7 +72,10 @@ public void testGPUSearchParamsInvalidIntermediateGraphDegree() { @Test public void testGPUSearchParamsInvalidWriterThreads() { for (int v : - new int[] {random.nextInt(Integer.MIN_VALUE, 1), random.nextInt(2, Integer.MAX_VALUE)}) { + new int[] { + random.nextInt(MIN_VALUE, MIN_WRITER_THREADS), + random.nextInt(MAX_WRITER_THREADS + 1, MAX_VALUE) + }) { assertThrows( IllegalArgumentException.class, () -> new GPUSearchParams.Builder().withWriterThreads(v).build()); From d18d9c79dee82f6c3fc91768568ee97fcbf1e930 Mon Sep 17 00:00:00 2001 From: Vivek Narang <123010842+narangvivek10@users.noreply.github.com> Date: Mon, 4 May 2026 11:07:47 -0400 Subject: [PATCH 078/105] Introduce a heuristic approach to CAGRA build algorithm selection and its index parameters (NVIDIA/cuvs-lucene#135) To introduce a heuristic approach to CAGRA build algorithm and its index parameter selection using underlying cuvs utilities. Summary of changes: - Introduce a centralized approach to producing instances of `CagraIndexParams` via `CagraIndexParamsFactory` - Add a parameter in the input parameter classes to select the strategy for either using the heuristic approach (default) that internally switches the cagra build algorithm based on the number of vectors and, most importantly, heuristically choose index build parameters for `IVF_PQ`, or a custom approach where everything, from the cagra build algorithm to its parameters, decided by the API user, is used internally. - Update the value of `DEFAULT_WRITER_THREADS` to 32. Fixes NVIDIA/cuvs-lucene#134 Authors: - Vivek Narang (https://github.com/narangvivek10) Approvers: - MithunR (https://github.com/mythrocks) URL: https://github.com/rapidsai/cuvs-lucene/pull/135 --- .../cuvs/lucene/AcceleratedHNSWParams.java | 139 ++++++++++- .../cuvs/lucene/AcceleratedHNSWUtils.java | 22 -- .../cuvs/lucene/CagraIndexParamsFactory.java | 225 ++++++++++++++++++ .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 8 +- .../nvidia/cuvs/lucene/GPUSearchParams.java | 149 +++++++++++- .../Lucene99AcceleratedHNSWVectorsWriter.java | 10 +- ...ratedHNSWBinaryQuantizedVectorsWriter.java | 9 +- ...ratedHNSWScalarQuantizedVectorsWriter.java | 9 +- .../lucene/TestAcceleratedHNSWParams.java | 35 +++ .../cuvs/lucene/TestGPUSearchParams.java | 35 +++ 10 files changed, 583 insertions(+), 58 deletions(-) create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java index ab387a6a14..dffb500467 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java @@ -6,6 +6,7 @@ package com.nvidia.cuvs.lucene; import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.CagraIndexParams.CuvsDistanceType; import com.nvidia.cuvs.CuVSIvfPqParams; import java.util.Objects; import java.util.concurrent.ExecutorService; @@ -14,6 +15,22 @@ public class AcceleratedHNSWParams { + public static enum Strategy { + /* + * This strategy allows for automatic selection of the underlying CAGRA build algorithm. + * With this strategy we use NN_DESCENT for dataset less than 5M vectors, else we use IVF_PQ. + * Indexing parameters, especially for IVF_PQ, are heuristically identified automatically. + * + * This is the default and the recommended strategy. + */ + HEURISTIC, + /* + * This is an option when the end-user would want to use custom parameter values. + * This strategy should only be used under expert guidance. + */ + CUSTOM + } + /* * TODO: Update boundaries for all parameters when a consensus is reached. * Issue: https://github.com/rapidsai/cuvs-lucene/issues/99 @@ -32,6 +49,8 @@ public class AcceleratedHNSWParams { public static final int MAX_BEAM_WIDTH = 512; public static final int MIN_NUM_MERGE_WORKERS = 1; public static final int MAX_NUM_MERGE_WORKERS = 512; + public static final int MIN_NN_DESCENT_NUM_ITERATIONS = 1; + public static final int MAX_NN_DESCENT_NUM_ITERATIONS = 100; public static final int DEFAULT_WRITER_THREADS = 1; public static final int DEFAULT_INT_GRAPH_DEGREE = 128; @@ -42,6 +61,9 @@ public class AcceleratedHNSWParams { public static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = CagraGraphBuildAlgo.NN_DESCENT; public static final int DEFAULT_NUM_MERGE_WORKERS = 1; + public static final Strategy DEFAULT_STRATEGY = Strategy.HEURISTIC; + public static final CuvsDistanceType DEFAULT_CUVS_DISTANCE_TYPE = CuvsDistanceType.L2Expanded; + public static final int DEFAULT_NN_DESCENT_NUM_ITERATIONS = 20; public static final Supplier DEFAULT_IVF_PQ_PARAMS = () -> { @@ -63,6 +85,9 @@ public class AcceleratedHNSWParams { private final CuVSIvfPqParams cuVSIvfPqParams; private final int numMergeWorkers; private final ExecutorService mergeExec; + private final Strategy strategy; + private final CuvsDistanceType cuvsDistanceType; + private final int nnDescentNumIterations; /** * Constructs an instance of {@link AcceleratedHNSWParams} with specific parameter values. @@ -78,6 +103,9 @@ public class AcceleratedHNSWParams { * @param cuVSIvfPqParams An instance of CuVSIvfPqParams containing IVF_PQ specific parameters. * @param numMergeWorkers The number of merge workers to use with the fallback mechanism. * @param mergeExec The instance of {@link ExecutorService} to use with the fallback mechanism. + * @param strategy either HEURISTIC [Default] that automatically chooses build algorithm and its parameters based on data set size or CUSTOM that uses the parameters passed though this class. + * @param cuvsDistanceType the cuvsDistanceType. The default option is L2Expanded. + * @param nnDescentNumIterations the number of Iterations to run if building with NN_DESCENT. */ private AcceleratedHNSWParams( int writerThreads, @@ -89,7 +117,10 @@ private AcceleratedHNSWParams( CagraGraphBuildAlgo cagraGraphBuildAlgo, CuVSIvfPqParams cuVSIvfPqParams, int numMergeWorkers, - ExecutorService mergeExec) { + ExecutorService mergeExec, + Strategy strategy, + CuvsDistanceType cuvsDistanceType, + int nnDescentNumIterations) { super(); this.writerThreads = writerThreads; this.intermediateGraphDegree = intermediateGraphDegree; @@ -101,6 +132,9 @@ private AcceleratedHNSWParams( this.cuVSIvfPqParams = cuVSIvfPqParams; this.numMergeWorkers = numMergeWorkers; this.mergeExec = mergeExec; + this.strategy = strategy; + this.cuvsDistanceType = cuvsDistanceType; + this.nnDescentNumIterations = nnDescentNumIterations; } /** @@ -193,6 +227,36 @@ public ExecutorService getMergeExec() { return mergeExec; } + /** + * Get the chosen strategy: + * + * When HEURISTIC [Default] is chosen, the CAGRA build algorithm and its indexing parameters are automatically chosen based on the size of the data set + * When CUSTOM is chosen, the build algorithm and its parameters (either defaults or overridden values with the use of With* methods) is used internally + * + * @return get the chosen {@link Strategy} + */ + public Strategy getStrategy() { + return strategy; + } + + /** + * Get the cuvs distance type + * + * @return the distance type + */ + public CuvsDistanceType getCuvsDistanceType() { + return cuvsDistanceType; + } + + /** + * get the number of Iterations to run if building with NN_DESCENT + * + * @return the number of iterations for NN_DESCENT + */ + public int getNNDescentNumIterations() { + return nnDescentNumIterations; + } + @Override public String toString() { return "AcceleratedHNSWParams [writerThreads=" @@ -209,10 +273,18 @@ public String toString() { + beamWidth + ", cagraGraphBuildAlgo=" + cagraGraphBuildAlgo + + ", cuVSIvfPqParams=" + + cuVSIvfPqParams + ", numMergeWorkers=" + numMergeWorkers + ", mergeExec=" + mergeExec + + ", strategy=" + + strategy + + ", cuvsDistanceType=" + + cuvsDistanceType + + ", nnDescentNumIterations=" + + nnDescentNumIterations + "]"; } @@ -231,6 +303,9 @@ public static class Builder { private int numMergeWorkers = DEFAULT_NUM_MERGE_WORKERS; private CuVSIvfPqParams cuVSIvfPqParams = null; private ExecutorService mergeExec = null; + private Strategy strategy = DEFAULT_STRATEGY; + private CuvsDistanceType cuvsDistanceType = DEFAULT_CUVS_DISTANCE_TYPE; + private int nnDescentNumIterations = DEFAULT_NN_DESCENT_NUM_ITERATIONS; /** * Set the number of cuVS writer threads while building the index @@ -357,6 +432,48 @@ public Builder withMergeExecutorService(ExecutorService mergeExec) { return this; } + /** + * Set the chosen strategy: + * + * When HEURISTIC [Default] is chosen, the CAGRA build algorithm and its indexing parameters are automatically chosen based on the size of the data set + * When CUSTOM is chosen, the build algorithm and its parameters (either defaults or overridden values with the use of With* methods) is used internally + * + * Valid options - HEURISTIC, CUSTOM + * Default value - HEURISTIC + * + * @param strategy, the strategy to choose + * @return instance of {@link Builder} + */ + public Builder withStrategy(Strategy strategy) { + this.strategy = strategy; + return this; + } + + /** + * Set the CuvsDistanceType + * + * @param cuvsDistanceType the CuvsDistanceType to set + * @return instance of {@link Builder} + */ + public Builder withCuvsDistanceType(CuvsDistanceType cuvsDistanceType) { + this.cuvsDistanceType = cuvsDistanceType; + return this; + } + + /** + * Set the number of Iterations to run if building with NN_DESCENT + * + * Valid range - Minimum: {@value MIN_NN_DESCENT_NUM_ITERATIONS}, Maximum: {@value MAX_NN_DESCENT_NUM_ITERATIONS} + * Default value - {@value DEFAULT_NN_DESCENT_NUM_ITERATIONS} + * + * @param nnDescentNumIterations number of merge workers to set + * @return instance of {@link Builder} + */ + public Builder withNNDescentNumIterations(int nnDescentNumIterations) { + this.nnDescentNumIterations = nnDescentNumIterations; + return this; + } + /** * Validates the input parameters. * @@ -423,6 +540,21 @@ private void validate() throws IllegalArgumentException { + MAX_NUM_MERGE_WORKERS + "]"); } + if (Objects.isNull(strategy)) { + throw new IllegalArgumentException("strategy cannot be null."); + } + if (Objects.isNull(cuvsDistanceType)) { + throw new IllegalArgumentException("cuvsDistanceType cannot be null."); + } + if (nnDescentNumIterations < MIN_NN_DESCENT_NUM_ITERATIONS + || nnDescentNumIterations > MAX_NN_DESCENT_NUM_ITERATIONS) { + throw new IllegalArgumentException( + "nnDescentNumIterations not in valid range. Valid range: [" + + MIN_NN_DESCENT_NUM_ITERATIONS + + ", " + + MAX_NN_DESCENT_NUM_ITERATIONS + + "]"); + } } /** @@ -448,7 +580,10 @@ public AcceleratedHNSWParams build() { cagraGraphBuildAlgo, cuVSIvfPqParams, numMergeWorkers, - mergeExec); + mergeExec, + strategy, + cuvsDistanceType, + nnDescentNumIterations); } } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java index ac4c94266a..2a8ab02f9d 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java @@ -10,8 +10,6 @@ import com.nvidia.cuvs.CagraIndex; import com.nvidia.cuvs.CagraIndexParams; -import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; -import com.nvidia.cuvs.CuVSIvfPqParams; import com.nvidia.cuvs.CuVSMatrix; import com.nvidia.cuvs.RowView; import java.io.IOException; @@ -399,26 +397,6 @@ public static void writeEmpty(FieldInfo fieldInfo, IndexOutput op) throws IOExce writeMeta(null, op, fieldInfo, 0, 0, 0, null, null, 0); } - /** - * Builds an instance of CagraIndexParams. - * - * @return instance of CagraIndexParams - */ - public static CagraIndexParams cagraIndexParams( - int cuvsWriterThreads, - int intGraphDegree, - int graphDegree, - CagraGraphBuildAlgo cagraGraphBuildAlgo, - CuVSIvfPqParams cuVSIvfPqParams) { - return new CagraIndexParams.Builder() - .withNumWriterThreads(cuvsWriterThreads) - .withIntermediateGraphDegree(intGraphDegree) - .withGraphDegree(graphDegree) - .withCagraGraphBuildAlgo(cagraGraphBuildAlgo) - .withCuVSIvfPqParams(cuVSIvfPqParams) - .build(); - } - /** * Quantizes FLOAT32 vectors to binary (1 bit per dimension, packed into bytes). * Binary quantization: each dimension is compared to a centroid (mean of all values for that dimension). diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java new file mode 100644 index 0000000000..20fed04bc9 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java @@ -0,0 +1,225 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nvidia.cuvs.lucene; + +import com.nvidia.cuvs.CagraIndexParams; +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.CagraIndexParams.CodebookGen; +import com.nvidia.cuvs.CagraIndexParams.CudaDataType; +import com.nvidia.cuvs.CagraIndexParams.CuvsDistanceType; +import com.nvidia.cuvs.CuVSIvfPqIndexParams; +import com.nvidia.cuvs.CuVSIvfPqParams; +import com.nvidia.cuvs.CuVSIvfPqSearchParams; + +/** + * A centralized approach to producing instances of {@link CagraIndexParams} based on the chosen strategy + */ +public class CagraIndexParamsFactory { + + private static final int ALGO_SWITCH_THRESHOLD = 5_000_000; + + /** + * Translation of the internal logic found here: + * https://github.com/rapidsai/cuvs/blob/main/cpp/include/cuvs/neighbors/ivf_pq.hpp#L3385-L3428 + * + * Ideally we should hook into the internal API but this is currently replicated to avoid complications + * in other parts of code base. + */ + private static CuVSIvfPqParams getCuVSIvfPqParams(long rows, long dimension) { + + int pqDim; + int pqBits; + + if (dimension <= 32) { + pqDim = 16; + pqBits = 8; + } else { + pqBits = 4; + if (dimension <= 64) { + pqDim = 32; + } else if (dimension <= 128) { + pqDim = 64; + } else if (dimension <= 192) { + pqDim = 96; + } else { + pqDim = (int) roundUpSafe(dimension / 2, 128); + } + } + + int nLists = (int) Math.max(1, rows / 2000); + final int kmeansNIters = 10; + final double kMinPointsPerCluster = 32; + double minKmeansTrainsetPoints = kMinPointsPerCluster * nLists; + final double maxKmeansTrainsetFraction = 1.0; + double minKmeansTrainsetFraction = + Math.min(maxKmeansTrainsetFraction, minKmeansTrainsetPoints / rows); + double kmeansTrainsetFraction = + Math.clamp( + 1.0 / Math.sqrt(rows * 1e-5), minKmeansTrainsetFraction, maxKmeansTrainsetFraction); + final CodebookGen codebookKind = CodebookGen.PER_SUBSPACE; + int nProbes = (int) Math.round(Math.sqrt(nLists) / 20 + 4); + final int refinementRate = 1; + + CuVSIvfPqIndexParams cuVSIvfPqIndexParams = + new CuVSIvfPqIndexParams.Builder() + .withCodebookKind(codebookKind) + .withKmeansNIters(kmeansNIters) + .withKmeansTrainsetFraction(kmeansTrainsetFraction) + .withNLists(nLists) + .withPqBits(pqBits) + .withPqDim(pqDim) + .withAddDataOnBuild(true) + .withConservativeMemoryAllocation(true) + .build(); + + CuVSIvfPqSearchParams cuVSIvfPqSearchParams = + new CuVSIvfPqSearchParams.Builder() + .withLutDtype(CudaDataType.CUDA_R_16F) + .withInternalDistanceDtype(CudaDataType.CUDA_R_16F) + .withNProbes(nProbes) + .build(); + + CuVSIvfPqParams cuVSIvfPqParams = + new CuVSIvfPqParams.Builder() + .withCuVSIvfPqIndexParams(cuVSIvfPqIndexParams) + .withCuVSIvfPqSearchParams(cuVSIvfPqSearchParams) + .withRefinementRate(refinementRate) + .build(); + + return cuVSIvfPqParams; + } + + /* + * Rough translation from raft's internal utility found here: + * https://github.com/rapidsai/raft/blob/main/cpp/include/raft/util/integer_utils.hpp#L47-L56 + */ + private static long roundUpSafe(long numberToRound, long modulus) { + long remainder = numberToRound % modulus; + if (remainder == 0) { + return numberToRound; + } + long roundedUp = numberToRound - remainder + modulus; + return roundedUp; + } + + private static CagraIndexParams getNNDescentParams( + int graphDegree, + int intGraphDegree, + int writerThreads, + long nnDescentNumIterations, + CuvsDistanceType cuvsDistanceType) { + return new CagraIndexParams.Builder() + .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) + .withGraphDegree(graphDegree) + .withIntermediateGraphDegree(intGraphDegree) + .withNNDescentNumIterations(nnDescentNumIterations) + .withNumWriterThreads(writerThreads) + .withMetric(cuvsDistanceType) + .build(); + } + + private static CagraIndexParams getIVFPQParams( + int graphDegree, + int intGraphDegree, + int writerThreads, + long rows, + long dimension, + CuvsDistanceType cuvsDistanceType) { + return new CagraIndexParams.Builder() + .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.IVF_PQ) + .withCuVSIvfPqParams(getCuVSIvfPqParams(rows, dimension)) + .withNumWriterThreads(writerThreads) + .withIntermediateGraphDegree(intGraphDegree) + .withGraphDegree(graphDegree) + .withMetric(cuvsDistanceType) + .build(); + } + + /** + * Creates an instance of {@link CagraIndexParams} based on the chosen strategy in the {@link GPUSearchParams}. + * + * @param gPUSearchParams an instance of {@link GPUSearchParams} containing input params incoming via the build and search on the GPU API. + * @param rows number of vectors in the data set + * @param dimension the dimension of the vectors in the data set + * @return an instance of {@link CagraIndexParams} + */ + public static CagraIndexParams create( + GPUSearchParams gPUSearchParams, long rows, long dimension) { + if (gPUSearchParams.getStrategy().equals(GPUSearchParams.Strategy.HEURISTIC)) { + if (rows < ALGO_SWITCH_THRESHOLD) { + return getNNDescentParams( + gPUSearchParams.getGraphdegree(), + gPUSearchParams.getIntermediateGraphDegree(), + gPUSearchParams.getWriterThreads(), + gPUSearchParams.getnNDescentNumIterations(), + gPUSearchParams.getCuvsDistanceType()); + } else { + return getIVFPQParams( + gPUSearchParams.getGraphdegree(), + gPUSearchParams.getIntermediateGraphDegree(), + gPUSearchParams.getWriterThreads(), + rows, + dimension, + gPUSearchParams.getCuvsDistanceType()); + } + } else { + return new CagraIndexParams.Builder() + .withNumWriterThreads(gPUSearchParams.getWriterThreads()) + .withIntermediateGraphDegree(gPUSearchParams.getIntermediateGraphDegree()) + .withGraphDegree(gPUSearchParams.getGraphdegree()) + .withCagraGraphBuildAlgo(gPUSearchParams.getCagraGraphBuildAlgo()) + .withCuVSIvfPqParams(gPUSearchParams.getCuVSIvfPqParams()) + .withNNDescentNumIterations(gPUSearchParams.getnNDescentNumIterations()) + .build(); + } + } + + /* + * Ideally there should be just one create method instead of two. + * We should do that when both the input parameter classes can be unified in the future. + */ + + /** + * Creates an instance of {@link CagraIndexParams} based on the chosen strategy in the {@link AcceleratedHNSWParams}. + * + * @param acceleratedHNSWParams an instance of {@link AcceleratedHNSWParams} containing input params incoming via the build and search on the GPU API. + * @param rows number of vectors in the data set + * @param dimension the dimension of the vectors in the data set + * @return an instance of {@link CagraIndexParams} + */ + public static CagraIndexParams create( + AcceleratedHNSWParams acceleratedHNSWParams, long rows, long dimension) { + if (acceleratedHNSWParams.getStrategy().equals(AcceleratedHNSWParams.Strategy.HEURISTIC)) { + if (rows + < ALGO_SWITCH_THRESHOLD) { // TODO: maybe consider making this threshold configurable from + // outside later. + return getNNDescentParams( + acceleratedHNSWParams.getGraphdegree(), + acceleratedHNSWParams.getIntermediateGraphDegree(), + acceleratedHNSWParams.getWriterThreads(), + acceleratedHNSWParams.getNNDescentNumIterations(), + acceleratedHNSWParams.getCuvsDistanceType()); + } else { + return getIVFPQParams( + acceleratedHNSWParams.getGraphdegree(), + acceleratedHNSWParams.getIntermediateGraphDegree(), + acceleratedHNSWParams.getWriterThreads(), + rows, + dimension, + acceleratedHNSWParams.getCuvsDistanceType()); + } + } else { + return new CagraIndexParams.Builder() + .withNumWriterThreads(acceleratedHNSWParams.getWriterThreads()) + .withIntermediateGraphDegree(acceleratedHNSWParams.getIntermediateGraphDegree()) + .withGraphDegree(acceleratedHNSWParams.getGraphdegree()) + .withCagraGraphBuildAlgo(acceleratedHNSWParams.getCagraGraphBuildAlgo()) + .withCuVSIvfPqParams(acceleratedHNSWParams.getCuVSIvfPqParams()) + .withNNDescentNumIterations(acceleratedHNSWParams.getNNDescentNumIterations()) + .build(); + } + } +} diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index 2bb6732cb9..e8fb304ece 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -242,13 +242,7 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro */ private void writeCagraIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { CagraIndexParams params = - new CagraIndexParams.Builder() - .withNumWriterThreads(gpuSearchParams.getWriterThreads()) - .withIntermediateGraphDegree(gpuSearchParams.getIntermediateGraphDegree()) - .withGraphDegree(gpuSearchParams.getGraphdegree()) - .withCagraGraphBuildAlgo(gpuSearchParams.getCagraGraphBuildAlgo()) - .withCuVSIvfPqParams(gpuSearchParams.getCuVSIvfPqParams()) - .build(); + CagraIndexParamsFactory.create(gpuSearchParams, dataset.size(), dataset.columns()); CagraIndex index = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(dataset) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java index cca0233097..cc44526887 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java @@ -6,6 +6,7 @@ package com.nvidia.cuvs.lucene; import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.CagraIndexParams.CuvsDistanceType; import com.nvidia.cuvs.CuVSIvfPqParams; import com.nvidia.cuvs.lucene.CuVS2510GPUVectorsWriter.IndexType; import java.util.Objects; @@ -13,6 +14,22 @@ public class GPUSearchParams { + public static enum Strategy { + /* + * This strategy allows for automatic selection of the underlining CAGRA build algorithm. + * With this strategy we use NN_DESCENT for data set less then 5M vectors else we use IVF_PQ. + * Indexing parameters, especially for IVF_PQ, are heuristically identified automatically. + * + * This is the default and the recommended strategy. + */ + HEURISTIC, + /* + * This is an option when the end-user would want to use custom parameter values. + * This strategy should only be used under expert guidance. + */ + CUSTOM + } + /* * TODO: Update boundaries for all parameters when a consensus is reached. * Issue: https://github.com/rapidsai/cuvs-lucene/issues/99 @@ -23,13 +40,18 @@ public class GPUSearchParams { public static final int MAX_INT_GRAPH_DEG = 512; public static final int MIN_GRAPH_DEG = 1; public static final int MAX_GRAPH_DEG = 512; + public static final int MIN_NN_DESCENT_NUM_ITERATIONS = 1; + public static final int MAX_NN_DESCENT_NUM_ITERATIONS = 100; public static final int DEFAULT_INT_GRAPH_DEGREE = 128; public static final int DEFAULT_GRAPH_DEGREE = 64; public static final CagraGraphBuildAlgo DEFAULT_CAGRA_GRAPH_BUILD_ALGO = CagraGraphBuildAlgo.NN_DESCENT; public static final IndexType DEFAULT_INDEX_TYPE = IndexType.CAGRA; - public static final int DEFAULT_WRITER_THREADS = 1; + public static final int DEFAULT_WRITER_THREADS = 32; + public static final Strategy DEFAULT_STRATEGY = Strategy.HEURISTIC; + public static final CuvsDistanceType DEFAULT_CUVS_DISTANCE_TYPE = CuvsDistanceType.L2Expanded; + public static final int DEFAULT_NN_DESCENT_NUM_ITERATIONS = 20; public static final Supplier DEFAULT_IVF_PQ_PARAMS = () -> { @@ -42,6 +64,9 @@ public class GPUSearchParams { private final CagraGraphBuildAlgo cagraGraphBuildAlgo; private final IndexType indexType; private final CuVSIvfPqParams cuVSIvfPqParams; + private final Strategy strategy; + private final CuvsDistanceType cuvsDistanceType; + private final int nnDescentNumIterations; /** * Constructs an instance of {@link GPUSearchParams} with specific parameter values. @@ -52,6 +77,10 @@ public class GPUSearchParams { * @param cagraGraphBuildAlgo The CAGRA build algorithm to use. * @param indexType The type of index to build - CAGRA, BRUTEFORCE, or both. * @param cuVSIvfPqParams An instance of CuVSIvfPqParams containing IVF_PQ specific parameters. + * @param strategy either HEURISTIC [Default] that automatically chooses build algorithm and its parameters based on data set size or CUSTOM that uses the parameters passed though this class. + * @param heuristicType the heuristic type. The default option is SAME_GRAPH_FOOTPRINT. + * @param cuvsDistanceType the cuvsDistanceType. The default option is L2Expanded. + * @param nnDescentNumIterations the number of Iterations to run if building with NN_DESCENT. */ private GPUSearchParams( int writerThreads, @@ -59,7 +88,10 @@ private GPUSearchParams( int graphdegree, CagraGraphBuildAlgo cagraGraphBuildAlgo, IndexType indexType, - CuVSIvfPqParams cuVSIvfPqParams) { + CuVSIvfPqParams cuVSIvfPqParams, + Strategy strategy, + CuvsDistanceType cuvsDistanceType, + int nnDescentNumIterations) { super(); this.writerThreads = writerThreads; this.intermediateGraphDegree = intermediateGraphDegree; @@ -67,6 +99,9 @@ private GPUSearchParams( this.cagraGraphBuildAlgo = cagraGraphBuildAlgo; this.indexType = indexType; this.cuVSIvfPqParams = cuVSIvfPqParams; + this.strategy = strategy; + this.cuvsDistanceType = cuvsDistanceType; + this.nnDescentNumIterations = nnDescentNumIterations; } /** @@ -123,6 +158,37 @@ public CuVSIvfPqParams getCuVSIvfPqParams() { return cuVSIvfPqParams; } + /** + * Get the chosen strategy: + * + * When HEURISTIC [Default] is chosen, the CAGRA build algorithm and its indexing parameters are automatically chosen based on the size of the data set + * When CUSTOM is chosen, the build algorithm and its parameters (either defaults or overridden values with the use of With* methods) is used internally + * + * + * @return get the chosen {@link Strategy} + */ + public Strategy getStrategy() { + return strategy; + } + + /** + * Get the cuvs distance type + * + * @return the distance type + */ + public CuvsDistanceType getCuvsDistanceType() { + return cuvsDistanceType; + } + + /** + * get the number of Iterations to run if building with NN_DESCENT + * + * @return the number of iterations for NN_DESCENT + */ + public int getnNDescentNumIterations() { + return nnDescentNumIterations; + } + @Override public String toString() { return "GPUSearchParams [writerThreads=" @@ -135,6 +201,14 @@ public String toString() { + cagraGraphBuildAlgo + ", indexType=" + indexType + + ", cuVSIvfPqParams=" + + cuVSIvfPqParams + + ", strategy=" + + strategy + + ", cuvsDistanceType=" + + cuvsDistanceType + + ", nnDescentNumIterations=" + + nnDescentNumIterations + "]"; } @@ -149,11 +223,14 @@ public static class Builder { private CagraGraphBuildAlgo cagraGraphBuildAlgo = DEFAULT_CAGRA_GRAPH_BUILD_ALGO; private IndexType indexType = DEFAULT_INDEX_TYPE; private CuVSIvfPqParams cuVSIvfPqParams = null; + private Strategy strategy = DEFAULT_STRATEGY; + private CuvsDistanceType cuvsDistanceType = DEFAULT_CUVS_DISTANCE_TYPE; + private int nnDescentNumIterations = DEFAULT_NN_DESCENT_NUM_ITERATIONS; /** * Set the number of cuVS writer threads while building the index * Valid range - Minimum: {@value MIN_WRITER_THREADS}, Maximum: {@value MAX_WRITER_THREADS} - * Default value - 64 + * Default value - {@value DEFAULT_WRITER_THREADS} * * @param writerThreads the number of cuVS writer threads * @return instance of {@link Builder} @@ -166,7 +243,7 @@ public Builder withWriterThreads(int writerThreads) { /** * Set the intermediate graph degree to use while building CAGRA index * Valid range - Minimum: {@value MIN_INT_GRAPH_DEG}, Maximum: {@value MAX_INT_GRAPH_DEG} - * Default value - 128 + * Default value - {@value DEFAULT_INT_GRAPH_DEGREE} * * @param intermediateGraphDegree the intermediate graph degree parameter * @return instance of {@link Builder} @@ -179,7 +256,7 @@ public Builder withIntermediateGraphDegree(int intermediateGraphDegree) { /** * Set the graph degree to use while building CAGRA index * Valid range - Minimum: {@value MIN_GRAPH_DEG}, Maximum: {@value MAX_GRAPH_DEG} - * Default value - 64 + * Default value - {@value DEFAULT_GRAPH_DEGREE} * * @param graphDegree the graph degree parameter * @return instance of {@link Builder} @@ -224,6 +301,48 @@ public Builder withCuVSIvfPqParams(CuVSIvfPqParams cuVSIvfPqParams) { return this; } + /** + * Set the chosen strategy: + * + * When HEURISTIC [Default] is chosen, the CAGRA build algorithm and its indexing parameters are automatically chosen based on the size of the data set + * When CUSTOM is chosen, the build algorithm and its parameters (either defaults or overridden values with the use of With* methods) is used internally + * + * Valid options - HEURISTIC, CUSTOM + * Default value - HEURISTIC + * + * @param strategy, the strategy to choose + * @return instance of {@link Builder} + */ + public Builder withStrategy(Strategy strategy) { + this.strategy = strategy; + return this; + } + + /** + * Set the CuvsDistanceType + * + * @param cuvsDistanceType the CuvsDistanceType to set + * @return instance of {@link Builder} + */ + public Builder withCuvsDistanceType(CuvsDistanceType cuvsDistanceType) { + this.cuvsDistanceType = cuvsDistanceType; + return this; + } + + /** + * Set the number of Iterations to run if building with NN_DESCENT + * + * Valid range - Minimum: {@value MIN_NN_DESCENT_NUM_ITERATIONS}, Maximum: {@value MAX_NN_DESCENT_NUM_ITERATIONS} + * Default value - {@value DEFAULT_NN_DESCENT_NUM_ITERATIONS} + * + * @param nnDescentNumIterations number of merge workers to set + * @return instance of {@link Builder} + */ + public Builder withNNDescentNumIterations(int nnDescentNumIterations) { + this.nnDescentNumIterations = nnDescentNumIterations; + return this; + } + /** * Validates the input parameters. * @@ -261,6 +380,21 @@ private void validate() throws IllegalArgumentException { if (Objects.isNull(indexType)) { throw new IllegalArgumentException("indexType cannot be null."); } + if (Objects.isNull(strategy)) { + throw new IllegalArgumentException("strategy cannot be null."); + } + if (Objects.isNull(cuvsDistanceType)) { + throw new IllegalArgumentException("cuvsDistanceType cannot be null."); + } + if (nnDescentNumIterations < MIN_NN_DESCENT_NUM_ITERATIONS + || nnDescentNumIterations > MAX_NN_DESCENT_NUM_ITERATIONS) { + throw new IllegalArgumentException( + "nnDescentNumIterations not in valid range. Valid range: [" + + MIN_NN_DESCENT_NUM_ITERATIONS + + ", " + + MAX_NN_DESCENT_NUM_ITERATIONS + + "]"); + } } /** @@ -279,7 +413,10 @@ public GPUSearchParams build() { graphdegree, cagraGraphBuildAlgo, indexType, - cuVSIvfPqParams); + cuVSIvfPqParams, + strategy, + cuvsDistanceType, + nnDescentNumIterations); } } } diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index bc79dfd3c3..35209ce590 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -4,7 +4,6 @@ */ package com.nvidia.cuvs.lucene; -import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.cagraIndexParams; import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.createMultiLayerHnswGraph; import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.createSingleVectorHnswGraph; import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.printInfoStream; @@ -159,13 +158,10 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro CuVSMatrix dataset = Utils.createFloatMatrix( vectors, fieldInfo.getVectorDimension(), getCuVSResourcesInstance()); + CagraIndexParams params = - cagraIndexParams( - acceleratedHNSWParams.getWriterThreads(), - acceleratedHNSWParams.getIntermediateGraphDegree(), - acceleratedHNSWParams.getGraphdegree(), - acceleratedHNSWParams.getCagraGraphBuildAlgo(), - acceleratedHNSWParams.getCuVSIvfPqParams()); + CagraIndexParamsFactory.create(acceleratedHNSWParams, dataset.size(), dataset.columns()); + CagraIndex cagraIndex = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(dataset) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java index ade0e70270..d8a98cc255 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java @@ -4,7 +4,6 @@ */ package com.nvidia.cuvs.lucene; -import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.cagraIndexParams; import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.createMultiLayerHnswGraph; import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.createSingleVectorHnswGraph; import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.printInfoStream; @@ -165,12 +164,8 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw } CagraIndexParams params = - cagraIndexParams( - acceleratedHNSWParams.getWriterThreads(), - acceleratedHNSWParams.getIntermediateGraphDegree(), - acceleratedHNSWParams.getGraphdegree(), - acceleratedHNSWParams.getCagraGraphBuildAlgo(), - acceleratedHNSWParams.getCuVSIvfPqParams()); + CagraIndexParamsFactory.create(acceleratedHNSWParams, dataset.size(), dataset.columns()); + CagraIndex cagraIndex = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(dataset) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java index 3e3713df91..21b4be3f13 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java @@ -4,7 +4,6 @@ */ package com.nvidia.cuvs.lucene; -import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.cagraIndexParams; import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.createMultiLayerHnswGraph; import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.createSingleVectorHnswGraph; import static com.nvidia.cuvs.lucene.AcceleratedHNSWUtils.printInfoStream; @@ -191,12 +190,8 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOE } CagraIndexParams params = - cagraIndexParams( - acceleratedHNSWParams.getWriterThreads(), - acceleratedHNSWParams.getIntermediateGraphDegree(), - acceleratedHNSWParams.getGraphdegree(), - acceleratedHNSWParams.getCagraGraphBuildAlgo(), - acceleratedHNSWParams.getCuVSIvfPqParams()); + CagraIndexParamsFactory.create(acceleratedHNSWParams, dataset.size(), dataset.columns()); + CagraIndex cagraIndex = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(dataset) diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java index 66a657e6e3..2273af8c9d 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java @@ -7,17 +7,21 @@ import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_BEAM_WIDTH; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_CAGRA_GRAPH_BUILD_ALGO; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_CUVS_DISTANCE_TYPE; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_GRAPH_DEGREE; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_HNSW_LAYERS; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_INT_GRAPH_DEGREE; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_MAX_CONN; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_NN_DESCENT_NUM_ITERATIONS; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_NUM_MERGE_WORKERS; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_STRATEGY; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.DEFAULT_WRITER_THREADS; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_BEAM_WIDTH; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_GRAPH_DEG; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_HNSW_LAYERS; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_INT_GRAPH_DEG; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_MAX_CONN; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_NN_DESCENT_NUM_ITERATIONS; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_NUM_MERGE_WORKERS; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MAX_WRITER_THREADS; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_BEAM_WIDTH; @@ -25,6 +29,7 @@ import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_HNSW_LAYERS; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_INT_GRAPH_DEG; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_MAX_CONN; +import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_NN_DESCENT_NUM_ITERATIONS; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_NUM_MERGE_WORKERS; import static com.nvidia.cuvs.lucene.AcceleratedHNSWParams.MIN_WRITER_THREADS; import static java.lang.Integer.MAX_VALUE; @@ -56,6 +61,9 @@ public void testAcceleratedHNSWParamsDefaultValues() { assertEquals(DEFAULT_WRITER_THREADS, params.getWriterThreads()); assertEquals(DEFAULT_NUM_MERGE_WORKERS, params.getNumMergeWorkers()); assertEquals(DEFAULT_CAGRA_GRAPH_BUILD_ALGO, params.getCagraGraphBuildAlgo()); + assertEquals(DEFAULT_STRATEGY, params.getStrategy()); + assertEquals(DEFAULT_CUVS_DISTANCE_TYPE, params.getCuvsDistanceType()); + assertEquals(DEFAULT_NN_DESCENT_NUM_ITERATIONS, params.getNNDescentNumIterations()); } @Test @@ -152,6 +160,33 @@ public void testAcceleratedHNSWParamsInvalidCagraGraphBuildAlgo() { () -> new AcceleratedHNSWParams.Builder().withCagraGraphBuildAlgo(null).build()); } + @Test + public void testAcceleratedHNSWParamsInvalidStrategy() { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withStrategy(null).build()); + } + + @Test + public void testAcceleratedHNSWParamsInvalidCuvsDistanceType() { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withCuvsDistanceType(null).build()); + } + + @Test + public void testAcceleratedHNSWParamsInvalidNNDescentNumIterations() { + for (int v : + new int[] { + random.nextInt(MIN_VALUE, (int) MIN_NN_DESCENT_NUM_ITERATIONS), + random.nextInt((int) (MAX_NN_DESCENT_NUM_ITERATIONS + 1), MAX_VALUE) + }) { + assertThrows( + IllegalArgumentException.class, + () -> new AcceleratedHNSWParams.Builder().withNNDescentNumIterations(v).build()); + } + } + @BeforeClass public static void beforeClass() { random = random(); diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java index 495033172f..483b689c22 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java @@ -6,15 +6,20 @@ package com.nvidia.cuvs.lucene; import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_CAGRA_GRAPH_BUILD_ALGO; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_CUVS_DISTANCE_TYPE; import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_GRAPH_DEGREE; import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_INDEX_TYPE; import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_INT_GRAPH_DEGREE; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_NN_DESCENT_NUM_ITERATIONS; +import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_STRATEGY; import static com.nvidia.cuvs.lucene.GPUSearchParams.DEFAULT_WRITER_THREADS; import static com.nvidia.cuvs.lucene.GPUSearchParams.MAX_GRAPH_DEG; import static com.nvidia.cuvs.lucene.GPUSearchParams.MAX_INT_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MAX_NN_DESCENT_NUM_ITERATIONS; import static com.nvidia.cuvs.lucene.GPUSearchParams.MAX_WRITER_THREADS; import static com.nvidia.cuvs.lucene.GPUSearchParams.MIN_GRAPH_DEG; import static com.nvidia.cuvs.lucene.GPUSearchParams.MIN_INT_GRAPH_DEG; +import static com.nvidia.cuvs.lucene.GPUSearchParams.MIN_NN_DESCENT_NUM_ITERATIONS; import static com.nvidia.cuvs.lucene.GPUSearchParams.MIN_WRITER_THREADS; import static java.lang.Integer.MAX_VALUE; import static java.lang.Integer.MIN_VALUE; @@ -42,6 +47,9 @@ public void testGPUSearchParamsDefaultValues() { assertEquals(DEFAULT_WRITER_THREADS, params.getWriterThreads()); assertEquals(DEFAULT_CAGRA_GRAPH_BUILD_ALGO, params.getCagraGraphBuildAlgo()); assertEquals(DEFAULT_INDEX_TYPE, params.getIndexType()); + assertEquals(DEFAULT_STRATEGY, params.getStrategy()); + assertEquals(DEFAULT_CUVS_DISTANCE_TYPE, params.getCuvsDistanceType()); + assertEquals(DEFAULT_NN_DESCENT_NUM_ITERATIONS, params.getnNDescentNumIterations()); } @Test @@ -96,6 +104,33 @@ public void testGPUSearchParamsInvalidIndexType() { () -> new GPUSearchParams.Builder().withIndexType(null).build()); } + @Test + public void testGPUSearchParamsInvalidStrategy() { + assertThrows( + IllegalArgumentException.class, + () -> new GPUSearchParams.Builder().withStrategy(null).build()); + } + + @Test + public void testGPUSearchParamsInvalidCuvsDistanceType() { + assertThrows( + IllegalArgumentException.class, + () -> new GPUSearchParams.Builder().withCuvsDistanceType(null).build()); + } + + @Test + public void testGPUSearchParamsInvalidNNDescentNumIterations() { + for (int v : + new int[] { + random.nextInt(MIN_VALUE, (int) MIN_NN_DESCENT_NUM_ITERATIONS), + random.nextInt((int) (MAX_NN_DESCENT_NUM_ITERATIONS + 1), MAX_VALUE) + }) { + assertThrows( + IllegalArgumentException.class, + () -> new GPUSearchParams.Builder().withNNDescentNumIterations(v).build()); + } + } + @BeforeClass public static void beforeClass() { random = random(); From 8426eda3071300ec8237ce61680648e9496084fb Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 12 May 2026 09:13:17 -0500 Subject: [PATCH 079/105] Build and test with CUDA 13.2.0 (NVIDIA/cuvs-lucene#144) Contributes to https://github.com/rapidsai/build-planning/issues/265 * uses CUDA 13.2.0 to build and test * updates to CUDA 13.2.0 devcontainers ## Notes for Reviewers This switches GitHub Actions workflows to the `cuda-13.2.0` branch from here: https://github.com/rapidsai/shared-workflows/pull/545 A future round of PRs will revert that back to `main`, once all of RAPIDS is migrated. --- java/cuvs-lucene/.github/workflows/build.yaml | 4 ++-- java/cuvs-lucene/.github/workflows/pr.yaml | 10 +++++----- java/cuvs-lucene/.github/workflows/test.yaml | 4 ++-- ...rch-aarch64.yaml => all_cuda-132_arch-aarch64.yaml} | 4 ++-- ..._arch-x86_64.yaml => all_cuda-132_arch-x86_64.yaml} | 4 ++-- java/cuvs-lucene/dependencies.yaml | 6 +++++- java/cuvs-lucene/examples/README.md | 2 +- 7 files changed, 19 insertions(+), 15 deletions(-) rename java/cuvs-lucene/conda/environments/{all_cuda-131_arch-aarch64.yaml => all_cuda-132_arch-aarch64.yaml} (90%) rename java/cuvs-lucene/conda/environments/{all_cuda-131_arch-x86_64.yaml => all_cuda-132_arch-x86_64.yaml} (90%) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 7a02427333..bc2fbeacc8 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -35,7 +35,7 @@ concurrency: jobs: java-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: @@ -43,7 +43,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.1.1' + - '13.2.0' with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index acfed0ce7d..6f5f7ba612 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda-13.2.0 if: always() with: needs: ${{ toJSON(needs) }} @@ -47,7 +47,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@cuda-13.2.0 with: files_yaml: | test_java: @@ -57,14 +57,14 @@ jobs: checks: needs: telemetry-setup secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda-13.2.0 with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" conda-java-tests: needs: [changed-files, checks] secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. @@ -73,7 +73,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.1.1' + - '13.2.0' with: build_type: pull-request node_type: "gpu-l4-latest-1" diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 07914dfc88..3e1c11785c 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -25,13 +25,13 @@ on: jobs: conda-java-tests: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 strategy: fail-fast: false matrix: cuda_version: - '12.9.1' - - '13.1.1' + - '13.2.0' with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} diff --git a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-132_arch-aarch64.yaml similarity index 90% rename from java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml rename to java/cuvs-lucene/conda/environments/all_cuda-132_arch-aarch64.yaml index 00444baa49..7b8f2c63ac 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-132_arch-aarch64.yaml @@ -10,7 +10,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-version=13.1 +- cuda-version=13.2 - cxx-compiler - gcc_linux-aarch64=14.* - libcublas-dev @@ -22,4 +22,4 @@ dependencies: - ninja - openjdk=22.* - sysroot_linux-aarch64==2.28 -name: all_cuda-131_arch-aarch64 +name: all_cuda-132_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-132_arch-x86_64.yaml similarity index 90% rename from java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml rename to java/cuvs-lucene/conda/environments/all_cuda-132_arch-x86_64.yaml index d1a66ae0e4..6699469660 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-131_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-132_arch-x86_64.yaml @@ -10,7 +10,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-version=13.1 +- cuda-version=13.2 - cxx-compiler - gcc_linux-64=14.* - libcublas-dev @@ -22,4 +22,4 @@ dependencies: - ninja - openjdk=22.* - sysroot_linux-64==2.28 -name: all_cuda-131_arch-x86_64 +name: all_cuda-132_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index 05922a300f..d131afd98e 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -6,7 +6,7 @@ files: all: output: conda matrix: - cuda: ["12.9", "13.1"] + cuda: ["12.9", "13.2"] arch: [x86_64, aarch64] includes: - cuda @@ -63,6 +63,10 @@ dependencies: cuda: "13.1" packages: - cuda-version=13.1 + - matrix: + cuda: "13.2" + packages: + - cuda-version=13.2 cuda: common: - output_types: [conda] diff --git a/java/cuvs-lucene/examples/README.md b/java/cuvs-lucene/examples/README.md index ad4caf9617..9480a969bf 100644 --- a/java/cuvs-lucene/examples/README.md +++ b/java/cuvs-lucene/examples/README.md @@ -16,7 +16,7 @@ cd .. Then do: ```sh -docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.06-cuda13.1.1-ubuntu24.04-py3.13 +docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.06-cuda13.2.0-ubuntu24.04-py3.13 ``` Inside the docker container (and in the `cuvs-lucene's` root directory) do: From 982f4452f34afd6d712071ea6af1d28ada11e01e Mon Sep 17 00:00:00 2001 From: Gil Forsyth Date: Thu, 14 May 2026 19:26:00 -0400 Subject: [PATCH 080/105] fix(ci): resolve all zizmor findings and add zizmor pre-commit checks (NVIDIA/cuvs-lucene#146) Similar to upstream changes in `shared-workflows`, this PR cleans up and annotates all of the workflows and adds the `zizmor` linter to make sure changes are checked. Part of https://github.com/rapidsai/build-planning/issues/275 Authors: - Gil Forsyth (https://github.com/gforsyth) Approvers: - https://github.com/jakirkham URL: https://github.com/rapidsai/cuvs-lucene/pull/146 --- java/cuvs-lucene/.github/workflows/build.yaml | 12 ++++-- java/cuvs-lucene/.github/workflows/pr.yaml | 37 ++++++++++++++++--- java/cuvs-lucene/.github/workflows/test.yaml | 11 ++++-- java/cuvs-lucene/.github/zizmor.yml | 9 +++++ java/cuvs-lucene/.pre-commit-config.yaml | 4 ++ 5 files changed, 61 insertions(+), 12 deletions(-) create mode 100644 java/cuvs-lucene/.github/zizmor.yml diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index bc2fbeacc8..b3e468458b 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -1,5 +1,4 @@ name: build - on: push: branches: @@ -27,14 +26,13 @@ on: description: "build_type: one of [branch, nightly, pull-request]" type: string default: nightly - concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} cancel-in-progress: true - +permissions: {} jobs: java-build: - secrets: inherit + secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. @@ -54,3 +52,9 @@ jobs: file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" sha: ${{ inputs.sha }} + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 6f5f7ba612..14c22be570 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -6,6 +6,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +permissions: {} jobs: pr-builder: needs: @@ -14,11 +15,17 @@ jobs: - checks - conda-java-tests - telemetry-setup - secrets: inherit + secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda-13.2.0 if: always() with: needs: ${{ toJSON(needs) }} + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read telemetry-setup: continue-on-error: true runs-on: ubuntu-latest @@ -38,7 +45,7 @@ jobs: steps: - name: Get PR Info id: get-pr-info - uses: nv-gha-runners/get-pr-info@main + uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf # main - name: Check if nightly CI is passing uses: rapidsai/shared-actions/check_nightly_success/dispatch@main with: @@ -46,7 +53,7 @@ jobs: target-branch: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.ref }} changed-files: needs: telemetry-setup - secrets: inherit + secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@cuda-13.2.0 with: files_yaml: | @@ -54,16 +61,28 @@ jobs: - '**' - '!.pre-commit-config.yaml' - '!README.md' + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read checks: needs: telemetry-setup - secrets: inherit + secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda-13.2.0 with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read conda-java-tests: needs: [changed-files, checks] - secrets: inherit + secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. @@ -82,6 +101,12 @@ jobs: script: "ci/test_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read telemetry-summarize: # This job must use a self-hosted runner to record telemetry traces. runs-on: linux-amd64-cpu4 @@ -93,3 +118,5 @@ jobs: uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main env: GH_TOKEN: ${{ github.token }} + permissions: + actions: write diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 3e1c11785c..6b6b62b249 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -1,5 +1,4 @@ name: test - on: workflow_dispatch: inputs: @@ -21,10 +20,10 @@ on: description: "build_type: one of [branch, nightly, pull-request]" type: string default: nightly - +permissions: {} jobs: conda-java-tests: - secrets: inherit + secrets: inherit # zizmor: ignore[secrets-inherit] uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 strategy: fail-fast: false @@ -41,3 +40,9 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:26.06-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read diff --git a/java/cuvs-lucene/.github/zizmor.yml b/java/cuvs-lucene/.github/zizmor.yml new file mode 100644 index 0000000000..1b6ea1e53f --- /dev/null +++ b/java/cuvs-lucene/.github/zizmor.yml @@ -0,0 +1,9 @@ +rules: + unpinned-uses: + config: + policies: + # We require SHA-pinning for all workflows and actions _except_ for those from + # rapidsai/shared-workflows and rapidsai/shared-actions + "rapidsai/shared-workflows/*": any + "rapidsai/shared-actions/*": any + "*": hash-pin diff --git a/java/cuvs-lucene/.pre-commit-config.yaml b/java/cuvs-lucene/.pre-commit-config.yaml index ed7392ef09..3ef7e2910a 100644 --- a/java/cuvs-lucene/.pre-commit-config.yaml +++ b/java/cuvs-lucene/.pre-commit-config.yaml @@ -32,3 +32,7 @@ repos: rev: v0.11.0.1 hooks: - id: shellcheck + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: v1.24.1 + hooks: + - id: zizmor From 5971bfe55621ea3bb71ad3a0abd85811933becdf Mon Sep 17 00:00:00 2001 From: jolorunyomi Date: Fri, 15 May 2026 09:45:13 -0500 Subject: [PATCH 081/105] Prepare release/26.06 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 8 ++++---- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/RAPIDS_BRANCH | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index b3e468458b..c3f6c9b3de 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -33,7 +33,7 @@ permissions: {} jobs: java-build: secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.06 # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 14c22be570..57a5a2b10c 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -16,7 +16,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda-13.2.0 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.06 if: always() with: needs: ${{ toJSON(needs) }} @@ -54,7 +54,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@cuda-13.2.0 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@release/26.06 with: files_yaml: | test_java: @@ -70,7 +70,7 @@ jobs: checks: needs: telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda-13.2.0 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@release/26.06 with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" @@ -83,7 +83,7 @@ jobs: conda-java-tests: needs: [changed-files, checks] secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.06 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 6b6b62b249..c9c7066294 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -24,7 +24,7 @@ permissions: {} jobs: conda-java-tests: secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.06 strategy: fail-fast: false matrix: diff --git a/java/cuvs-lucene/RAPIDS_BRANCH b/java/cuvs-lucene/RAPIDS_BRANCH index ba2906d066..5a399d5c50 100644 --- a/java/cuvs-lucene/RAPIDS_BRANCH +++ b/java/cuvs-lucene/RAPIDS_BRANCH @@ -1 +1 @@ -main +release/26.06 From 05e9a4c350401139608323114e4f541b4911474c Mon Sep 17 00:00:00 2001 From: jolorunyomi Date: Fri, 15 May 2026 09:45:14 -0500 Subject: [PATCH 082/105] Revert "Prepare release/26.06" This reverts commit 5971bfe55621ea3bb71ad3a0abd85811933becdf. --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 8 ++++---- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/RAPIDS_BRANCH | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index c3f6c9b3de..b3e468458b 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -33,7 +33,7 @@ permissions: {} jobs: java-build: secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.06 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 57a5a2b10c..14c22be570 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -16,7 +16,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.06 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda-13.2.0 if: always() with: needs: ${{ toJSON(needs) }} @@ -54,7 +54,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@release/26.06 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@cuda-13.2.0 with: files_yaml: | test_java: @@ -70,7 +70,7 @@ jobs: checks: needs: telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@release/26.06 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda-13.2.0 with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" @@ -83,7 +83,7 @@ jobs: conda-java-tests: needs: [changed-files, checks] secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.06 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index c9c7066294..6b6b62b249 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -24,7 +24,7 @@ permissions: {} jobs: conda-java-tests: secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.06 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 strategy: fail-fast: false matrix: diff --git a/java/cuvs-lucene/RAPIDS_BRANCH b/java/cuvs-lucene/RAPIDS_BRANCH index 5a399d5c50..ba2906d066 100644 --- a/java/cuvs-lucene/RAPIDS_BRANCH +++ b/java/cuvs-lucene/RAPIDS_BRANCH @@ -1 +1 @@ -release/26.06 +main From dd3f051c0cfe51ed60e73b0782a0f1d23fec123d Mon Sep 17 00:00:00 2001 From: jolorunyomi Date: Fri, 15 May 2026 09:46:48 -0500 Subject: [PATCH 083/105] Update to 26.08.00 --- java/cuvs-lucene/.github/workflows/build.yaml | 4 ++-- java/cuvs-lucene/.github/workflows/pr.yaml | 10 +++++----- java/cuvs-lucene/.github/workflows/test.yaml | 4 ++-- java/cuvs-lucene/VERSION | 2 +- java/cuvs-lucene/bench/pom.xml | 2 +- java/cuvs-lucene/build.sh | 2 +- .../conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- .../conda/environments/all_cuda-132_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-132_arch-x86_64.yaml | 2 +- java/cuvs-lucene/dependencies.yaml | 2 +- java/cuvs-lucene/examples/README.md | 6 +++--- java/cuvs-lucene/examples/pom.xml | 4 ++-- java/cuvs-lucene/pom.xml | 4 ++-- 14 files changed, 24 insertions(+), 24 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index b3e468458b..1fa894e2d3 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -33,7 +33,7 @@ permissions: {} jobs: java-build: secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: @@ -47,7 +47,7 @@ jobs: branch: ${{ inputs.branch }} arch: "amd64" date: ${{ inputs.date }} - container_image: "rapidsai/ci-conda:26.06-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.08-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/build_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 14c22be570..1f7e40c3b0 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -16,7 +16,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@cuda-13.2.0 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main if: always() with: needs: ${{ toJSON(needs) }} @@ -54,7 +54,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@cuda-13.2.0 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main with: files_yaml: | test_java: @@ -70,7 +70,7 @@ jobs: checks: needs: telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@cuda-13.2.0 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" @@ -83,7 +83,7 @@ jobs: conda-java-tests: needs: [changed-files, checks] secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. @@ -97,7 +97,7 @@ jobs: build_type: pull-request node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:26.06-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.08-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 6b6b62b249..7f49c398d4 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -24,7 +24,7 @@ permissions: {} jobs: conda-java-tests: secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@cuda-13.2.0 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main strategy: fail-fast: false matrix: @@ -38,7 +38,7 @@ jobs: sha: ${{ inputs.sha }} node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:26.06-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.08-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" permissions: actions: read diff --git a/java/cuvs-lucene/VERSION b/java/cuvs-lucene/VERSION index cdb610a24d..4e6864b4ca 100644 --- a/java/cuvs-lucene/VERSION +++ b/java/cuvs-lucene/VERSION @@ -1 +1 @@ -26.06.00 +26.08.00 diff --git a/java/cuvs-lucene/bench/pom.xml b/java/cuvs-lucene/bench/pom.xml index 8f7262afe7..de46107481 100644 --- a/java/cuvs-lucene/bench/pom.xml +++ b/java/cuvs-lucene/bench/pom.xml @@ -31,7 +31,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 26.06.0 + 26.08.0 commons-io diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index f3a7998142..4ec8c0870e 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -8,7 +8,7 @@ set -e -u -o pipefail ARGS="$*" NUMARGS=$# -VERSION="26.06.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked +VERSION="26.08.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked GROUP_ID="com.nvidia.cuvs.lucene" function hasArg { diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index e9953801d8..caf4537fec 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -17,7 +17,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.06.* +- libcuvs==26.08.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index 12fad5b4f9..761a1a12dd 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -17,7 +17,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.06.* +- libcuvs==26.08.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/conda/environments/all_cuda-132_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-132_arch-aarch64.yaml index 7b8f2c63ac..2cc700a096 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-132_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-132_arch-aarch64.yaml @@ -17,7 +17,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.06.* +- libcuvs==26.08.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/conda/environments/all_cuda-132_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-132_arch-x86_64.yaml index 6699469660..76ff4538d0 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-132_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-132_arch-x86_64.yaml @@ -17,7 +17,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.06.* +- libcuvs==26.08.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index d131afd98e..a94ab383cc 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -78,7 +78,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev - - libcuvs==26.06.* + - libcuvs==26.08.* java: common: - output_types: conda diff --git a/java/cuvs-lucene/examples/README.md b/java/cuvs-lucene/examples/README.md index 9480a969bf..26aa83b278 100644 --- a/java/cuvs-lucene/examples/README.md +++ b/java/cuvs-lucene/examples/README.md @@ -16,7 +16,7 @@ cd .. Then do: ```sh -docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.06-cuda13.2.0-ubuntu24.04-py3.13 +docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.08-cuda13.2.0-ubuntu24.04-py3.13 ``` Inside the docker container (and in the `cuvs-lucene's` root directory) do: @@ -26,10 +26,10 @@ Inside the docker container (and in the `cuvs-lucene's` root directory) do: To run Accelerated HNSW example do: ```sh -mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.06.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.AcceleratedHnswExample +mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.08.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.AcceleratedHnswExample ``` To run the Index and Search on GPU example do: ```sh -mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.06.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.IndexAndSearchonGPUExample +mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.08.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.IndexAndSearchonGPUExample ``` diff --git a/java/cuvs-lucene/examples/pom.xml b/java/cuvs-lucene/examples/pom.xml index 193374853b..1bc5cdd49d 100644 --- a/java/cuvs-lucene/examples/pom.xml +++ b/java/cuvs-lucene/examples/pom.xml @@ -11,7 +11,7 @@ com.nvidia.cuvs.lucene.examples examples - 26.06.0 + 26.08.0 examples @@ -51,7 +51,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 26.06.0 + 26.08.0 diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 19479f10f3..e13e5d2b6a 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -12,7 +12,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 26.06.0 + 26.08.0 cuvs-lucene jar @@ -75,7 +75,7 @@ com.nvidia.cuvs cuvs-java - 26.06.0 + 26.08.0 From 39910bf650573a17287eb34561b151830df3902c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 29 May 2026 09:03:00 -0500 Subject: [PATCH 084/105] Add SECURITY.md (NVIDIA/cuvs-lucene#154) ## Description Contributes to https://github.com/rapidsai/build-planning/issues/281 * adds a `SECURITY.md` describing how to report security vulnerabilities ## Notes for Reviewers ### Why not just set this org-wide? An org-wide default is set at https://github.com/rapidsai/.github/blob/main/SECURITY.md, but adding an actual file in each repo offers a few benefits: * ensures security policy travels with the repo to forks, clones, mirrors, etc. * allows per-repo governance over the security policy (via PR review, CODEOWNERS, etc.) --- java/cuvs-lucene/.github/CODEOWNERS | 3 ++ java/cuvs-lucene/.github/workflows/pr.yaml | 1 + java/cuvs-lucene/SECURITY.md | 40 ++++++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 java/cuvs-lucene/SECURITY.md diff --git a/java/cuvs-lucene/.github/CODEOWNERS b/java/cuvs-lucene/.github/CODEOWNERS index 4b8b023cb7..ea66ecb2c8 100644 --- a/java/cuvs-lucene/.github/CODEOWNERS +++ b/java/cuvs-lucene/.github/CODEOWNERS @@ -18,3 +18,6 @@ build.sh @rapidsai/packaging-codeowners dependencies.yaml @rapidsai/packaging-codeowners pom.xml @rapidsai/cuvs-build-codeowners /.pre-commit-config.yaml @rapidsai/packaging-codeowners + +# Ops code owners +/SECURITY.md @rapidsai/ops-codeowners diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 1f7e40c3b0..1f9b336281 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -61,6 +61,7 @@ jobs: - '**' - '!.pre-commit-config.yaml' - '!README.md' + - '!SECURITY.md' permissions: actions: read contents: read diff --git a/java/cuvs-lucene/SECURITY.md b/java/cuvs-lucene/SECURITY.md new file mode 100644 index 0000000000..ada890837b --- /dev/null +++ b/java/cuvs-lucene/SECURITY.md @@ -0,0 +1,40 @@ +# Security + +## Reporting Security Issues + +> [!WARNING] +> Do not report security vulnerabilities through public GitHub issues! + +Instead, please submit a private vulnerability report, see below. + +## Reporting a Vulnerability + +1. **NVIDIA Vulnerability Disclosure Program (preferred)** + Submit through the NVIDIA Product Security Incident Response Team (PSIRT) web form () + This is the fastest path to triage and tracking. + +2. **Email NVIDIA PSIRT** + `psirt@nvidia.com` — encrypt sensitive reports with the + [NVIDIA PSIRT PGP key](https://www.nvidia.com/en-us/security/pgp-key). + +3. **GitHub Private Vulnerability Reporting** + Use the **Security and quality** tab on this repository → *Report a vulnerability*. + +## Report Details + +We prefer all communications to be in English. + +Reports should include the following: + +* reproducible example showing how the vulnerability can be exploited +* statement about the impact (including affected versions) + +And we'd appreciate if they also include: + +* statement about whether you are interested in implementing the fix yourself + +## Disclosure Policy + +NVIDIA PSIRT will acknowledge receipt and coordinate triage, fix development, and coordinated disclosure. + +More on NVIDIA's response process: . From 3b5a08485c5e6a4ae16dd24893375ef4d9de9dce Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 25 Jun 2026 19:20:31 -0500 Subject: [PATCH 085/105] use GCS mirror for Maven, fix benchmarks version, other small changes (NVIDIA/cuvs-lucene#166) Fixes NVIDIA/cuvs-lucene#145 Contributes to https://github.com/rapidsai/build-planning/issues/297 The root cause of NVIDIA/cuvs-lucene#145 appears to be that we were getting rate-limited by Maven Central. Similar to https://github.com/NVIDIA/cuvs/pull/2253, this proposes fixing that by using the same read-only Maven mirror that Apache Orc, Lucene, Spark and others use in their builds (https://storage-download.googleapis.com/maven-central/index.html). Other changes: * fixes microbenchmarks version (was still 26.02 because it used the cuVS pattern for version replacement) * adding default `mvn` options to request packages more slowly and wait longer between retries * updating all `pre-commit` hooks w/ `pre-commit autoupdate` * adding `check-xml` hook to validate that `pom.xml` are valid XML docs * reformatting READMEs per https://yihui.org/en/2021/06/markdown-breath/ Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Corey J. Nolet (https://github.com/cjnolet) - https://github.com/jakirkham - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cuvs-lucene/pull/166 --- java/cuvs-lucene/.mvn/maven.config | 6 +++ java/cuvs-lucene/.pre-commit-config.yaml | 9 ++-- java/cuvs-lucene/README.md | 3 ++ java/cuvs-lucene/bench/pom.xml | 56 ++++++++++++++++++++++- java/cuvs-lucene/examples/README.md | 6 +++ java/cuvs-lucene/examples/pom.xml | 57 ++++++++++++++++++++++-- java/cuvs-lucene/pom.xml | 54 ++++++++++++++++++++++ 7 files changed, 182 insertions(+), 9 deletions(-) create mode 100644 java/cuvs-lucene/.mvn/maven.config diff --git a/java/cuvs-lucene/.mvn/maven.config b/java/cuvs-lucene/.mvn/maven.config new file mode 100644 index 0000000000..c83cb17ff4 --- /dev/null +++ b/java/cuvs-lucene/.mvn/maven.config @@ -0,0 +1,6 @@ +-e +-B +-Daether.connector.basic.downstreamThreads=1 +-Daether.transport.http.retryHandler.count=5 +-Daether.transport.http.retryHandler.interval=10000 +-Dmaven.wagon.http.retryHandler.count=5 diff --git a/java/cuvs-lucene/.pre-commit-config.yaml b/java/cuvs-lucene/.pre-commit-config.yaml index 3ef7e2910a..a40771e556 100644 --- a/java/cuvs-lucene/.pre-commit-config.yaml +++ b/java/cuvs-lucene/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 repos: @@ -9,8 +9,9 @@ repos: - id: trailing-whitespace - id: end-of-file-fixer - id: check-symlinks + - id: check-xml - repo: https://github.com/rapidsai/pre-commit-hooks - rev: v1.4.2 + rev: v1.5.1 hooks: - id: verify-copyright name: verify-copyright @@ -24,7 +25,7 @@ repos: dependencies[.]yaml$| ^[.]pre-commit-config[.]yaml$ - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.20.2 + rev: v1.21.0 hooks: - id: rapids-dependency-file-generator args: ["--clean", "--warn-all", "--strict"] @@ -33,6 +34,6 @@ repos: hooks: - id: shellcheck - repo: https://github.com/zizmorcore/zizmor-pre-commit - rev: v1.24.1 + rev: v1.26.1 hooks: - id: zizmor diff --git a/java/cuvs-lucene/README.md b/java/cuvs-lucene/README.md index d88d3401a9..7e177c457a 100644 --- a/java/cuvs-lucene/README.md +++ b/java/cuvs-lucene/README.md @@ -9,6 +9,7 @@ This library provides a new [KnnVectorFormat](https://lucene.apache.org/core/10_ ## Building ### Prerequisites + - [CUDA 12.0+](https://developer.nvidia.com/cuda-toolkit-archive), - [Maven 3.9.6+](https://maven.apache.org/download.cgi), - [JDK 22](https://jdk.java.net/archive/) @@ -16,9 +17,11 @@ This library provides a new [KnnVectorFormat](https://lucene.apache.org/core/10_ ```sh mvn clean compile package ``` + The artifacts would be built and available in the target / folder. ### Running Tests + ```sh export LD_LIBRARY_PATH={ PATH TO YOUR LOCAL libcuvs_c.so }:$LD_LIBRARY_PATH && mvn clean test ``` diff --git a/java/cuvs-lucene/bench/pom.xml b/java/cuvs-lucene/bench/pom.xml index de46107481..04ea66c581 100644 --- a/java/cuvs-lucene/bench/pom.xml +++ b/java/cuvs-lucene/bench/pom.xml @@ -11,11 +11,65 @@ com.nvidia.cuvs.lucene.benchmarks cuvs-lucene-benchmarks - 26.02.0 + 26.08.0 jar cuvs-lucene-benchmarks + + + + gcs-maven-central-mirror + GCS Maven Central mirror + https://maven-central.storage-download.googleapis.com/maven2/ + + true + + + false + + + + + central + Maven Central + https://repo.maven.apache.org/maven2 + + true + + + false + + + + + + + + gcs-maven-central-mirror + GCS Maven Central mirror + https://maven-central.storage-download.googleapis.com/maven2/ + + true + + + false + + + + + central + Maven Plugin Repository + https://repo.maven.apache.org/maven2 + + true + + + false + + + + org.openjdk.jmh diff --git a/java/cuvs-lucene/examples/README.md b/java/cuvs-lucene/examples/README.md index 26aa83b278..1c3876b325 100644 --- a/java/cuvs-lucene/examples/README.md +++ b/java/cuvs-lucene/examples/README.md @@ -3,6 +3,7 @@ This maven project contains basic examples that showcase how `cuvs-lucene` can be used. ## Prerequisites + - [Docker](https://www.docker.com/) - [Nvidia Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) - A machine with an Nvidia GPU @@ -10,26 +11,31 @@ This maven project contains basic examples that showcase how `cuvs-lucene` can b ## Steps If you are currently in this directory (and to be in the `cuvs-lucene's` root directory) do: + ```sh cd .. ``` Then do: + ```sh docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.08-cuda13.2.0-ubuntu24.04-py3.13 ``` Inside the docker container (and in the `cuvs-lucene's` root directory) do: + ```sh ./ci/build_java.sh && conda activate java && cd examples ``` To run Accelerated HNSW example do: + ```sh mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.08.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.AcceleratedHnswExample ``` To run the Index and Search on GPU example do: + ```sh mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.08.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.IndexAndSearchonGPUExample ``` diff --git a/java/cuvs-lucene/examples/pom.xml b/java/cuvs-lucene/examples/pom.xml index 1bc5cdd49d..c719123474 100644 --- a/java/cuvs-lucene/examples/pom.xml +++ b/java/cuvs-lucene/examples/pom.xml @@ -21,10 +21,59 @@ UTF-8 - - - - + + + + gcs-maven-central-mirror + GCS Maven Central mirror + https://maven-central.storage-download.googleapis.com/maven2/ + + true + + + false + + + + + central + Maven Central + https://repo.maven.apache.org/maven2 + + true + + + false + + + + + + + + gcs-maven-central-mirror + GCS Maven Central mirror + https://maven-central.storage-download.googleapis.com/maven2/ + + true + + + false + + + + + central + Maven Plugin Repository + https://repo.maven.apache.org/maven2 + + true + + + false + + + diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index e13e5d2b6a..14390391ab 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -43,6 +43,60 @@ UTF-8 + + + + gcs-maven-central-mirror + GCS Maven Central mirror + https://maven-central.storage-download.googleapis.com/maven2/ + + true + + + false + + + + + central + Maven Central + https://repo.maven.apache.org/maven2 + + true + + + false + + + + + + + + gcs-maven-central-mirror + GCS Maven Central mirror + https://maven-central.storage-download.googleapis.com/maven2/ + + true + + + false + + + + + central + Maven Plugin Repository + https://repo.maven.apache.org/maven2 + + true + + + false + + + + org.apache.lucene From 8ffc4104aaf28279b492ad2ab55b476f366bf45d Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 25 Jun 2026 19:42:19 -0500 Subject: [PATCH 086/105] Build and test with CUDA 13.3.0 (NVIDIA/cuvs-lucene#164) Contributes to https://github.com/rapidsai/build-planning/issues/286 * uses CUDA 13.3.0 to build and test * updates to CUDA 13.3.0 devcontainers ## Notes for Reviewers This switches GitHub Actions workflows to the `cuda-13.3.0` branch from here: https://github.com/rapidsai/shared-workflows/pull/574 A future round of PRs will revert that back to `main`, once all of RAPIDS is migrated. --------- Co-authored-by: James Lamb Co-authored-by: James Lamb --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 2 +- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- ...2_arch-aarch64.yaml => all_cuda-133_arch-aarch64.yaml} | 4 ++-- ...132_arch-x86_64.yaml => all_cuda-133_arch-x86_64.yaml} | 4 ++-- java/cuvs-lucene/dependencies.yaml | 8 ++++++-- java/cuvs-lucene/examples/README.md | 2 +- 7 files changed, 14 insertions(+), 10 deletions(-) rename java/cuvs-lucene/conda/environments/{all_cuda-132_arch-aarch64.yaml => all_cuda-133_arch-aarch64.yaml} (90%) rename java/cuvs-lucene/conda/environments/{all_cuda-132_arch-x86_64.yaml => all_cuda-133_arch-x86_64.yaml} (90%) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 1fa894e2d3..214571bdb7 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -41,7 +41,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.2.0' + - '13.3.0' with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 1f9b336281..d298cfc716 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -93,7 +93,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.2.0' + - '13.3.0' with: build_type: pull-request node_type: "gpu-l4-latest-1" diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 7f49c398d4..9b9c518b68 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -30,7 +30,7 @@ jobs: matrix: cuda_version: - '12.9.1' - - '13.2.0' + - '13.3.0' with: build_type: ${{ inputs.build_type }} branch: ${{ inputs.branch }} diff --git a/java/cuvs-lucene/conda/environments/all_cuda-132_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-133_arch-aarch64.yaml similarity index 90% rename from java/cuvs-lucene/conda/environments/all_cuda-132_arch-aarch64.yaml rename to java/cuvs-lucene/conda/environments/all_cuda-133_arch-aarch64.yaml index 2cc700a096..c8f9074c42 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-132_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-133_arch-aarch64.yaml @@ -10,7 +10,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-version=13.2 +- cuda-version=13.3 - cxx-compiler - gcc_linux-aarch64=14.* - libcublas-dev @@ -22,4 +22,4 @@ dependencies: - ninja - openjdk=22.* - sysroot_linux-aarch64==2.28 -name: all_cuda-132_arch-aarch64 +name: all_cuda-133_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-132_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-133_arch-x86_64.yaml similarity index 90% rename from java/cuvs-lucene/conda/environments/all_cuda-132_arch-x86_64.yaml rename to java/cuvs-lucene/conda/environments/all_cuda-133_arch-x86_64.yaml index 76ff4538d0..79eca1d1e3 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-132_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-133_arch-x86_64.yaml @@ -10,7 +10,7 @@ dependencies: - cuda-nvcc - cuda-nvtx-dev - cuda-profiler-api -- cuda-version=13.2 +- cuda-version=13.3 - cxx-compiler - gcc_linux-64=14.* - libcublas-dev @@ -22,4 +22,4 @@ dependencies: - ninja - openjdk=22.* - sysroot_linux-64==2.28 -name: all_cuda-132_arch-x86_64 +name: all_cuda-133_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index a94ab383cc..4ed7776c21 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # Dependency list for https://github.com/rapidsai/dependency-file-generator @@ -6,7 +6,7 @@ files: all: output: conda matrix: - cuda: ["12.9", "13.2"] + cuda: ["12.9", "13.3"] arch: [x86_64, aarch64] includes: - cuda @@ -67,6 +67,10 @@ dependencies: cuda: "13.2" packages: - cuda-version=13.2 + - matrix: + cuda: "13.3" + packages: + - cuda-version=13.3 cuda: common: - output_types: [conda] diff --git a/java/cuvs-lucene/examples/README.md b/java/cuvs-lucene/examples/README.md index 1c3876b325..b5af3e57ed 100644 --- a/java/cuvs-lucene/examples/README.md +++ b/java/cuvs-lucene/examples/README.md @@ -19,7 +19,7 @@ cd .. Then do: ```sh -docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.08-cuda13.2.0-ubuntu24.04-py3.13 +docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.08-cuda13.3.0-ubuntu24.04-py3.13 ``` Inside the docker container (and in the `cuvs-lucene's` root directory) do: From ff0878ccda0379022aa666b5c0b053b4ef56c723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9phine=20Oberholtzer?= Date: Fri, 26 Jun 2026 13:42:23 -0400 Subject: [PATCH 087/105] Update CODEOWNERS team names for rapidsai->NVIDIA migration --- java/cuvs-lucene/.github/CODEOWNERS | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/java/cuvs-lucene/.github/CODEOWNERS b/java/cuvs-lucene/.github/CODEOWNERS index ea66ecb2c8..cf284e772b 100644 --- a/java/cuvs-lucene/.github/CODEOWNERS +++ b/java/cuvs-lucene/.github/CODEOWNERS @@ -1,23 +1,23 @@ # default group # (will be overridden by more specific matches below) -* @rapidsai/lucene-cuvs-write +* @NVIDIA/cuvs-lucene-write # java code owners -src/ @rapidsai/cuvs-java-codeowners +src/ @NVIDIA/cuvs-java-codeowners # docs code owners -LICENSE @rapidsai/cuvs-docs-codeowners -/README.md @rapidsai/cuvs-docs-codeowners +LICENSE @NVIDIA/cuvs-docs-codeowners +/README.md @NVIDIA/cuvs-docs-codeowners # CI code owners -/.github/ @rapidsai/ci-codeowners -/ci/ @rapidsai/ci-codeowners +/.github/ @NVIDIA/adi-ci-codeowners +/ci/ @NVIDIA/adi-ci-codeowners # packaging code owners -build.sh @rapidsai/packaging-codeowners -dependencies.yaml @rapidsai/packaging-codeowners -pom.xml @rapidsai/cuvs-build-codeowners -/.pre-commit-config.yaml @rapidsai/packaging-codeowners +build.sh @NVIDIA/adi-packaging-codeowners +dependencies.yaml @NVIDIA/adi-packaging-codeowners +pom.xml @NVIDIA/cuvs-build-codeowners +/.pre-commit-config.yaml @NVIDIA/adi-packaging-codeowners # Ops code owners -/SECURITY.md @rapidsai/ops-codeowners +/SECURITY.md @NVIDIA/adi-ops-codeowners From 535aaf8b6ae9e263ba0bcf99934b5470d470e690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9phine=20Oberholtzer?= Date: Fri, 26 Jun 2026 14:00:45 -0400 Subject: [PATCH 088/105] Add verify-codeowners pre-commit check --- java/cuvs-lucene/.pre-commit-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/java/cuvs-lucene/.pre-commit-config.yaml b/java/cuvs-lucene/.pre-commit-config.yaml index a40771e556..09e502ff98 100644 --- a/java/cuvs-lucene/.pre-commit-config.yaml +++ b/java/cuvs-lucene/.pre-commit-config.yaml @@ -24,6 +24,8 @@ repos: meta[.]yaml$| dependencies[.]yaml$| ^[.]pre-commit-config[.]yaml$ + - id: verify-codeowners + args: [--fix, --org=NVIDIA, --project-prefix=cuvs-lucene] - repo: https://github.com/rapidsai/dependency-file-generator rev: v1.21.0 hooks: From dca9597e5bc63ab3c79c4f4f8764f8540ae7d5b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9phine=20Oberholtzer?= Date: Fri, 26 Jun 2026 14:22:37 -0400 Subject: [PATCH 089/105] Disable verify-codeowners check --- java/cuvs-lucene/.pre-commit-config.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/java/cuvs-lucene/.pre-commit-config.yaml b/java/cuvs-lucene/.pre-commit-config.yaml index 09e502ff98..63f7fced53 100644 --- a/java/cuvs-lucene/.pre-commit-config.yaml +++ b/java/cuvs-lucene/.pre-commit-config.yaml @@ -24,8 +24,9 @@ repos: meta[.]yaml$| dependencies[.]yaml$| ^[.]pre-commit-config[.]yaml$ - - id: verify-codeowners - args: [--fix, --org=NVIDIA, --project-prefix=cuvs-lucene] + # TODO: Re-enable once verify-codeowners supports --org parameter + # - id: verify-codeowners + # args: [--fix, --org=NVIDIA, --project-prefix=cuvs-lucene] - repo: https://github.com/rapidsai/dependency-file-generator rev: v1.21.0 hooks: From 2b95f5a8b5e153efce764625726f4d4d460c8369 Mon Sep 17 00:00:00 2001 From: nvzm123 Date: Wed, 15 Jul 2026 09:07:20 -0700 Subject: [PATCH 090/105] cuvs-lucene-137: increase max-number of HNSW layers in CAGRA_HNSW's conversion from CAGRA to HNSW (NVIDIA/cuvs-lucene#140) The natural number of HNSW layers is ln(segment_size)/ln(graph_degree), which is often around 6, yet we had a constraint that was limiting us to a maximum of 3 HNSW layers. This change effectively makes the maximum number of layers unbounded. Authors: - https://github.com/nvzm123 - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Corey J. Nolet (https://github.com/cjnolet) URL: https://github.com/NVIDIA/cuvs-lucene/pull/140 --- .../java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java index dffb500467..aac314153d 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -42,7 +42,7 @@ public static enum Strategy { public static final int MIN_GRAPH_DEG = 1; public static final int MAX_GRAPH_DEG = 512; public static final int MIN_HNSW_LAYERS = 1; - public static final int MAX_HNSW_LAYERS = 3; + public static final int MAX_HNSW_LAYERS = 99; public static final int MIN_MAX_CONN = 1; public static final int MAX_MAX_CONN = 512; public static final int MIN_BEAM_WIDTH = 1; From 69bb9dafdf9caccd1ec875b9083fab6bf87e24cf Mon Sep 17 00:00:00 2001 From: jolorunyomi Date: Thu, 16 Jul 2026 17:32:36 -0500 Subject: [PATCH 091/105] Prepare release/26.08 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 8 ++++---- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/RAPIDS_BRANCH | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 214571bdb7..fd13603088 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -33,7 +33,7 @@ permissions: {} jobs: java-build: secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.08 # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index d298cfc716..059b1e235c 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -16,7 +16,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.08 if: always() with: needs: ${{ toJSON(needs) }} @@ -54,7 +54,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@release/26.08 with: files_yaml: | test_java: @@ -71,7 +71,7 @@ jobs: checks: needs: telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@release/26.08 with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" @@ -84,7 +84,7 @@ jobs: conda-java-tests: needs: [changed-files, checks] secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.08 if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 9b9c518b68..06ee5a57bd 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -24,7 +24,7 @@ permissions: {} jobs: conda-java-tests: secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.08 strategy: fail-fast: false matrix: diff --git a/java/cuvs-lucene/RAPIDS_BRANCH b/java/cuvs-lucene/RAPIDS_BRANCH index ba2906d066..8ce6b73837 100644 --- a/java/cuvs-lucene/RAPIDS_BRANCH +++ b/java/cuvs-lucene/RAPIDS_BRANCH @@ -1 +1 @@ -main +release/26.08 From b7048c45ce9c4389512a9786598e9080f0e1d943 Mon Sep 17 00:00:00 2001 From: jolorunyomi Date: Thu, 16 Jul 2026 17:32:36 -0500 Subject: [PATCH 092/105] Revert "Prepare release/26.08" This reverts commit 69bb9dafdf9caccd1ec875b9083fab6bf87e24cf. --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 8 ++++---- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/RAPIDS_BRANCH | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index fd13603088..214571bdb7 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -33,7 +33,7 @@ permissions: {} jobs: java-build: secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. strategy: diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index 059b1e235c..d298cfc716 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -16,7 +16,7 @@ jobs: - conda-java-tests - telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.08 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main if: always() with: needs: ${{ toJSON(needs) }} @@ -54,7 +54,7 @@ jobs: changed-files: needs: telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@release/26.08 + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main with: files_yaml: | test_java: @@ -71,7 +71,7 @@ jobs: checks: needs: telemetry-setup secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@release/26.08 + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main with: enable_check_generated_files: false ignored_pr_jobs: "telemetry-summarize" @@ -84,7 +84,7 @@ jobs: conda-java-tests: needs: [changed-files, checks] secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 06ee5a57bd..9b9c518b68 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -24,7 +24,7 @@ permissions: {} jobs: conda-java-tests: secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@release/26.08 + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main strategy: fail-fast: false matrix: diff --git a/java/cuvs-lucene/RAPIDS_BRANCH b/java/cuvs-lucene/RAPIDS_BRANCH index 8ce6b73837..ba2906d066 100644 --- a/java/cuvs-lucene/RAPIDS_BRANCH +++ b/java/cuvs-lucene/RAPIDS_BRANCH @@ -1 +1 @@ -release/26.08 +main From abc5517bb4178516694901dd462ea5eb7d4ca421 Mon Sep 17 00:00:00 2001 From: jolorunyomi Date: Thu, 16 Jul 2026 17:36:36 -0500 Subject: [PATCH 093/105] Update to 26.10.00 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 2 +- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- java/cuvs-lucene/VERSION | 2 +- java/cuvs-lucene/bench/pom.xml | 4 ++-- java/cuvs-lucene/build.sh | 4 ++-- .../conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- .../conda/environments/all_cuda-133_arch-aarch64.yaml | 2 +- .../conda/environments/all_cuda-133_arch-x86_64.yaml | 2 +- java/cuvs-lucene/dependencies.yaml | 2 +- java/cuvs-lucene/examples/README.md | 6 +++--- java/cuvs-lucene/examples/pom.xml | 4 ++-- java/cuvs-lucene/pom.xml | 4 ++-- 14 files changed, 20 insertions(+), 20 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index 214571bdb7..bc1f50311b 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -47,7 +47,7 @@ jobs: branch: ${{ inputs.branch }} arch: "amd64" date: ${{ inputs.date }} - container_image: "rapidsai/ci-conda:26.08-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/build_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index d298cfc716..bd8b8efead 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -98,7 +98,7 @@ jobs: build_type: pull-request node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:26.08-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" file_to_upload: "target/" artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 9b9c518b68..2a55400c1d 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -38,7 +38,7 @@ jobs: sha: ${{ inputs.sha }} node_type: "gpu-l4-latest-1" arch: "amd64" - container_image: "rapidsai/ci-conda:26.08-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" + container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" script: "ci/test_java.sh" permissions: actions: read diff --git a/java/cuvs-lucene/VERSION b/java/cuvs-lucene/VERSION index 4e6864b4ca..6549ba6527 100644 --- a/java/cuvs-lucene/VERSION +++ b/java/cuvs-lucene/VERSION @@ -1 +1 @@ -26.08.00 +26.10.00 diff --git a/java/cuvs-lucene/bench/pom.xml b/java/cuvs-lucene/bench/pom.xml index 04ea66c581..d4bc7fbc22 100644 --- a/java/cuvs-lucene/bench/pom.xml +++ b/java/cuvs-lucene/bench/pom.xml @@ -11,7 +11,7 @@ com.nvidia.cuvs.lucene.benchmarks cuvs-lucene-benchmarks - 26.08.0 + 26.10.0 jar cuvs-lucene-benchmarks @@ -85,7 +85,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 26.08.0 + 26.10.0 commons-io diff --git a/java/cuvs-lucene/build.sh b/java/cuvs-lucene/build.sh index 4ec8c0870e..0ae9381ae7 100755 --- a/java/cuvs-lucene/build.sh +++ b/java/cuvs-lucene/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -e -u -o pipefail @@ -8,7 +8,7 @@ set -e -u -o pipefail ARGS="$*" NUMARGS=$# -VERSION="26.08.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked +VERSION="26.10.0" # Note: The version is updated automatically when ci/release/update-version.sh is invoked GROUP_ID="com.nvidia.cuvs.lucene" function hasArg { diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml index caf4537fec..7a689fe768 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -17,7 +17,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.08.* +- libcuvs==26.10.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml index 761a1a12dd..562abae6d0 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -17,7 +17,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.08.* +- libcuvs==26.10.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/conda/environments/all_cuda-133_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-133_arch-aarch64.yaml index c8f9074c42..8ddd2484ce 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-133_arch-aarch64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-133_arch-aarch64.yaml @@ -17,7 +17,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.08.* +- libcuvs==26.10.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/conda/environments/all_cuda-133_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-133_arch-x86_64.yaml index 79eca1d1e3..34ee83dd9a 100644 --- a/java/cuvs-lucene/conda/environments/all_cuda-133_arch-x86_64.yaml +++ b/java/cuvs-lucene/conda/environments/all_cuda-133_arch-x86_64.yaml @@ -17,7 +17,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev -- libcuvs==26.08.* +- libcuvs==26.10.* - maven - ninja - openjdk=22.* diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml index 4ed7776c21..b7c09a93ed 100644 --- a/java/cuvs-lucene/dependencies.yaml +++ b/java/cuvs-lucene/dependencies.yaml @@ -82,7 +82,7 @@ dependencies: - libcurand-dev - libcusolver-dev - libcusparse-dev - - libcuvs==26.08.* + - libcuvs==26.10.* java: common: - output_types: conda diff --git a/java/cuvs-lucene/examples/README.md b/java/cuvs-lucene/examples/README.md index b5af3e57ed..8a9b61a7b7 100644 --- a/java/cuvs-lucene/examples/README.md +++ b/java/cuvs-lucene/examples/README.md @@ -19,7 +19,7 @@ cd .. Then do: ```sh -docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.08-cuda13.3.0-ubuntu24.04-py3.13 +docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.10-cuda13.3.0-ubuntu24.04-py3.13 ``` Inside the docker container (and in the `cuvs-lucene's` root directory) do: @@ -31,11 +31,11 @@ Inside the docker container (and in the `cuvs-lucene's` root directory) do: To run Accelerated HNSW example do: ```sh -mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.08.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.AcceleratedHnswExample +mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.10.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.AcceleratedHnswExample ``` To run the Index and Search on GPU example do: ```sh -mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.08.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.IndexAndSearchonGPUExample +mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.10.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.IndexAndSearchonGPUExample ``` diff --git a/java/cuvs-lucene/examples/pom.xml b/java/cuvs-lucene/examples/pom.xml index c719123474..9b8fcaaff4 100644 --- a/java/cuvs-lucene/examples/pom.xml +++ b/java/cuvs-lucene/examples/pom.xml @@ -11,7 +11,7 @@ com.nvidia.cuvs.lucene.examples examples - 26.08.0 + 26.10.0 examples @@ -100,7 +100,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 26.08.0 + 26.10.0 diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 14390391ab..70f246d27f 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -12,7 +12,7 @@ com.nvidia.cuvs.lucene cuvs-lucene - 26.08.0 + 26.10.0 cuvs-lucene jar @@ -129,7 +129,7 @@ com.nvidia.cuvs cuvs-java - 26.08.0 + 26.10.0 From 93ccb67fd87f7593cc6a4b2edcf58966f936d05e Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 20 Jul 2026 14:00:53 -1000 Subject: [PATCH 094/105] Upgrade cuda 12.9.1 -> 12.9.2 (NVIDIA/cuvs-lucene#179) After the upgrade to 26.10 in NVIDIA/cuvs-lucene#176, the build is failing due to the absence of the rapidsai/ci-conda:26.10-cuda12.9.1-ubuntu24.04-py3.13 image on Docker Hub. We also need to update the CUDA version to 12.9.2. Authors: - Igor Motov (https://github.com/imotov) Approvers: - Bradley Dice (https://github.com/bdice) URL: https://github.com/NVIDIA/cuvs-lucene/pull/179 --- java/cuvs-lucene/.github/workflows/build.yaml | 2 +- java/cuvs-lucene/.github/workflows/pr.yaml | 2 +- java/cuvs-lucene/.github/workflows/test.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml index bc1f50311b..16b9fe935f 100644 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ b/java/cuvs-lucene/.github/workflows/build.yaml @@ -40,7 +40,7 @@ jobs: fail-fast: false matrix: cuda_version: - - '12.9.1' + - '12.9.2' - '13.3.0' with: build_type: ${{ inputs.build_type || 'branch' }} diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml index bd8b8efead..1ce0ab836a 100644 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ b/java/cuvs-lucene/.github/workflows/pr.yaml @@ -92,7 +92,7 @@ jobs: fail-fast: false matrix: cuda_version: - - '12.9.1' + - '12.9.2' - '13.3.0' with: build_type: pull-request diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml index 2a55400c1d..132420f0bf 100644 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ b/java/cuvs-lucene/.github/workflows/test.yaml @@ -29,7 +29,7 @@ jobs: fail-fast: false matrix: cuda_version: - - '12.9.1' + - '12.9.2' - '13.3.0' with: build_type: ${{ inputs.build_type }} From 9d57c529c3c92772f6e36ca8a22ca654ff1a8cb6 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 21 Jul 2026 12:42:35 -1000 Subject: [PATCH 095/105] Delete unnecessary cuvs-lucene files --- java/cuvs-lucene/.github/CODEOWNERS | 23 -- java/cuvs-lucene/.github/copy-pr-bot.yaml | 5 - java/cuvs-lucene/.github/ops-bot.yaml | 10 - java/cuvs-lucene/.github/release.yml | 27 --- java/cuvs-lucene/.github/workflows/build.yaml | 60 ----- java/cuvs-lucene/.github/workflows/pr.yaml | 123 ---------- java/cuvs-lucene/.github/workflows/test.yaml | 48 ---- java/cuvs-lucene/.github/zizmor.yml | 9 - java/cuvs-lucene/.pre-commit-config.yaml | 42 ---- java/cuvs-lucene/CONTRIBUTING.md | 133 ----------- java/cuvs-lucene/LICENSE | 225 ------------------ java/cuvs-lucene/RAPIDS_BRANCH | 1 - java/cuvs-lucene/SECURITY.md | 40 ---- java/cuvs-lucene/VERSION | 1 - java/cuvs-lucene/ci/build_java.sh | 66 ----- java/cuvs-lucene/ci/check_style.sh | 24 -- java/cuvs-lucene/ci/release/update-version.sh | 126 ---------- java/cuvs-lucene/ci/test_java.sh | 15 -- .../all_cuda-129_arch-aarch64.yaml | 25 -- .../all_cuda-129_arch-x86_64.yaml | 25 -- .../all_cuda-133_arch-aarch64.yaml | 25 -- .../all_cuda-133_arch-x86_64.yaml | 25 -- java/cuvs-lucene/dependencies.yaml | 109 --------- 23 files changed, 1187 deletions(-) delete mode 100644 java/cuvs-lucene/.github/CODEOWNERS delete mode 100644 java/cuvs-lucene/.github/copy-pr-bot.yaml delete mode 100644 java/cuvs-lucene/.github/ops-bot.yaml delete mode 100644 java/cuvs-lucene/.github/release.yml delete mode 100644 java/cuvs-lucene/.github/workflows/build.yaml delete mode 100644 java/cuvs-lucene/.github/workflows/pr.yaml delete mode 100644 java/cuvs-lucene/.github/workflows/test.yaml delete mode 100644 java/cuvs-lucene/.github/zizmor.yml delete mode 100644 java/cuvs-lucene/.pre-commit-config.yaml delete mode 100644 java/cuvs-lucene/CONTRIBUTING.md delete mode 100644 java/cuvs-lucene/LICENSE delete mode 100644 java/cuvs-lucene/RAPIDS_BRANCH delete mode 100644 java/cuvs-lucene/SECURITY.md delete mode 100644 java/cuvs-lucene/VERSION delete mode 100755 java/cuvs-lucene/ci/build_java.sh delete mode 100755 java/cuvs-lucene/ci/check_style.sh delete mode 100755 java/cuvs-lucene/ci/release/update-version.sh delete mode 100755 java/cuvs-lucene/ci/test_java.sh delete mode 100644 java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml delete mode 100644 java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml delete mode 100644 java/cuvs-lucene/conda/environments/all_cuda-133_arch-aarch64.yaml delete mode 100644 java/cuvs-lucene/conda/environments/all_cuda-133_arch-x86_64.yaml delete mode 100644 java/cuvs-lucene/dependencies.yaml diff --git a/java/cuvs-lucene/.github/CODEOWNERS b/java/cuvs-lucene/.github/CODEOWNERS deleted file mode 100644 index cf284e772b..0000000000 --- a/java/cuvs-lucene/.github/CODEOWNERS +++ /dev/null @@ -1,23 +0,0 @@ -# default group -# (will be overridden by more specific matches below) -* @NVIDIA/cuvs-lucene-write - -# java code owners -src/ @NVIDIA/cuvs-java-codeowners - -# docs code owners -LICENSE @NVIDIA/cuvs-docs-codeowners -/README.md @NVIDIA/cuvs-docs-codeowners - -# CI code owners -/.github/ @NVIDIA/adi-ci-codeowners -/ci/ @NVIDIA/adi-ci-codeowners - -# packaging code owners -build.sh @NVIDIA/adi-packaging-codeowners -dependencies.yaml @NVIDIA/adi-packaging-codeowners -pom.xml @NVIDIA/cuvs-build-codeowners -/.pre-commit-config.yaml @NVIDIA/adi-packaging-codeowners - -# Ops code owners -/SECURITY.md @NVIDIA/adi-ops-codeowners diff --git a/java/cuvs-lucene/.github/copy-pr-bot.yaml b/java/cuvs-lucene/.github/copy-pr-bot.yaml deleted file mode 100644 index 4e99200c8c..0000000000 --- a/java/cuvs-lucene/.github/copy-pr-bot.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# Configuration file for `copy-pr-bot` GitHub App -# https://docs.gha-runners.nvidia.com/platform/apps/copy-pr-bot/ - -enabled: true -auto_sync_draft: false diff --git a/java/cuvs-lucene/.github/ops-bot.yaml b/java/cuvs-lucene/.github/ops-bot.yaml deleted file mode 100644 index 2d5999296a..0000000000 --- a/java/cuvs-lucene/.github/ops-bot.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# This file controls which features from the `ops-bot` repository below are enabled. -# - https://github.com/rapidsai/ops-bot - -auto_merger: true -branch_checker: true -label_checker: true -release_drafter: true -recently_updated: true -forward_merger: true -merge_barriers: true diff --git a/java/cuvs-lucene/.github/release.yml b/java/cuvs-lucene/.github/release.yml deleted file mode 100644 index 2c9a85805c..0000000000 --- a/java/cuvs-lucene/.github/release.yml +++ /dev/null @@ -1,27 +0,0 @@ -# GitHub Auto-Generated Release Notes Configuration for RAPIDS -# This file configures how GitHub automatically generates release notes - -changelog: - exclude: - labels: - - ignore-for-release - - dependencies - authors: - - rapids-bot[bot] - - dependabot[bot] - categories: - - title: 🚨 Breaking Changes - labels: - - breaking - - title: 🐛 Bug Fixes - labels: - - bug - - title: 📖 Documentation - labels: - - doc - - title: 🚀 New Features - labels: - - feature request - - title: 🛠️ Improvements - labels: - - improvement diff --git a/java/cuvs-lucene/.github/workflows/build.yaml b/java/cuvs-lucene/.github/workflows/build.yaml deleted file mode 100644 index 16b9fe935f..0000000000 --- a/java/cuvs-lucene/.github/workflows/build.yaml +++ /dev/null @@ -1,60 +0,0 @@ -name: build -on: - push: - branches: - - "main" - - "release/*" - tags: - - v[0-9][0-9].[0-9][0-9].[0-9][0-9] - workflow_dispatch: - inputs: - branch: - description: | - branch: git branch the workflow run targets. - Required even when 'sha' is provided because it is also used for organizing artifacts. - required: true - type: string - date: - description: "date: Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" - required: true - type: string - sha: - description: "sha: full git commit SHA to check out" - required: true - type: string - build_type: - description: "build_type: one of [branch, nightly, pull-request]" - type: string - default: nightly -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }} - cancel-in-progress: true -permissions: {} -jobs: - java-build: - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. - strategy: - fail-fast: false - matrix: - cuda_version: - - '12.9.2' - - '13.3.0' - with: - build_type: ${{ inputs.build_type || 'branch' }} - branch: ${{ inputs.branch }} - arch: "amd64" - date: ${{ inputs.date }} - container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" - script: "ci/build_java.sh" - file_to_upload: "target/" - artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" - sha: ${{ inputs.sha }} - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read diff --git a/java/cuvs-lucene/.github/workflows/pr.yaml b/java/cuvs-lucene/.github/workflows/pr.yaml deleted file mode 100644 index 1ce0ab836a..0000000000 --- a/java/cuvs-lucene/.github/workflows/pr.yaml +++ /dev/null @@ -1,123 +0,0 @@ -name: pr -on: - push: - branches: - - "pull-request/[0-9]+" -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -permissions: {} -jobs: - pr-builder: - needs: - - check-nightly-ci - - changed-files - - checks - - conda-java-tests - - telemetry-setup - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main - if: always() - with: - needs: ${{ toJSON(needs) }} - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - telemetry-setup: - continue-on-error: true - runs-on: ubuntu-latest - env: - OTEL_SERVICE_NAME: 'pr-cuvs' - steps: - - name: Telemetry setup - if: ${{ vars.TELEMETRY_ENABLED == 'true' }} - uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env-vars@main - check-nightly-ci: - runs-on: ubuntu-latest - permissions: - actions: read - id-token: write - env: - GH_TOKEN: ${{ github.token }} - steps: - - name: Get PR Info - id: get-pr-info - uses: nv-gha-runners/get-pr-info@090577647b8ddc4e06e809e264f7881650ecdccf # main - - name: Check if nightly CI is passing - uses: rapidsai/shared-actions/check_nightly_success/dispatch@main - with: - repo: ${{ github.repository }} - target-branch: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.ref }} - changed-files: - needs: telemetry-setup - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@main - with: - files_yaml: | - test_java: - - '**' - - '!.pre-commit-config.yaml' - - '!README.md' - - '!SECURITY.md' - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - checks: - needs: telemetry-setup - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@main - with: - enable_check_generated_files: false - ignored_pr_jobs: "telemetry-summarize" - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - conda-java-tests: - needs: [changed-files, checks] - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java - # Artifacts are not published from these jobs, so it's safe to run for multiple CUDA versions. - # If these jobs start producing artifacts, the names will have to differentiate between CUDA versions. - strategy: - fail-fast: false - matrix: - cuda_version: - - '12.9.2' - - '13.3.0' - with: - build_type: pull-request - node_type: "gpu-l4-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" - script: "ci/test_java.sh" - file_to_upload: "target/" - artifact-name: "cuvs-lucene-cuda${{ matrix.cuda_version }}" - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read - telemetry-summarize: - # This job must use a self-hosted runner to record telemetry traces. - runs-on: linux-amd64-cpu4 - needs: pr-builder - if: ${{ vars.TELEMETRY_ENABLED == 'true' && !cancelled() }} - continue-on-error: true - steps: - - name: Telemetry summarize - uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main - env: - GH_TOKEN: ${{ github.token }} - permissions: - actions: write diff --git a/java/cuvs-lucene/.github/workflows/test.yaml b/java/cuvs-lucene/.github/workflows/test.yaml deleted file mode 100644 index 132420f0bf..0000000000 --- a/java/cuvs-lucene/.github/workflows/test.yaml +++ /dev/null @@ -1,48 +0,0 @@ -name: test -on: - workflow_dispatch: - inputs: - branch: - description: | - branch: git branch the workflow run targets. - Required even when 'sha' is provided because it is also used for organizing artifacts. - required: true - type: string - date: - description: "date: Date (YYYY-MM-DD) this run is for. Used to organize artifacts produced by nightly builds" - required: true - type: string - sha: - description: "sha: full git commit SHA to check out" - required: true - type: string - build_type: - description: "build_type: one of [branch, nightly, pull-request]" - type: string - default: nightly -permissions: {} -jobs: - conda-java-tests: - secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main - strategy: - fail-fast: false - matrix: - cuda_version: - - '12.9.2' - - '13.3.0' - with: - build_type: ${{ inputs.build_type }} - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - node_type: "gpu-l4-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.cuda_version }}-ubuntu24.04-py3.13" - script: "ci/test_java.sh" - permissions: - actions: read - contents: read - id-token: write - packages: read - pull-requests: read diff --git a/java/cuvs-lucene/.github/zizmor.yml b/java/cuvs-lucene/.github/zizmor.yml deleted file mode 100644 index 1b6ea1e53f..0000000000 --- a/java/cuvs-lucene/.github/zizmor.yml +++ /dev/null @@ -1,9 +0,0 @@ -rules: - unpinned-uses: - config: - policies: - # We require SHA-pinning for all workflows and actions _except_ for those from - # rapidsai/shared-workflows and rapidsai/shared-actions - "rapidsai/shared-workflows/*": any - "rapidsai/shared-actions/*": any - "*": hash-pin diff --git a/java/cuvs-lucene/.pre-commit-config.yaml b/java/cuvs-lucene/.pre-commit-config.yaml deleted file mode 100644 index 63f7fced53..0000000000 --- a/java/cuvs-lucene/.pre-commit-config.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 - hooks: - - id: check-json - - id: trailing-whitespace - - id: end-of-file-fixer - - id: check-symlinks - - id: check-xml - - repo: https://github.com/rapidsai/pre-commit-hooks - rev: v1.5.1 - hooks: - - id: verify-copyright - name: verify-copyright - args: [--fix, --spdx] - files: | - (?x) - [.](cmake|cpp|cu|cuh|h|hpp|sh|pxd|py|pyx|rs|java)$| - CMakeLists[.]txt$| - CMakeLists_standalone[.]txt$| - meta[.]yaml$| - dependencies[.]yaml$| - ^[.]pre-commit-config[.]yaml$ - # TODO: Re-enable once verify-codeowners supports --org parameter - # - id: verify-codeowners - # args: [--fix, --org=NVIDIA, --project-prefix=cuvs-lucene] - - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.21.0 - hooks: - - id: rapids-dependency-file-generator - args: ["--clean", "--warn-all", "--strict"] - - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.11.0.1 - hooks: - - id: shellcheck - - repo: https://github.com/zizmorcore/zizmor-pre-commit - rev: v1.26.1 - hooks: - - id: zizmor diff --git a/java/cuvs-lucene/CONTRIBUTING.md b/java/cuvs-lucene/CONTRIBUTING.md deleted file mode 100644 index ff9a3b91f7..0000000000 --- a/java/cuvs-lucene/CONTRIBUTING.md +++ /dev/null @@ -1,133 +0,0 @@ -# Contributing - -If you are interested in contributing to Lucene-cuVS, your contributions will fall -into three categories: -1. You want to report a bug, feature request, or documentation issue - - File an [issue](https://github.com/rapidsai/cuvs/issues/new/choose) - describing what you encountered or what you want to see changed. - - The RAPIDS team will evaluate the issues and triage them, scheduling - them for a release. If you believe the issue needs priority attention - comment on the issue to notify the team. -2. You want to propose a new Feature and implement it - - Post about your intended feature, and we shall discuss the design and - implementation. - - Once we agree that the plan looks good, go ahead and implement it, using - the [code contributions](#code-contributions) guide below. -3. You want to implement a feature or bug-fix for an outstanding issue - - Follow the [code contributions](#code-contributions) guide below. - - If you need more context on a particular issue, please ask and we shall - provide. - - -## Code contributions - -### Your first issue - -1. Read the project's [README.md](https://github.com/rapidsai/cuvs) - to learn how to setup the development environment -2. Find an issue to work on. The best way is to look for the [good first issue](https://github.com/rapidsai/CUVS/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - or [help wanted](https://github.com/rapidsai/cuvs/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) labels -3. Comment on the issue saying you are going to work on it -4. Code! Make sure to update unit tests! -5. When done, [create your pull request](https://github.com/rapidsai/cuvs/compare) -6. Verify that CI passes all [status checks](https://help.github.com/articles/about-status-checks/). Fix if needed -7. Wait for other developers to review your code and update code as needed -8. Once reviewed and approved, a RAPIDS developer will merge your pull request - -Remember, if you are unsure about anything, don't hesitate to comment on issues -and ask for clarifications! - - -### Python / Pre-commit hooks - -Lucene-cuVS uses [pre-commit](https://pre-commit.com/) to execute code linters and formatters. These tools ensure a consistent code format -throughout the project. Using pre-commit ensures that linter versions and options are aligned for -all developers. Additionally, there is a CI check in place to enforce that committed code follows -our standards. - -To use `pre-commit`, install via `conda` or `pip`: - -```bash -conda install -c conda-forge pre-commit -``` - -```bash -pip install pre-commit -``` - -Then run pre-commit hooks before committing code: - -```bash -pre-commit run -``` - -Optionally, you may set up the pre-commit hooks to run automatically when you make a git commit. This can be done by running: - -```bash -pre-commit install -``` - -Now code linters and formatters will be run each time you commit changes. - -You can skip these checks with `git commit --no-verify` or with the short version `git commit -n`. - - -### Seasoned developers - -Once you have gotten your feet wet and are more comfortable with the code, you -can look at the prioritized issues of our next release in our [project boards](https://github.com/rapidsai/CUVS/projects). - -> **Pro Tip:** Always look at the release board with the highest number for -issues to work on. This is where RAPIDS developers also focus their efforts. - -Look at the unassigned issues, and find an issue you are comfortable with -contributing to. Start with _Step 3_ from above, commenting on the issue to let -others know you are working on it. If you have any questions related to the -implementation of the issue, ask them in the issue instead of the PR. - - -### Signing Your Work - -* We require that all third party contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license. - - * Any contribution which contains commits that are not Signed-Off will not be accepted. - -* To sign off on a commit you simply use the `--signoff` (or `-s`) option when committing your changes: - ```bash - $ git commit -s -m "Add cool feature." - ``` - This will append the following to your commit message: - ``` - Signed-off-by: Your Name - ``` - -* Full text of the DCO: - - ``` - Developer Certificate of Origin - Version 1.1 - - Copyright (C) 2004, 2006 The Linux Foundation and its contributors. - 1 Letterman Drive - Suite D4700 - San Francisco, CA, 94129 - - Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. - ``` - - ``` - Developer's Certificate of Origin 1.1 - - By making a contribution to this project, I certify that: - - (a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or - - (b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or - - (c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it. - - (d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved. - ``` - -## Attribution -Portions adopted from https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md diff --git a/java/cuvs-lucene/LICENSE b/java/cuvs-lucene/LICENSE deleted file mode 100644 index 24376a8922..0000000000 --- a/java/cuvs-lucene/LICENSE +++ /dev/null @@ -1,225 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -========================================================================== -The following license applies to the JQuery JavaScript library --------------------------------------------------------------------------- -Copyright (c) 2010 John Resig, http://jquery.com/ - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/java/cuvs-lucene/RAPIDS_BRANCH b/java/cuvs-lucene/RAPIDS_BRANCH deleted file mode 100644 index ba2906d066..0000000000 --- a/java/cuvs-lucene/RAPIDS_BRANCH +++ /dev/null @@ -1 +0,0 @@ -main diff --git a/java/cuvs-lucene/SECURITY.md b/java/cuvs-lucene/SECURITY.md deleted file mode 100644 index ada890837b..0000000000 --- a/java/cuvs-lucene/SECURITY.md +++ /dev/null @@ -1,40 +0,0 @@ -# Security - -## Reporting Security Issues - -> [!WARNING] -> Do not report security vulnerabilities through public GitHub issues! - -Instead, please submit a private vulnerability report, see below. - -## Reporting a Vulnerability - -1. **NVIDIA Vulnerability Disclosure Program (preferred)** - Submit through the NVIDIA Product Security Incident Response Team (PSIRT) web form () - This is the fastest path to triage and tracking. - -2. **Email NVIDIA PSIRT** - `psirt@nvidia.com` — encrypt sensitive reports with the - [NVIDIA PSIRT PGP key](https://www.nvidia.com/en-us/security/pgp-key). - -3. **GitHub Private Vulnerability Reporting** - Use the **Security and quality** tab on this repository → *Report a vulnerability*. - -## Report Details - -We prefer all communications to be in English. - -Reports should include the following: - -* reproducible example showing how the vulnerability can be exploited -* statement about the impact (including affected versions) - -And we'd appreciate if they also include: - -* statement about whether you are interested in implementing the fix yourself - -## Disclosure Policy - -NVIDIA PSIRT will acknowledge receipt and coordinate triage, fix development, and coordinated disclosure. - -More on NVIDIA's response process: . diff --git a/java/cuvs-lucene/VERSION b/java/cuvs-lucene/VERSION deleted file mode 100644 index 6549ba6527..0000000000 --- a/java/cuvs-lucene/VERSION +++ /dev/null @@ -1 +0,0 @@ -26.10.00 diff --git a/java/cuvs-lucene/ci/build_java.sh b/java/cuvs-lucene/ci/build_java.sh deleted file mode 100755 index 173ca55c28..0000000000 --- a/java/cuvs-lucene/ci/build_java.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -# TODO: Remove this argument-handling when build and test workflows are separated, -# and test_java.sh no longer calls build_java.sh -# ref: https://github.com/rapidsai/cuvs/issues/868 -EXTRA_BUILD_ARGS=() -if [[ "${1:-}" == "--run-java-tests" ]]; then - EXTRA_BUILD_ARGS+=("--run-java-tests") -fi - -# Always build cuvs-java when running the pipeline -EXTRA_BUILD_ARGS+=("--build-cuvs-java") - -# shellcheck disable=SC1091 -. /opt/conda/etc/profile.d/conda.sh - -rapids-logger "Configuring conda strict channel priority" -conda config --set channel_priority strict - -rapids-logger "Generate Java testing dependencies" - -ENV_YAML_DIR="$(mktemp -d)" - -rapids-dependency-file-generator \ - --output conda \ - --file-key java \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" - -rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n java - -# Temporarily allow unbound variables for conda activation. -set +u -conda activate java -set -u - -rapids-print-env - -# Locates the libcuvs.so file path and appends it to LD_LIBRARY_PATH -rapids-logger "Find libcuvs so file and prepend paths to LD_LIBRARY_PATH" - -CONDA_PKG_CACHE_DIR="/opt/conda/pkgs" # comes from `conda info`. Dont know if this ever changes. -if [ -d "$CONDA_PKG_CACHE_DIR" ]; then - echo "==> Directory '$CONDA_PKG_CACHE_DIR' exists." - LIBCUVS_SO_FILE="libcuvs.so" - LIBCUVS_PATH=$(find $CONDA_PKG_CACHE_DIR -name $LIBCUVS_SO_FILE) - if [ -z "$LIBCUVS_PATH" ]; then - echo "==> Could not find the so file. Not updating LD_LIBRARY_PATH" - exit 1 - else - LIBCUVS_DIR=$(dirname "$LIBCUVS_PATH") - export LD_LIBRARY_PATH="$LIBCUVS_DIR:$LD_LIBRARY_PATH" - echo "LD_LIBRARY_PATH is: $LD_LIBRARY_PATH" - fi -else - echo "==> Directory '$CONDA_PKG_CACHE_DIR' does not exist. Not updating LD_LIBRARY_PATH" - exit 1 -fi - -rapids-logger "Run Java build" - -bash ./build.sh "${EXTRA_BUILD_ARGS[@]}" diff --git a/java/cuvs-lucene/ci/check_style.sh b/java/cuvs-lucene/ci/check_style.sh deleted file mode 100755 index 9b6dc337b7..0000000000 --- a/java/cuvs-lucene/ci/check_style.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -rapids-logger "Create checks conda environment" -# shellcheck disable=SC1091 -. /opt/conda/etc/profile.d/conda.sh - -rapids-logger "Configuring conda strict channel priority" -conda config --set channel_priority strict - -rapids-dependency-file-generator \ - --output conda \ - --file-key checks \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml - -rapids-mamba-retry env create --yes -f env.yaml -n checks -conda activate checks - -# Run pre-commit checks -pre-commit run --all-files --show-diff-on-failure diff --git a/java/cuvs-lucene/ci/release/update-version.sh b/java/cuvs-lucene/ci/release/update-version.sh deleted file mode 100755 index 714d92b84f..0000000000 --- a/java/cuvs-lucene/ci/release/update-version.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -######################## -# CUVS Version Updater # -######################## - -## Usage -# Primary interface: bash update-version.sh [--run-context=main|release] -# Fallback interface: [RAPIDS_RUN_CONTEXT=main|release] bash update-version.sh -# CLI arguments take precedence over environment variables -# Defaults to main when no run-context is specified - -set -euo pipefail - -# Verify we're running from the repository root -if [[ ! -d ".git" ]]; then - echo "Error: This script must be run from the repository root directory." - echo "Expected to find: .git/" - echo "Current directory: $(pwd)" - exit 1 -fi - -# Parse command line arguments -CLI_RUN_CONTEXT="" -VERSION_ARG="" - -for arg in "$@"; do - case $arg in - --run-context=*) - CLI_RUN_CONTEXT="${arg#*=}" - shift - ;; - *) - if [[ -z "$VERSION_ARG" ]]; then - VERSION_ARG="$arg" - fi - ;; - esac -done - -# Format is YY.MM.PP - no leading 'v' or trailing 'a' -NEXT_FULL_TAG="$VERSION_ARG" - -# Determine RUN_CONTEXT with CLI precedence over environment variable, defaulting to main -if [[ -n "$CLI_RUN_CONTEXT" ]]; then - RUN_CONTEXT="$CLI_RUN_CONTEXT" - echo "Using run-context from CLI: $RUN_CONTEXT" -elif [[ -n "${RAPIDS_RUN_CONTEXT}" ]]; then - RUN_CONTEXT="$RAPIDS_RUN_CONTEXT" - echo "Using run-context from environment: $RUN_CONTEXT" -else - RUN_CONTEXT="main" - echo "No run-context provided, defaulting to: $RUN_CONTEXT" -fi - -# Validate RUN_CONTEXT value -if [[ "${RUN_CONTEXT}" != "main" && "${RUN_CONTEXT}" != "release" ]]; then - echo "Error: Invalid run-context value '${RUN_CONTEXT}'" - echo "Valid values: main, release" - exit 1 -fi - -# Validate version argument -if [[ -z "$NEXT_FULL_TAG" ]]; then - echo "Error: Version argument is required" - echo "Usage: $0 [--run-context=]" - echo " or: [RAPIDS_RUN_CONTEXT=] $0 " - echo "Note: Defaults to main when run-context is not specified" - exit 1 -fi - -# Get current version -CURRENT_TAG=$(git tag --merged HEAD | grep -xE '^v.*' | sort --version-sort | tail -n 1 | tr -d 'v') - -# Get . for next version -NEXT_MAJOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[1]}') -NEXT_MINOR=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[2]}') -NEXT_PATCH=$(echo "$NEXT_FULL_TAG" | awk '{split($0, a, "."); print a[3]}') -NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} - -# Strip leading 0s in versions, so e.g. '25.10.00' becomes '25.10.0' -PATCH_PEP440=$(python -c "from packaging.version import Version; print(Version('${NEXT_PATCH}'))") - -# Log update context -if [[ "${RUN_CONTEXT}" == "main" ]]; then - RAPIDS_BRANCH_NAME="main" - echo "Preparing development branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting main branch)" -elif [[ "${RUN_CONTEXT}" == "release" ]]; then - RAPIDS_BRANCH_NAME="release/${NEXT_SHORT_TAG}" - echo "Preparing release branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting release/${NEXT_SHORT_TAG} branch)" -fi - -# Inplace sed replace; workaround for Linux and Mac -function sed_runner() { - sed -i.bak ''"$1"'' "$2" && rm -f "${2}".bak -} - -# Centralized version file update -echo "${NEXT_FULL_TAG}" > VERSION -echo "${RAPIDS_BRANCH_NAME}" > RAPIDS_BRANCH - -# Update Java version -NEXT_FULL_JAVA_TAG="${NEXT_SHORT_TAG}.${PATCH_PEP440}" -sed_runner "s/VERSION=\".*\"/VERSION=\"${NEXT_FULL_JAVA_TAG}\"/g" build.sh - -for FILE in pom.xml examples/pom.xml bench/pom.xml; do - sed_runner "/.*/s//${NEXT_FULL_JAVA_TAG}<\/version>/g" "${FILE}" -done - -sed_runner "s| CuVS [[:digit:]]\{2\}\.[[:digit:]]\{2\} | CuVS ${NEXT_SHORT_TAG} |g" README.md - -for FILE in dependencies.yaml conda/environments/*.yaml; do - sed_runner "s/libcuvs==.*/libcuvs==${NEXT_SHORT_TAG}.*/g" "${FILE}" -done - -for FILE in .github/workflows/*.yaml; do - sed_runner "/shared-workflows/ s|@.*|@${RAPIDS_BRANCH_NAME}|g" "${FILE}" - sed_runner "s/:[0-9]*\\.[0-9]*-/:${NEXT_SHORT_TAG}-/g" "${FILE}" -done - -# Update versions in the examples readme file -sed_runner "s/rapidsai\/ci-conda:[\.0-9]*-cuda/rapidsai\/ci-conda:${NEXT_SHORT_TAG}-cuda/g" "examples/README.md" -sed_runner "s/target\/examples-[\.0-9]*-jar/target\/examples-${NEXT_FULL_JAVA_TAG}-jar/g" "examples/README.md" diff --git a/java/cuvs-lucene/ci/test_java.sh b/java/cuvs-lucene/ci/test_java.sh deleted file mode 100755 index db17f1b62c..0000000000 --- a/java/cuvs-lucene/ci/test_java.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -rapids-logger "Check GPU usage" -nvidia-smi - -rapids-logger "Run Java build and tests" - -# TODO: switch to installing pre-built artifacts instead of rebuilding in test jobs -# ref: https://github.com/rapidsai/cuvs/issues/868 -ci/build_java.sh --run-java-tests diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml deleted file mode 100644 index 7a689fe768..0000000000 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-aarch64.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai-nightly -- conda-forge -dependencies: -- c-compiler -- cmake>=3.30.4 -- cuda-cudart-dev -- cuda-nvcc -- cuda-nvtx-dev -- cuda-profiler-api -- cuda-version=12.9 -- cxx-compiler -- gcc_linux-aarch64=14.* -- libcublas-dev -- libcurand-dev -- libcusolver-dev -- libcusparse-dev -- libcuvs==26.10.* -- maven -- ninja -- openjdk=22.* -- sysroot_linux-aarch64==2.28 -name: all_cuda-129_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml deleted file mode 100644 index 562abae6d0..0000000000 --- a/java/cuvs-lucene/conda/environments/all_cuda-129_arch-x86_64.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai-nightly -- conda-forge -dependencies: -- c-compiler -- cmake>=3.30.4 -- cuda-cudart-dev -- cuda-nvcc -- cuda-nvtx-dev -- cuda-profiler-api -- cuda-version=12.9 -- cxx-compiler -- gcc_linux-64=14.* -- libcublas-dev -- libcurand-dev -- libcusolver-dev -- libcusparse-dev -- libcuvs==26.10.* -- maven -- ninja -- openjdk=22.* -- sysroot_linux-64==2.28 -name: all_cuda-129_arch-x86_64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-133_arch-aarch64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-133_arch-aarch64.yaml deleted file mode 100644 index 8ddd2484ce..0000000000 --- a/java/cuvs-lucene/conda/environments/all_cuda-133_arch-aarch64.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai-nightly -- conda-forge -dependencies: -- c-compiler -- cmake>=3.30.4 -- cuda-cudart-dev -- cuda-nvcc -- cuda-nvtx-dev -- cuda-profiler-api -- cuda-version=13.3 -- cxx-compiler -- gcc_linux-aarch64=14.* -- libcublas-dev -- libcurand-dev -- libcusolver-dev -- libcusparse-dev -- libcuvs==26.10.* -- maven -- ninja -- openjdk=22.* -- sysroot_linux-aarch64==2.28 -name: all_cuda-133_arch-aarch64 diff --git a/java/cuvs-lucene/conda/environments/all_cuda-133_arch-x86_64.yaml b/java/cuvs-lucene/conda/environments/all_cuda-133_arch-x86_64.yaml deleted file mode 100644 index 34ee83dd9a..0000000000 --- a/java/cuvs-lucene/conda/environments/all_cuda-133_arch-x86_64.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# This file is generated by `rapids-dependency-file-generator`. -# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. -channels: -- rapidsai-nightly -- conda-forge -dependencies: -- c-compiler -- cmake>=3.30.4 -- cuda-cudart-dev -- cuda-nvcc -- cuda-nvtx-dev -- cuda-profiler-api -- cuda-version=13.3 -- cxx-compiler -- gcc_linux-64=14.* -- libcublas-dev -- libcurand-dev -- libcusolver-dev -- libcusparse-dev -- libcuvs==26.10.* -- maven -- ninja -- openjdk=22.* -- sysroot_linux-64==2.28 -name: all_cuda-133_arch-x86_64 diff --git a/java/cuvs-lucene/dependencies.yaml b/java/cuvs-lucene/dependencies.yaml deleted file mode 100644 index b7c09a93ed..0000000000 --- a/java/cuvs-lucene/dependencies.yaml +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Dependency list for https://github.com/rapidsai/dependency-file-generator -files: - all: - output: conda - matrix: - cuda: ["12.9", "13.3"] - arch: [x86_64, aarch64] - includes: - - cuda - - cuda_version - - java - checks: - output: none - includes: - - checks - java: - output: none - includes: - - cuda - - cuda_version - - java -channels: - - rapidsai-nightly - - conda-forge -dependencies: - checks: - common: - - output_types: [conda, requirements] - packages: - - pre-commit - cuda_version: - specific: - - output_types: conda - matrices: - - matrix: - cuda: "12.0" - packages: - - cuda-version=12.0 - - matrix: - cuda: "12.2" - packages: - - cuda-version=12.2 - - matrix: - cuda: "12.5" - packages: - - cuda-version=12.5 - - matrix: - cuda: "12.8" - packages: - - cuda-version=12.8 - - matrix: - cuda: "12.9" - packages: - - cuda-version=12.9 - - matrix: - cuda: "13.0" - packages: - - cuda-version=13.0 - - matrix: - cuda: "13.1" - packages: - - cuda-version=13.1 - - matrix: - cuda: "13.2" - packages: - - cuda-version=13.2 - - matrix: - cuda: "13.3" - packages: - - cuda-version=13.3 - cuda: - common: - - output_types: [conda] - packages: - - cuda-nvtx-dev - - cuda-cudart-dev - - cuda-profiler-api - - libcublas-dev - - libcurand-dev - - libcusolver-dev - - libcusparse-dev - - libcuvs==26.10.* - java: - common: - - output_types: conda - packages: - - c-compiler - - cmake>=3.30.4 - - cuda-nvcc - - maven - - ninja - - openjdk=22.* - - cxx-compiler - specific: - - output_types: conda - matrices: - - matrix: - arch: x86_64 - packages: - - gcc_linux-64=14.* - - sysroot_linux-64==2.28 - - matrix: - arch: aarch64 - packages: - - gcc_linux-aarch64=14.* - - sysroot_linux-aarch64==2.28 From 4cbf3451b3f47a0f19bb924f5c43224ff08829ca Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 21 Jul 2026 15:44:48 -1000 Subject: [PATCH 096/105] Update build.sh scripts --- build.sh | 19 +++++++++++++++++-- java/cuvs-lucene/build.sh | 25 +++++++++---------------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/build.sh b/build.sh index d5780b481a..7eeb877e44 100755 --- a/build.sh +++ b/build.sh @@ -19,7 +19,7 @@ ARGS=$* # scripts, and that this script resides in the repo dir! REPODIR=$(cd "$(dirname "$0")"; pwd) -VALIDARGS="clean libcuvs python rust go java docs tests bench-ann examples --uninstall -v -g -n --allgpuarch --no-mg --mnmg-tests --no-cpu --cpu-only --no-shared-libs --no-nvtx --show_depr_warn --incl-cache-stats --time -h --run-java-tests" +VALIDARGS="clean libcuvs python rust go java lucene docs tests bench-ann examples --uninstall -v -g -n --allgpuarch --no-mg --mnmg-tests --no-cpu --cpu-only --no-shared-libs --no-nvtx --show_depr_warn --incl-cache-stats --time -h --run-java-tests" HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool=] [--limit-tests=] [--limit-bench-ann=] [--build-metrics=] where is: clean - remove all existing build artifacts and configuration (start over) @@ -29,6 +29,7 @@ HELP="$0 [ ...] [ ...] [--cmake-args=\"\"] [--cache-tool= Date: Wed, 22 Jul 2026 09:18:17 -1000 Subject: [PATCH 097/105] Update CODEOWNERS and update-version.sh --- .github/CODEOWNERS | 3 +++ ci/release/update-version.sh | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b897d49c50..30036b650f 100755 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,6 +11,9 @@ examples/cpp/ @NVIDIA/cuvs-cpp-codeowners java/ @NVIDIA/cuvs-java-codeowners examples/java/ @NVIDIA/cuvs-java-codeowners +#cuvs lucene code owners +java/cuvs-lucene @NVIDIA/cuvs-lucene-write + #python code owners python/ @NVIDIA/cuvs-python-codeowners diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index f36c14f771..5708bece66 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -175,3 +175,11 @@ done sed_runner "s| CuVS [[:digit:]]\{2\}\.[[:digit:]]\{2\} | CuVS ${NEXT_SHORT_TAG} |g" java/README.md sed_runner "s|-[[:digit:]]\{2\}\.[[:digit:]]\{2\}\.[[:digit:]]\{1,2\}\.jar|-${NEXT_FULL_JAVA_TAG}\.jar|g" java/examples/README.md sed_runner "s|/[[:digit:]]\{2\}\.[[:digit:]]\{2\}\.[[:digit:]]\{1,2\}/|/${NEXT_FULL_JAVA_TAG}/|g" java/examples/README.md + +# Update cuvs-lucene (integrated Java subproject; uses its own CUVS_LUCENE# markers) +sed_runner "s/VERSION=\".*\"/VERSION=\"${NEXT_FULL_JAVA_TAG}\"/g" java/cuvs-lucene/build.sh +for FILE in java/cuvs-lucene/pom.xml java/cuvs-lucene/examples/pom.xml java/cuvs-lucene/bench/pom.xml; do + sed_runner "/.*/s//${NEXT_FULL_JAVA_TAG}<\/version>/g" "${FILE}" +done +sed_runner "s|rapidsai/ci-conda:[\.0-9]*-cuda|rapidsai/ci-conda:${NEXT_SHORT_TAG}-cuda|g" java/cuvs-lucene/examples/README.md +sed_runner "s|target/examples-[\.0-9]*-jar|target/examples-${NEXT_FULL_JAVA_TAG}-jar|g" java/cuvs-lucene/examples/README.md From e3595cc8014fa5cc59db25d165d48d78f0a5e326 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 22 Jul 2026 10:27:14 -1000 Subject: [PATCH 098/105] Fix .gitignore --- java/.gitignore | 2 ++ java/cuvs-lucene/.gitignore | 9 --------- 2 files changed, 2 insertions(+), 9 deletions(-) delete mode 100644 java/cuvs-lucene/.gitignore diff --git a/java/.gitignore b/java/.gitignore index 2e98aff7ba..a4da3c4db7 100644 --- a/java/.gitignore +++ b/java/.gitignore @@ -7,3 +7,5 @@ openjdk-22-jextract* /cuvs-java/bin/ /cuvs-java/src/main/java22/com/nvidia/cuvs/internal/panama/ /cuvs-java/*.cag +# cuvs-lucene +/cuvs-lucene/bin/ diff --git a/java/cuvs-lucene/.gitignore b/java/cuvs-lucene/.gitignore deleted file mode 100644 index f7ee48d382..0000000000 --- a/java/cuvs-lucene/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. -# SPDX-License-Identifier: Apache-2.0 - -*.jar -target -**/.DS_Store -bin -.project -cuvs-workdir From 9be9cdca22a0153bf65f0e149eebb8704c5e9d52 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 22 Jul 2026 10:49:54 -1000 Subject: [PATCH 099/105] First stab at CI scripts --- .github/workflows/build.yaml | 33 ++++++++++++++++ .github/workflows/pr.yaml | 36 +++++++++++++++++ .github/workflows/test.yaml | 34 ++++++++++++++++ ci/build_lucene.sh | 76 ++++++++++++++++++++++++++++++++++++ ci/release/update-version.sh | 2 +- ci/test_lucene.sh | 24 ++++++++++++ java/cuvs-lucene/pom.xml | 8 ++-- 7 files changed, 208 insertions(+), 5 deletions(-) create mode 100755 ci/build_lucene.sh create mode 100755 ci/test_lucene.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 804c3cd877..78785a6a00 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -184,6 +184,39 @@ jobs: artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}" file_to_upload: "java/cuvs-java/target/" sha: ${{ inputs.sha }} + lucene-build-matrix: + needs: cpp-build + permissions: + contents: read + uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main + with: + build_type: pull-request + matrix_name: conda-cpp-build + matrix_filter: map(select(.ARCH == "amd64")) + lucene-build: + # Consumes the cuvs-java artifact uploaded by java-build instead of rebuilding cuvs-java. + needs: [lucene-build-matrix, java-build] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.lucene-build-matrix.outputs.matrix) }} + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + arch: "amd64" + date: ${{ inputs.date }} + container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + script: "ci/build_lucene.sh" + artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}" + file_to_upload: "java/cuvs-lucene/target/" + sha: ${{ inputs.sha }} python-build: needs: [cpp-build] permissions: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2154fe74a7..ad0af5eb52 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,6 +25,8 @@ jobs: - rocky8-clib-tests - conda-java-build-and-tests-matrix - conda-java-build-and-tests + - conda-lucene-build-and-tests-matrix + - conda-lucene-build-and-tests - rust-build-matrix - rust-build - go-build-matrix @@ -548,6 +550,40 @@ jobs: script: "ci/test_java.sh" artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}" file_to_upload: "java/cuvs-java/target/" + conda-lucene-build-and-tests-matrix: + needs: [conda-cpp-build, changed-files] + permissions: + contents: read + uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main + # cuvs-lucene lives under java/, so java/cuvs-lucene changes trigger the test_java group. + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_java || fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + with: + build_type: pull-request + matrix_name: conda-cpp-build + matrix_filter: map(select(.ARCH == "amd64")) + conda-lucene-build-and-tests: + # Depends on conda-java-build-and-tests to consume its uploaded cuvs-java artifact + # (installed into the local Maven repo) instead of rebuilding cuvs-java. + needs: [conda-java-build-and-tests, conda-lucene-build-and-tests-matrix] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.conda-lucene-build-and-tests-matrix.outputs.matrix) }} + with: + build_type: pull-request + node_type: "gpu-l4-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + script: "ci/test_lucene.sh" + artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}" + file_to_upload: "java/cuvs-lucene/target/" rust-build-matrix: needs: [conda-cpp-build, changed-files] permissions: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 26739e2deb..61e4aefac5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -104,6 +104,40 @@ jobs: script: "ci/test_java.sh" artifact-name: "cuvs-java-cuda${{ matrix.CUDA_VER }}" file_to_upload: "java/cuvs-java/target/" + conda-lucene-build-and-tests-matrix: + permissions: + contents: read + uses: rapidsai/shared-workflows/.github/workflows/compute-matrix.yaml@main + with: + build_type: ${{ inputs.build_type }} + matrix_name: conda-cpp-build + matrix_filter: map(select(.ARCH == "amd64")) + conda-lucene-build-and-tests: + # Consumes the cuvs-java artifact uploaded by conda-java-build-and-tests instead of + # rebuilding cuvs-java. + needs: [conda-java-build-and-tests, conda-lucene-build-and-tests-matrix] + permissions: + actions: read + contents: read + id-token: write + packages: read + pull-requests: read + secrets: inherit # zizmor: ignore[secrets-inherit] + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@main + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.conda-lucene-build-and-tests-matrix.outputs.matrix) }} + with: + build_type: ${{ inputs.build_type }} + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + node_type: "gpu-l4-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:26.10-cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PY_VER }}" + script: "ci/test_lucene.sh" + artifact-name: "cuvs-lucene-cuda${{ matrix.CUDA_VER }}" + file_to_upload: "java/cuvs-lucene/target/" wheel-tests-cuvs: permissions: actions: read diff --git a/ci/build_lucene.sh b/ci/build_lucene.sh new file mode 100755 index 0000000000..4d1dbad619 --- /dev/null +++ b/ci/build_lucene.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +# TODO: Remove this argument-handling when build and test workflows are separated, +# and test_lucene.sh no longer calls build_lucene.sh +# ref: https://github.com/rapidsai/cuvs/issues/868 +EXTRA_BUILD_ARGS=() +if [[ "${1:-}" == "--run-java-tests" ]]; then + EXTRA_BUILD_ARGS+=("--run-java-tests") +fi + +if [ -e "/opt/conda/etc/profile.d/conda.sh" ]; then + . /opt/conda/etc/profile.d/conda.sh +fi + +rapids-logger "Configuring conda strict channel priority" +conda config --set channel_priority strict + +rapids-logger "Downloading artifacts from previous jobs" +# libcuvs C++ conda package (provides libcuvs.so / libcuvs_c.so used at test time) +CPP_CHANNEL=$(rapids-download-from-github "$(rapids-artifact-name conda_cpp libcuvs cuvs --cuda "$RAPIDS_CUDA_VERSION")") +# cuvs-java jar built by the java-build job (avoids rebuilding cuvs-java here) +CUVS_JAVA_ARTIFACT_DIR=$(rapids-download-from-github "cuvs-java-cuda${RAPIDS_CUDA_VERSION}") + +rapids-logger "Generate Java testing dependencies" + +ENV_YAML_DIR="$(mktemp -d)" + +rapids-dependency-file-generator \ + --output conda \ + --file-key java \ + --prepend-channel "${CPP_CHANNEL}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee "${ENV_YAML_DIR}/env.yaml" + +rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n java + +# Temporarily allow unbound variables for conda activation. +set +u +conda activate java +set -u + +rapids-print-env + +# cuvs-lucene depends on the plain cuvs-java jar (no bundled native libraries), so it loads +# libcuvs through the system loader. Make the conda-provided libcuvs discoverable at test time. +export LD_LIBRARY_PATH="${CONDA_PREFIX}/lib:${LD_LIBRARY_PATH:-}" + +# Install the cuvs-java artifact built by the java-build job into the local Maven repository +# so that cuvs-lucene resolves com.nvidia.cuvs:cuvs-java without rebuilding it. The GAV is +# taken from the artifact's own pom.xml (bundled alongside the jar by java/build.sh). +VERSION=$(grep -oP 'VERSION="\K[^"]+' java/build.sh) +CUVS_JAVA_JAR=$(find "${CUVS_JAVA_ARTIFACT_DIR}" -name "cuvs-java-${VERSION}.jar" | head -1) +CUVS_JAVA_POM=$(find "${CUVS_JAVA_ARTIFACT_DIR}" -name "pom.xml" | head -1) +if [[ -z "${CUVS_JAVA_JAR}" || -z "${CUVS_JAVA_POM}" ]]; then + echo "Error: could not locate cuvs-java-${VERSION}.jar and/or pom.xml in ${CUVS_JAVA_ARTIFACT_DIR}" + exit 1 +fi +rapids-logger "Installing cuvs-java ${VERSION} from ${CUVS_JAVA_JAR}" +mvn install:install-file -Dfile="${CUVS_JAVA_JAR}" -DpomFile="${CUVS_JAVA_POM}" + +EXITCODE=0 +trap "EXITCODE=1" ERR +set +e + +rapids-logger "Run cuVS Lucene build" + +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +export RAPIDS_CUDA_MAJOR + +bash ./build.sh lucene "${EXTRA_BUILD_ARGS[@]}" + +rapids-logger "Build script exiting with value: $EXITCODE" +exit ${EXITCODE} diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 5708bece66..81df4f7dc0 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 ######################## # CUVS Version Updater # diff --git a/ci/test_lucene.sh b/ci/test_lucene.sh new file mode 100755 index 0000000000..04e4c4fab5 --- /dev/null +++ b/ci/test_lucene.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2020-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +EXITCODE=0 +trap "EXITCODE=1" ERR +set +e + +rapids-logger "Check GPU usage" +nvidia-smi + +rapids-logger "Run cuVS Lucene build and tests" + +RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" +export RAPIDS_CUDA_MAJOR + +# TODO: switch to installing pre-built artifacts instead of rebuilding in test jobs +# ref: https://github.com/rapidsai/cuvs/issues/868 +ci/build_lucene.sh --run-java-tests + +rapids-logger "Test script exiting with value: $EXITCODE" +exit ${EXITCODE} diff --git a/java/cuvs-lucene/pom.xml b/java/cuvs-lucene/pom.xml index 70f246d27f..f76b6e6111 100644 --- a/java/cuvs-lucene/pom.xml +++ b/java/cuvs-lucene/pom.xml @@ -1,6 +1,6 @@ @@ -30,10 +30,10 @@ - scm:git:https://github.com/rapidsai/cuvs-lucene.git - scm:git:git@github.com:rapidsai/cuvs-lucene.git + scm:git:https://github.com/rapidsai/cuvs.git + scm:git:git@github.com:rapidsai/cuvs.git HEAD - https://github.com/rapidsai/cuvs-lucene + https://github.com/rapidsai/cuvs From 0cb2cc665b677657bde450b7ca65f61d05c6953b Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 22 Jul 2026 12:10:34 -1000 Subject: [PATCH 100/105] Update README.md files --- java/README.md | 10 +++++ java/cuvs-lucene/README.md | 45 +++++++++++++++++++--- java/cuvs-lucene/examples/README.md | 59 +++++++++++++++++++++++------ 3 files changed, 98 insertions(+), 16 deletions(-) diff --git a/java/README.md b/java/README.md index ff62ff1225..a14995b761 100644 --- a/java/README.md +++ b/java/README.md @@ -59,3 +59,13 @@ times, by passing `-Dtests.iters=10` through the command line. ## Examples A few starter examples of CAGRA, HNSW, and Bruteforce indexing and searching are provided in the `examples` directory. + +## cuVS Lucene + +The [`cuvs-lucene`](cuvs-lucene) directory contains the cuVS integration for +[Apache Lucene](https://github.com/apache/lucene) — a pluggable `KnnVectorsFormat` that enables +GPU-accelerated vector search. It depends on the `cuvs-java` bindings described above. + +`lucene` is a separate top-level build target: build it with `./build.sh lucene` (or +`./build.sh java lucene` to rebuild `cuvs-java` first, or `./build.sh libcuvs java lucene` to build +everything from scratch). See [`cuvs-lucene/README.md`](cuvs-lucene/README.md) for details. diff --git a/java/cuvs-lucene/README.md b/java/cuvs-lucene/README.md index 7e177c457a..b26bf33ef8 100644 --- a/java/cuvs-lucene/README.md +++ b/java/cuvs-lucene/README.md @@ -6,26 +6,61 @@ This is a project for using [cuVS](https://github.com/rapidsai/cuvs), NVIDIA's G This library provides a new [KnnVectorFormat](https://lucene.apache.org/core/10_3_1/core/org/apache/lucene/codecs/KnnVectorsFormat.html) which can be plugged into a Lucene codec. +cuvs-lucene is part of the [cuVS](https://github.com/rapidsai/cuvs) repository and depends on +the `com.nvidia.cuvs:cuvs-java` artifact built by [`../cuvs-java`](../cuvs-java). See +[`../README.md`](../README.md) for an overview of the Java projects. + ## Building ### Prerequisites -- [CUDA 12.0+](https://developer.nvidia.com/cuda-toolkit-archive), -- [Maven 3.9.6+](https://maven.apache.org/download.cgi), +- [CUDA 12.0+](https://developer.nvidia.com/cuda-toolkit-archive) +- [Maven 3.9.6+](https://maven.apache.org/download.cgi) - [JDK 22](https://jdk.java.net/archive/) +- The `libcuvs` C/C++ libraries and the `cuvs-java` bindings (both built from this repo, see below) + +### From the repository root + +`lucene` is a build target of the top-level `build.sh`, alongside `libcuvs` and `java`: ```sh -mvn clean compile package +# Build everything cuvs-lucene needs, from scratch: +./build.sh libcuvs java lucene + +# If libcuvs and cuvs-java are already built and installed, just (re)build cuvs-lucene: +./build.sh lucene ``` -The artifacts would be built and available in the target / folder. +Like `java` (which builds against an already-built `libcuvs`), `lucene` builds against the +already-installed `cuvs-java` artifact, so it can be rebuilt on its own without waiting for +`cuvs-java`. Add `java` (i.e. `./build.sh java lucene`) when `cuvs-java` has changed and needs +rebuilding first. The `cuvs-lucene` jar is produced under `target/`. + +### From this directory + +Once `cuvs-java` has been installed into your local Maven repository (via `../build.sh java`), +you can build cuvs-lucene directly: + +```sh +./build.sh +``` ### Running Tests +Append `--run-java-tests` to run the test suite. `build.sh` sets `LD_LIBRARY_PATH` to the in-tree +`../../cpp/build` so the tests can locate `libcuvs`: + ```sh -export LD_LIBRARY_PATH={ PATH TO YOUR LOCAL libcuvs_c.so }:$LD_LIBRARY_PATH && mvn clean test +# from the repository root +./build.sh lucene --run-java-tests + +# or from this directory +./build.sh --run-java-tests ``` +If your `libcuvs` libraries live elsewhere (e.g. a conda environment), set `LD_LIBRARY_PATH` to +the directory containing the matching `libcuvs_c.so` before running the tests. + ## Contributing > [!NOTE] diff --git a/java/cuvs-lucene/examples/README.md b/java/cuvs-lucene/examples/README.md index 8a9b61a7b7..f4a18caab4 100644 --- a/java/cuvs-lucene/examples/README.md +++ b/java/cuvs-lucene/examples/README.md @@ -2,40 +2,77 @@ This maven project contains basic examples that showcase how `cuvs-lucene` can be used. -## Prerequisites +You can build the prerequisites either inside a RAPIDS conda container (the quickest way to try +things out, since it provides a prebuilt `libcuvs` without compiling the C/C++ libraries) or +entirely from source. + +## Option A — RAPIDS container (no local libcuvs build) + +### Prerequisites - [Docker](https://www.docker.com/) - [Nvidia Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) - A machine with an Nvidia GPU -## Steps +### Steps -If you are currently in this directory (and to be in the `cuvs-lucene's` root directory) do: +Launch the container **from the repository root** (the directory containing `build.sh` and +`dependencies.yaml`) — not from this `examples/` directory — so the whole repo is mounted and +becomes the container's working directory: ```sh -cd .. +docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.10-cuda13.3.0-ubuntu24.04-py3.13 ``` -Then do: +Inside the container you are now at the repository root. Create a conda environment with `libcuvs` +and the Java toolchain (this reads the repo's `dependencies.yaml`, so it must be run from the +repository root), then build `cuvs-java` and `cuvs-lucene` against it: ```sh -docker run --rm --gpus all --pull=always --volume $PWD:$PWD --workdir $PWD -it rapidsai/ci-conda:26.10-cuda13.3.0-ubuntu24.04-py3.13 +rapids-dependency-file-generator --output conda --file-key java \ + --matrix "cuda=13.3;arch=$(arch)" | tee /tmp/java_env.yaml +rapids-mamba-retry env create --yes -f /tmp/java_env.yaml -n java +conda activate java +export CMAKE_PREFIX_PATH=$CONDA_PREFIX +export LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH +./build.sh java lucene ``` -Inside the docker container (and in the `cuvs-lucene's` root directory) do: +## Option B — build everything from source + +### Prerequisites + +- A local [CUDA toolkit](https://developer.nvidia.com/cuda-toolkit-archive) and a machine with an Nvidia GPU +- [Maven 3.9.6+](https://maven.apache.org/download.cgi) and [JDK 22](https://jdk.java.net/archive/) + +### Steps + +From the repository root, build `libcuvs`, the `cuvs-java` bindings, and `cuvs-lucene` (see the +[top-level Java README](../../README.md)), then point `LD_LIBRARY_PATH` at the in-tree libraries: + +```sh +./build.sh libcuvs java lucene +export LD_LIBRARY_PATH=$PWD/cpp/build:$LD_LIBRARY_PATH +``` + +## Build and run the examples + +With either option done (and its `LD_LIBRARY_PATH`/conda env active), build the examples from this +directory: ```sh -./ci/build_java.sh && conda activate java && cd examples +cd java/cuvs-lucene/examples +mvn clean install ``` -To run Accelerated HNSW example do: +To run the Accelerated HNSW example do: ```sh -mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.10.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.AcceleratedHnswExample +java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.10.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.AcceleratedHnswExample ``` To run the Index and Search on GPU example do: ```sh -mvn clean install && java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.10.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.IndexAndSearchonGPUExample +java -Djava.util.logging.config.file=src/main/resources/logging.properties -cp target/examples-26.10.0-jar-with-merged-services.jar com.nvidia.cuvs.lucene.examples.IndexAndSearchonGPUExample ``` From 4b41d07e228fd646a40334ed14b771ff75c8c1fd Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 22 Jul 2026 12:37:44 -1000 Subject: [PATCH 101/105] Comment out cuvs-lucene-write for now --- .github/CODEOWNERS | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 30036b650f..656a87a7b9 100755 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -12,7 +12,8 @@ java/ @NVIDIA/cuvs-java-codeowners examples/java/ @NVIDIA/cuvs-java-codeowners #cuvs lucene code owners -java/cuvs-lucene @NVIDIA/cuvs-lucene-write +# TODO: should we have this? +#java/cuvs-lucene @NVIDIA/cuvs-lucene-write #python code owners python/ @NVIDIA/cuvs-python-codeowners From 150b1dedf3c64531ad70998555a40ff8219e59a2 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 22 Jul 2026 13:05:33 -1000 Subject: [PATCH 102/105] Fix copyright headers in cuvs-lucene folder --- java/cuvs-lucene/bench/pom.xml | 2 +- .../lucene/benchmarks/AcceleratedHnswIndexingBenchmarks.java | 2 +- .../cuvs/lucene/benchmarks/AcceleratedHnswSearchBenchmarks.java | 2 +- .../nvidia/cuvs/lucene/benchmarks/CagraIndexingBenchmarks.java | 2 +- .../nvidia/cuvs/lucene/benchmarks/CagraSearchBenchmarks.java | 2 +- .../src/main/java/com/nvidia/cuvs/lucene/benchmarks/Utils.java | 2 +- java/cuvs-lucene/examples/pom.xml | 2 +- .../com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java | 2 +- .../nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java | 2 +- .../src/main/java/com/nvidia/cuvs/lucene/examples/Utils.java | 2 +- .../main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java | 2 +- .../java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java | 2 +- .../java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java | 2 +- .../java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java | 2 +- .../java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java | 2 +- .../src/main/java/com/nvidia/cuvs/lucene/FieldWriter.java | 2 +- .../main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java | 2 +- .../java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java | 2 +- .../src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java | 2 +- .../src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java | 2 +- .../src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java | 2 +- .../java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java | 2 +- .../java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java | 2 +- .../src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java | 2 +- .../main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java | 2 +- .../java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java | 2 +- .../com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java | 2 +- .../cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java | 2 +- .../cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java | 2 +- .../cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java | 2 +- .../LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java | 2 +- .../LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java | 2 +- .../cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java | 2 +- .../LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java | 2 +- .../LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java | 2 +- .../src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java | 2 +- .../nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java | 2 +- .../cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java | 2 +- .../src/main/java/com/nvidia/cuvs/lucene/package-info.java | 2 +- .../nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java | 2 +- .../java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java | 2 +- .../src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java | 2 +- .../cuvs/lucene/TestCagraToHnswSerializationAndSearch.java | 2 +- ...TestCagraToHnswSerializationAndSearchWithFallbackWriter.java | 2 +- .../cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java | 2 +- .../com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java | 2 +- .../java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java | 2 +- .../src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java | 2 +- .../nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java | 2 +- .../com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java | 2 +- .../test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java | 2 +- .../test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java | 2 +- .../com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java | 2 +- .../cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java | 2 +- .../src/test/java/com/nvidia/cuvs/lucene/TestMerge.java | 2 +- .../com/nvidia/cuvs/lucene/TestMultithreadedCuVSGPUSearch.java | 2 +- .../com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java | 2 +- .../src/test/java/com/nvidia/cuvs/lucene/TestUtils.java | 2 +- 58 files changed, 58 insertions(+), 58 deletions(-) diff --git a/java/cuvs-lucene/bench/pom.xml b/java/cuvs-lucene/bench/pom.xml index d4bc7fbc22..29557d18a9 100644 --- a/java/cuvs-lucene/bench/pom.xml +++ b/java/cuvs-lucene/bench/pom.xml @@ -1,6 +1,6 @@ diff --git a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswIndexingBenchmarks.java b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswIndexingBenchmarks.java index e91e816ce6..08a41f6a5b 100644 --- a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswIndexingBenchmarks.java +++ b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswIndexingBenchmarks.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene.benchmarks; diff --git a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswSearchBenchmarks.java b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswSearchBenchmarks.java index 161c6a3934..32c8d6f970 100644 --- a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswSearchBenchmarks.java +++ b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/AcceleratedHnswSearchBenchmarks.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene.benchmarks; diff --git a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraIndexingBenchmarks.java b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraIndexingBenchmarks.java index 119f95a175..644e852794 100644 --- a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraIndexingBenchmarks.java +++ b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraIndexingBenchmarks.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene.benchmarks; diff --git a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraSearchBenchmarks.java b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraSearchBenchmarks.java index 45f35caff7..9032701095 100644 --- a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraSearchBenchmarks.java +++ b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/CagraSearchBenchmarks.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene.benchmarks; diff --git a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/Utils.java b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/Utils.java index f311c82b49..ceda520be4 100644 --- a/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/Utils.java +++ b/java/cuvs-lucene/bench/src/main/java/com/nvidia/cuvs/lucene/benchmarks/Utils.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene.benchmarks; diff --git a/java/cuvs-lucene/examples/pom.xml b/java/cuvs-lucene/examples/pom.xml index 9b8fcaaff4..82cda4981e 100644 --- a/java/cuvs-lucene/examples/pom.xml +++ b/java/cuvs-lucene/examples/pom.xml @@ -1,6 +1,6 @@ diff --git a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java index b0cf575c5a..d9fbffe9f0 100644 --- a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java +++ b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/AcceleratedHnswExample.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene.examples; diff --git a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java index 834544fec9..9d32032cc7 100644 --- a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java +++ b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/IndexAndSearchonGPUExample.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene.examples; diff --git a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/Utils.java b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/Utils.java index a7a14e22c4..5e27130f6a 100644 --- a/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/Utils.java +++ b/java/cuvs-lucene/examples/src/main/java/com/nvidia/cuvs/lucene/examples/Utils.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene.examples; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java index 2a8ab02f9d..18d2645335 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java index 2de761a78a..9eb5317b64 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUSearchCodec.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java index ccc61eaee1..a38a28289f 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsFormat.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java index c783660d81..4480d7d78f 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsReader.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index e8fb304ece..757ca0573e 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FieldWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FieldWriter.java index 627d931daa..8e2d9a70e2 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FieldWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FieldWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index 07e64fb1ff..56c59b6b0c 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java index 83cebafa1a..1341738df8 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSServiceProvider.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java index b0b6286d2b..7e9f888e32 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUBuiltHnswGraph.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java index 45255da499..d159c5c144 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUFieldWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java index 76cd916b6d..f71c8f6f87 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUIndex.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java index 6a9daae772..e260e26a73 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUKnnFloatVectorQuery.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java index 421b2baa7c..ede8d2555b 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUPerLeafCuVSKnnCollector.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java index cc44526887..a39480f0d6 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java index 3ab1412295..0749b70a28 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexInputInputStream.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java index b5fc120b7d..c8c9f796fa 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/IndexOutputOutputStream.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java index b4c5a33d90..e9f4d6fead 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene101AcceleratedHNSWCodec.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java index 3c42707c45..61c9b41c3a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsFormat.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index 35209ce590..0e558669e7 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java index f2c1aa3719..0b1653bc14 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedCodec.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java index 0f8d960297..d1810bc540 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsFormat.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java index d8a98cc255..e00b2d07b0 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java index 0c7736a037..0705ed0a55 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedCodec.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java index 534390b843..8d599a54ef 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsFormat.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java index 21b4be3f13..a542cf8967 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java index 7635e32323..9caa2a6aa3 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/LuceneProvider.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java index 9e259e27c1..34c95031b2 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/ThreadLocalCuVSResourcesProvider.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java index c55aa7c3e3..e4a20d2b4d 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/Utils.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java index 01f086761a..f14c45183a 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/package-info.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java index d164af5560..cd4aaa6ec8 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWDeletedDocuments.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java index 2273af8c9d..03b5633a87 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWParams.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java index 2de6e660fa..d638180b06 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBackCompat.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java index 96be6c0112..67e6b0ad30 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearch.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java index 04ad20a090..8999d8e81e 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCagraToHnswSerializationAndSearchWithFallbackWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java index d3dc866e9a..10d3ceeec2 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWDeletedDocuments.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java index b0dae38ed1..fa0936e67e 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSAcceleratedHNSWGaps.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java index de868c7c25..7303d04d2f 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSDeletedDocuments.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java index a3e2f59a6f..31958ef0db 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSGaps.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java index c2e2f2f6ae..e630a0c3a1 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedHNSWVectorSearch.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java index 3e77e2edf2..85720c3025 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSRandomizedVectorSearch.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java index 1c75cb2318..2b960bf3cd 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestCuVSVectorsFormat.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java index 483b689c22..32e5c81426 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestGPUSearchParams.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java index 04b22b3d0f..0dbff6a73a 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java index 2428fc5053..9bb140228b 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestLucene99AcceleratedHNSWVectorsFormat.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java index bcc29178b5..a42f55b3fb 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMerge.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMultithreadedCuVSGPUSearch.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMultithreadedCuVSGPUSearch.java index 25dc13b88a..914fa963a8 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMultithreadedCuVSGPUSearch.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestMultithreadedCuVSGPUSearch.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java index 4c3c280993..5140782fc7 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestQuantizedVectorsFormats.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestUtils.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestUtils.java index 05e37f8480..88ca3150fd 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestUtils.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestUtils.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; From 39aa3079302abb7da2e9238a3d2516f369892f9b Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 22 Jul 2026 13:23:49 -1000 Subject: [PATCH 103/105] Fix codespell errors --- .../nvidia/cuvs/lucene/TestIndexOutputOutputStream.java | 8 ++++---- pyproject.toml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java index 0dbff6a73a..39fc9b8c67 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestIndexOutputOutputStream.java @@ -24,8 +24,8 @@ public void testBasic() throws IOException { out.close(); } - try (var indexIn = dir.openInput("test", IOContext.DEFAULT); - var in = new IndexInputInputStream(indexIn)) { + try (var indexInput = dir.openInput("test", IOContext.DEFAULT); + var in = new IndexInputInputStream(indexInput)) { byte[] ba = new byte[6]; assertEquals(6, in.read(ba)); assertArrayEquals(new byte[] {0x56, 0x10, 0x11, 0x12, 0x13, 0x14}, ba); @@ -67,8 +67,8 @@ public void testWithRandom() throws IOException { out.close(); } - try (var indexIn = dir.openInput("test", IOContext.DEFAULT); - var in = new IndexInputInputStream(indexIn); ) { + try (var indexInput = dir.openInput("test", IOContext.DEFAULT); + var in = new IndexInputInputStream(indexInput); ) { int i = 0; while (i < data.length) { if (random.nextBoolean()) { diff --git a/pyproject.toml b/pyproject.toml index 50755ed387..121866a8e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 [tool.ruff] @@ -47,6 +47,6 @@ follow_imports = "skip" skip = "./.git,./.github,./cpp/build,.*egg-info.*,./.mypy_cache,.*_skbuild" # ignore short words, and typename parameters like OffsetT ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b" -ignore-words-list = "inout,unparseable,numer,rocess,thirdparty,couldn,subtile" +ignore-words-list = "inout,unparseable,numer,rocess,thirdparty,couldn,subtile,expectd,atleast" builtin = "clear" quiet-level = 3 From 30250180de1caae50807a82a6f0b966677b5da21 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 23 Jul 2026 16:03:51 -1000 Subject: [PATCH 104/105] Temp: tesing build consistency --- c/include/cuvs/core/c_api.h | 10 ++++++ c/src/core/c_api.cpp | 2 ++ .../nvidia/cuvs/lucene/BogusPropagation.java | 31 +++++++++++++++++++ .../cuvs/lucene/TestBogusAddPropagation.java | 26 ++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/BogusPropagation.java create mode 100644 java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBogusAddPropagation.java diff --git a/c/include/cuvs/core/c_api.h b/c/include/cuvs/core/c_api.h index d2c7c15982..388b1f479c 100644 --- a/c/include/cuvs/core/c_api.h +++ b/c/include/cuvs/core/c_api.h @@ -270,6 +270,16 @@ CUVS_EXPORT cuvsError_t cuvsRMMHostFree(void* ptr, size_t bytes); */ CUVS_EXPORT cuvsError_t cuvsVersionGet(uint16_t* major, uint16_t* minor, uint16_t* patch); +/** + * @brief Bogus function used to verify end-to-end build propagation from the cuVS C + * layer, through the cuvs-java bindings, to cuvs-lucene. Returns the sum of its + * two arguments. + * @param[in] a First addend + * @param[in] b Second addend + * @return The sum @p a + @p b + */ +CUVS_EXPORT int cuvsBogusAdd(int a, int b); + /** * @brief Copy a matrix * diff --git a/c/src/core/c_api.cpp b/c/src/core/c_api.cpp index 0683af6ee3..98c490b5d1 100644 --- a/c/src/core/c_api.cpp +++ b/c/src/core/c_api.cpp @@ -257,6 +257,8 @@ extern "C" cuvsError_t cuvsVersionGet(uint16_t* major, uint16_t* minor, uint16_t return CUVS_SUCCESS; } +extern "C" int cuvsBogusAdd(int a, int b) { return a + b; } + namespace { template void _copy_matrix(cuvsResources_t res, DLManagedTensor* src_managed, DLManagedTensor* dst_managed) diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/BogusPropagation.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/BogusPropagation.java new file mode 100644 index 0000000000..aa53e249a0 --- /dev/null +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/BogusPropagation.java @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nvidia.cuvs.lucene; + +import com.nvidia.cuvs.spi.CuVSProvider; + +/** + * Bogus helper used to verify end-to-end build propagation from the cuVS C layer, through the + * cuvs-java bindings, to cuvs-lucene. It delegates to {@link CuVSProvider#bogusAdd(int, int)}, which + * in turn calls the {@code cuvsBogusAdd} C function. If any layer of the build fails to propagate, + * code that references this method will fail to compile or fail at runtime. + */ +public final class BogusPropagation { + + private BogusPropagation() {} + + /** + * Returns the sum of {@code a} and {@code b} by delegating to the bogus native function exposed + * through cuvs-java. + * + * @param a first addend + * @param b second addend + * @return the sum {@code a + b} + */ + public static int bogusAdd(int a, int b) { + return CuVSProvider.provider().bogusAdd(a, b); + } +} diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBogusAddPropagation.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBogusAddPropagation.java new file mode 100644 index 0000000000..d39eb4a75e --- /dev/null +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBogusAddPropagation.java @@ -0,0 +1,26 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nvidia.cuvs.lucene; + +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.junit.Test; + +/** + * Verifies end-to-end build propagation from the cuVS C layer, through the cuvs-java bindings, to + * cuvs-lucene: the bogus {@code cuvsBogusAdd} C function (added in {@code c/src/core/c_api.cpp}) is + * exposed as {@code CuVSProvider.bogusAdd} in cuvs-java and is called from the cuvs-lucene {@link + * BogusPropagation#bogusAdd(int, int)} method exercised here. If any layer of the build fails to + * propagate, this test will fail to compile or fail at runtime. + */ +@SuppressSysoutChecks(bugUrl = "") +public class TestBogusAddPropagation extends LuceneTestCase { + + @Test + public void testBogusAddReachableFromLucene() { + assertEquals(5, BogusPropagation.bogusAdd(2, 3)); + } +} From 085107c84664cec33a1673beee3a0d978706c5fd Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 24 Jul 2026 18:40:59 -1000 Subject: [PATCH 105/105] Temp: tesing build consistency second attempt --- c/include/cuvs/core/c_api.h | 2 +- c/src/core/c_api.cpp | 2 +- .../main/java/com/nvidia/cuvs/spi/CuVSProvider.java | 12 ++++++++++++ .../main/java22/com/nvidia/cuvs/spi/JDKProvider.java | 5 +++++ .../com/nvidia/cuvs/lucene/BogusPropagation.java | 8 ++++---- .../nvidia/cuvs/lucene/TestBogusAddPropagation.java | 8 ++++---- 6 files changed, 27 insertions(+), 10 deletions(-) diff --git a/c/include/cuvs/core/c_api.h b/c/include/cuvs/core/c_api.h index 388b1f479c..88efe7f189 100644 --- a/c/include/cuvs/core/c_api.h +++ b/c/include/cuvs/core/c_api.h @@ -278,7 +278,7 @@ CUVS_EXPORT cuvsError_t cuvsVersionGet(uint16_t* major, uint16_t* minor, uint16_ * @param[in] b Second addend * @return The sum @p a + @p b */ -CUVS_EXPORT int cuvsBogusAdd(int a, int b); +CUVS_EXPORT int cuvsBogusAddV1(int a, int b); /** * @brief Copy a matrix diff --git a/c/src/core/c_api.cpp b/c/src/core/c_api.cpp index 98c490b5d1..2d98838966 100644 --- a/c/src/core/c_api.cpp +++ b/c/src/core/c_api.cpp @@ -257,7 +257,7 @@ extern "C" cuvsError_t cuvsVersionGet(uint16_t* major, uint16_t* minor, uint16_t return CUVS_SUCCESS; } -extern "C" int cuvsBogusAdd(int a, int b) { return a + b; } +extern "C" int cuvsBogusAddV1(int a, int b) { return a + b; } namespace { template diff --git a/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java b/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java index 60a2928aec..89a5acf880 100644 --- a/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java +++ b/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java @@ -218,6 +218,18 @@ default CagraIndex mergeCagraIndexes(CagraIndex[] indexes, CagraIndexParams merg /** Disables pooled memory on the current device, reverting back to the default setting. */ void resetRMMPooledMemory(); + /** + * Bogus method used to verify end-to-end build propagation from the cuVS C layer, through the + * cuvs-java bindings, to cuvs-lucene. Returns the sum of its two arguments. + * + * @param a first addend + * @param b second addend + * @return the sum {@code a + b} + */ + default int bogusAddV1(int a, int b) { + throw new UnsupportedOperationException("bogusAddV1 is not supported by this provider"); + } + /** Retrieves the system-wide provider. */ static CuVSProvider provider() { return CuVSServiceProvider.Holder.INSTANCE; diff --git a/java/cuvs-java/src/main/java22/com/nvidia/cuvs/spi/JDKProvider.java b/java/cuvs-java/src/main/java22/com/nvidia/cuvs/spi/JDKProvider.java index c9cff1e272..29d73d89b3 100644 --- a/java/cuvs-java/src/main/java22/com/nvidia/cuvs/spi/JDKProvider.java +++ b/java/cuvs-java/src/main/java22/com/nvidia/cuvs/spi/JDKProvider.java @@ -473,6 +473,11 @@ public void resetRMMPooledMemory() { checkCuVSError(cuvsRMMMemoryResourceResetInvoker.apply(), "cuvsRMMMemoryResourceReset"); } + @Override + public int bogusAddV1(int a, int b) { + return cuvsBogusAddV1(a, b); + } + @Override public CuVSMatrix.Builder newHostMatrixBuilder( long size, long columns, CuVSMatrix.DataType dataType) { diff --git a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/BogusPropagation.java b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/BogusPropagation.java index aa53e249a0..0a7c1d39ae 100644 --- a/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/BogusPropagation.java +++ b/java/cuvs-lucene/src/main/java/com/nvidia/cuvs/lucene/BogusPropagation.java @@ -9,8 +9,8 @@ /** * Bogus helper used to verify end-to-end build propagation from the cuVS C layer, through the - * cuvs-java bindings, to cuvs-lucene. It delegates to {@link CuVSProvider#bogusAdd(int, int)}, which - * in turn calls the {@code cuvsBogusAdd} C function. If any layer of the build fails to propagate, + * cuvs-java bindings, to cuvs-lucene. It delegates to {@link CuVSProvider#bogusAddV1(int, int)}, which + * in turn calls the {@code cuvsBogusAddV1} C function. If any layer of the build fails to propagate, * code that references this method will fail to compile or fail at runtime. */ public final class BogusPropagation { @@ -25,7 +25,7 @@ private BogusPropagation() {} * @param b second addend * @return the sum {@code a + b} */ - public static int bogusAdd(int a, int b) { - return CuVSProvider.provider().bogusAdd(a, b); + public static int bogusAddV1(int a, int b) { + return CuVSProvider.provider().bogusAddV1(a, b); } } diff --git a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBogusAddPropagation.java b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBogusAddPropagation.java index d39eb4a75e..2f096c65fe 100644 --- a/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBogusAddPropagation.java +++ b/java/cuvs-lucene/src/test/java/com/nvidia/cuvs/lucene/TestBogusAddPropagation.java @@ -11,9 +11,9 @@ /** * Verifies end-to-end build propagation from the cuVS C layer, through the cuvs-java bindings, to - * cuvs-lucene: the bogus {@code cuvsBogusAdd} C function (added in {@code c/src/core/c_api.cpp}) is - * exposed as {@code CuVSProvider.bogusAdd} in cuvs-java and is called from the cuvs-lucene {@link - * BogusPropagation#bogusAdd(int, int)} method exercised here. If any layer of the build fails to + * cuvs-lucene: the bogus {@code cuvsBogusAddV1} C function (added in {@code c/src/core/c_api.cpp}) is + * exposed as {@code CuVSProvider.bogusAddV1} in cuvs-java and is called from the cuvs-lucene {@link + * BogusPropagation#bogusAddV1(int, int)} method exercised here. If any layer of the build fails to * propagate, this test will fail to compile or fail at runtime. */ @SuppressSysoutChecks(bugUrl = "") @@ -21,6 +21,6 @@ public class TestBogusAddPropagation extends LuceneTestCase { @Test public void testBogusAddReachableFromLucene() { - assertEquals(5, BogusPropagation.bogusAdd(2, 3)); + assertEquals(5, BogusPropagation.bogusAddV1(2, 3)); } }