diff --git a/.gitignore b/.gitignore index e299e15e43c4..fd80f9edf700 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ target bin +dependency-reduced-pom.xml .project .settings .classpath diff --git a/nifi-assembly/NOTICE b/nifi-assembly/NOTICE index 3c66982fdc92..455f47323cbd 100644 --- a/nifi-assembly/NOTICE +++ b/nifi-assembly/NOTICE @@ -1997,6 +1997,11 @@ The following binary components are provided under the Apache Software License v the terms of a BSD style license. The original software and related information is available at http://www.jcraft.com/jsch/. + + (ASLv2) DataStax Java Driver for Apache Cassandra - Core + The following NOTICE information applies: + DataStax Java Driver for Apache Cassandra - Core + Copyright (C) 2012-2017 DataStax Inc. (ASLv2) bytebuffer-collections The following NOTICE information applies: bytebuffer-collections diff --git a/nifi-assembly/pom.xml b/nifi-assembly/pom.xml index e3cdf316ca57..b69e8b698c54 100644 --- a/nifi-assembly/pom.xml +++ b/nifi-assembly/pom.xml @@ -1202,5 +1202,44 @@ language governing permissions and limitations under the License. --> false + + include-cql + + + false + + allProfiles + + + + + org.apache.nifi + nifi-cql-nar + 2.12.0-SNAPSHOT + nar + + + org.apache.nifi + nifi-cql-services-api-nar + 2.12.0-SNAPSHOT + nar + + + org.apache.nifi + nifi-cassandra-session-provider-service-nar + 2.12.0-SNAPSHOT + nar + + + org.apache.nifi + nifi-scylladb-session-provider-service-nar + 2.12.0-SNAPSHOT + nar + + + diff --git a/nifi-code-coverage/pom.xml b/nifi-code-coverage/pom.xml index 1be8e671e662..ab7cd13d85e4 100644 --- a/nifi-code-coverage/pom.xml +++ b/nifi-code-coverage/pom.xml @@ -695,6 +695,21 @@ nifi-box-services-api 2.12.0-SNAPSHOT + + org.apache.nifi + nifi-cql-processors + 2.12.0-SNAPSHOT + + + org.apache.nifi + nifi-cassandra-session-provider-service + 2.12.0-SNAPSHOT + + + org.apache.nifi + nifi-cql-services-api + 2.12.0-SNAPSHOT + org.apache.nifi nifi-cdc-api diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service-nar/pom.xml b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service-nar/pom.xml new file mode 100644 index 000000000000..96897195adb2 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service-nar/pom.xml @@ -0,0 +1,51 @@ + + + + + nifi-cql-bundle + org.apache.nifi + 2.12.0-SNAPSHOT + + 4.0.0 + + nifi-cassandra-session-provider-service-nar + nar + + + + + + com.google.guava + guava + provided + + + + + + + org.apache.nifi + nifi-cql-services-api-nar + 2.12.0-SNAPSHOT + nar + + + org.apache.nifi + nifi-cassandra-session-provider-service + 2.12.0-SNAPSHOT + + + \ No newline at end of file diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service-nar/src/main/resources/META-INF/LICENSE b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service-nar/src/main/resources/META-INF/LICENSE new file mode 100644 index 000000000000..c567ce25dfd7 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service-nar/src/main/resources/META-INF/LICENSE @@ -0,0 +1,352 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +APACHE NIFI SUBCOMPONENTS: + +The Apache NiFi project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +This product bundles 'libffi' which is available under an MIT style license. + libffi - Copyright (c) 1996-2014 Anthony Green, Red Hat, Inc and others. + see https://github.com/java-native-access/jna/blob/master/native/libffi/LICENSE + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + ``Software''), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +This product bundles 'asm' which is available under a 3-Clause BSD style license. +For details see http://asm.ow2.org/asmdex-license.html + + Copyright (c) 2012 France Télécom + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + + The binary distribution of this product bundles 'Bouncy Castle JDK 1.5' + under an MIT style license. + + Copyright (c) 2000 - 2015 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +The binary distribution of this product bundles 'JNR x86asm' under an MIT +style license. + + Copyright (C) 2010 Wayne Meissner + Copyright (c) 2008-2009, Petr Kobalicek + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + +This product bundles 'logback' which is dual-licensed under the EPL v1.0 +and the LGPL 2.1. + + Logback: the reliable, generic, fast and flexible logging framework. + + Copyright (C) 1999-2017, QOS.ch. All rights reserved. + + This program and the accompanying materials are dual-licensed under + either the terms of the Eclipse Public License v1.0 as published by + the Eclipse Foundation or (per the licensee's choosing) under the + terms of the GNU Lesser General Public License version 2.1 as + published by the Free Software Foundation. + +The binary distribution of this product bundles 'ANTLR 3' which is available +under a "3-clause BSD" license. For details see http://www.antlr.org/license.html + + Copyright (c) 2012 Terence Parr and Sam Harwell + All rights reserved. + Redistribution and use in source and binary forms, with or without modification, are permitted + provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this list of + conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright notice, this list of + conditions and the following disclaimer in the documentation and/or other materials + provided with the distribution. + + Neither the name of the author nor the names of its contributors may be used to endorse + or promote products derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service-nar/src/main/resources/META-INF/NOTICE b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service-nar/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..8e2e0dcfb324 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service-nar/src/main/resources/META-INF/NOTICE @@ -0,0 +1,313 @@ +nifi-cassandra-services-nar +Copyright 2016-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +****************** +Apache Software License v2 +****************** + +The following binary components are provided under the Apache Software License v2 + + (ASLv2) DataStax Java Driver for Apache Cassandra - Core + The following NOTICE information applies: + DataStax Java Driver for Apache Cassandra - Core + Copyright (C) 2012-2017 DataStax Inc. + + (ASLv2) Jackson JSON processor + The following NOTICE information applies: + # Jackson JSON processor + + Jackson is a high-performance, Free/Open Source JSON processing library. + It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has + been in development since 2007. + It is currently developed by a community of developers, as well as supported + commercially by FasterXML.com. + + ## Licensing + + Jackson core and extension components may licensed under different licenses. + To find the details that apply to this artifact see the accompanying LICENSE file. + For more information, including possible other licensing options, contact + FasterXML.com (http://fasterxml.com). + + ## Credits + + A list of contributors may be found from CREDITS file, which is included + in some artifacts (usually source distributions); but is always available + from the source code management (SCM) system project uses. + + (ASLv2) Apache Commons Codec + The following NOTICE information applies: + Apache Commons Codec + Copyright 2002-2014 The Apache Software Foundation + + src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java + contains test data from http://aspell.net/test/orig/batch0.tab. + Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + + =============================================================================== + + The content of package org.apache.commons.codec.language.bm has been translated + from the original php source code available at http://stevemorse.org/phoneticinfo.htm + with permission from the original authors. + Original source copyright: + Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + + (ASLv2) Apache Commons Lang + The following NOTICE information applies: + Apache Commons Lang + Copyright 2001-2017 The Apache Software Foundation + + This product includes software from the Spring Framework, + under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + + (ASLv2) Guava + The following NOTICE information applies: + Guava + Copyright 2015 The Guava Authors + + (ASLv2) JSON-SMART + The following NOTICE information applies: + Copyright 2011 JSON-SMART authors + + (ASLv2) Dropwizard Metrics + The following NOTICE information applies: + Copyright (c) 2010-2013 Coda Hale, Yammer.com + + This product includes software developed by Coda Hale and Yammer, Inc. + + This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, + LongAdder), which was released with the following comments: + + Written by Doug Lea with assistance from members of JCP JSR-166 + Expert Group and released to the public domain, as explained at + http://creativecommons.org/publicdomain/zero/1.0/ + + (ASLv2) The Netty Project + The following NOTICE information applies: + Copyright 2014 The Netty Project + ------------------------------------------------------------------------------- + This product contains the extensions to Java Collections Framework which has + been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + + This product contains a modified version of Robert Harder's Public Domain + Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + + This product contains a modified portion of 'Webbit', an event based + WebSocket and HTTP server, which can be obtained at: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + + This product contains a modified portion of 'SLF4J', a simple logging + facade for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + + This product contains a modified portion of 'Apache Harmony', an open source + Java SE, which can be obtained at: + + * LICENSE: + * license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: + * http://archive.apache.org/dist/harmony/ + + This product contains a modified portion of 'jbzip2', a Java bzip2 compression + and decompression library written by Matthew J. Francis. It can be obtained at: + + * LICENSE: + * license/LICENSE.jbzip2.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jbzip2/ + + This product contains a modified portion of 'libdivsufsort', a C API library to construct + the suffix array and the Burrows-Wheeler transformed string for any input string of + a constant-size alphabet written by Yuta Mori. It can be obtained at: + + * LICENSE: + * license/LICENSE.libdivsufsort.txt (MIT License) + * HOMEPAGE: + * https://github.com/y-256/libdivsufsort + + This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, + which can be obtained at: + + * LICENSE: + * license/LICENSE.jctools.txt (ASL2 License) + * HOMEPAGE: + * https://github.com/JCTools/JCTools + + This product optionally depends on 'JZlib', a re-implementation of zlib in + pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + + This product optionally depends on 'Compress-LZF', a Java library for encoding and + decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: + + * LICENSE: + * license/LICENSE.compress-lzf.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/ning/compress + + This product optionally depends on 'lz4', a LZ4 Java compression + and decompression library written by Adrien Grand. It can be obtained at: + + * LICENSE: + * license/LICENSE.lz4.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jpountz/lz4-java + + This product optionally depends on 'lzma-java', a LZMA Java compression + and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.lzma-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jponge/lzma-java + + This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression + and decompression library written by William Kinney. It can be obtained at: + + * LICENSE: + * license/LICENSE.jfastlz.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jfastlz/ + + This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data + interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/protobuf + + This product optionally depends on 'Bouncy Castle Crypto APIs' to generate + a temporary self-signed X.509 certificate when the JVM does not provide the + equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + + This product optionally depends on 'Snappy', a compression library produced + by Google Inc, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/snappy + + This product optionally depends on 'JBoss Marshalling', an alternative Java + serialization API, which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://www.jboss.org/jbossmarshalling + + This product optionally depends on 'Caliper', Google's micro- + benchmarking framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.caliper.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/google/caliper + + This product optionally depends on 'Apache Log4J', a logging framework, which + can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + + This product optionally depends on 'Aalto XML', an ultra-high performance + non-blocking XML processor, which can be obtained at: + + * LICENSE: + * license/LICENSE.aalto-xml.txt (Apache License 2.0) + * HOMEPAGE: + * http://wiki.fasterxml.com/AaltoHome + + This product contains a modified version of 'HPACK', a Java implementation of + the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: + + * LICENSE: + * license/LICENSE.hpack.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/twitter/hpack + + This product contains a modified portion of 'Apache Commons Lang', a Java library + provides utilities for the java.lang API, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-lang.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/proper/commons-lang/ + + This product contains a forked and modified version of Tomcat Native + + * LICENSE: + * ASL2 + * HOMEPAGE: + * http://tomcat.apache.org/native-doc/ + * https://svn.apache.org/repos/asf/tomcat/native/ + + (ASLv2) Objenesis + The following NOTICE information applies: + Objenesis + Copyright 2006-2013 Joe Walnes, Henri Tremblay, Leonardo Mesquita + +************************ +Eclipse Public License 1.0 +************************ + +The following binary components are provided under the Eclipse Public License 1.0. See project link for details. + + (EPL 2.0)(GPL 2)(LGPL 2.1) JNR Posix ( jnr.posix ) https://github.com/jnr/jnr-posix/blob/master/LICENSE.txt + (EPL 1.0)(LGPL 2.1) Logback Classic (ch.qos.logback:logback-classic:jar:1.2.6 - http://logback.qos.ch/) + (EPL 1.0)(LGPL 2.1) Logback Core (ch.qos.logback:logback-core:jar:1.2.6 - http://logback.qos.ch/) + +************************ +Test-Only Resources +************************ + +The following are not distributed as part of this module's binary (.nar) build, but are present in the +source tree under nifi-cassandra-session-provider-service/src/test/resources/ and are copied, with +explanatory comments trimmed, from the stock cassandra.yaml shipped in the corresponding Apache Cassandra +release: + + - cassandra-base-config/cassandra.yaml (from Apache Cassandra 5.0) + - cassandra-auth-config-4.1/cassandra.yaml (from Apache Cassandra 4.1) + - cassandra-auth-config-3.11/cassandra.yaml (from Apache Cassandra 3.11) + + (ASLv2) Apache Cassandra + The following NOTICE information applies: + Apache Cassandra + Copyright 2009- The Apache Software Foundation + + This product includes software developed by The Apache Software + Foundation (http://www.apache.org/). diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/pom.xml b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/pom.xml new file mode 100644 index 000000000000..03ecf7d48ab3 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/pom.xml @@ -0,0 +1,107 @@ + + + + + nifi-cql-bundle + org.apache.nifi + 2.12.0-SNAPSHOT + + 4.0.0 + + nifi-cassandra-session-provider-service + jar + + + 4.19.3 + + + + + org.apache.nifi + nifi-api + + + org.apache.nifi + nifi-utils + + + org.apache.nifi + nifi-cql-services-api + 2.12.0-SNAPSHOT + provided + + + + org.apache.nifi + nifi-cql-session-provider-shared + 2.12.0-SNAPSHOT + + + + + org.apache.cassandra + java-driver-core + ${driver.version} + + + + + org.apache.nifi + nifi-ssl-context-service-api + + + org.apache.nifi + nifi-framework-api + + + + org.apache.nifi + nifi-mock + test + + + + org.testcontainers + testcontainers-cassandra + test + + + org.testcontainers + testcontainers-junit-jupiter + ${testcontainers.version} + test + + + org.apache.nifi + nifi-security-cert-builder + 2.12.0-SNAPSHOT + test + + + org.apache.nifi + nifi-ssl-context-service + 2.12.0-SNAPSHOT + test + + + org.apache.nifi + nifi-cql-it-common + 2.12.0-SNAPSHOT + tests + test + + + diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService new file mode 100644 index 000000000000..c5b134d343f5 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/main/resources/META-INF/services/org.apache.nifi.controller.ControllerService @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.nifi.service.cassandra.CassandraCQLExecutionService \ No newline at end of file diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraConfigOverrides.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraConfigOverrides.java new file mode 100644 index 000000000000..4fa76296f03b --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraConfigOverrides.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.UnaryOperator; + +/** + * Patches the shared {@code cassandra-base-config/cassandra.yaml} at test runtime rather than requiring a + * full near-duplicate copy of that ~250-line file per IT scenario. {@code CassandraContainer# + * withConfigurationOverride()} only accepts a classpath resource directory, so the patched result is + * written to a temp file and copied into the container directly (as {@code /etc/cassandra/cassandra.yaml}) + * with {@code withCopyFileToContainer()} instead. + */ +final class CassandraConfigOverrides { + + private static final String BASE_CONFIG_RESOURCE = "cassandra-base-config/cassandra.yaml"; + + static final String CONTAINER_CASSANDRA_YAML_PATH = "/etc/cassandra/cassandra.yaml"; + + private CassandraConfigOverrides() { + } + + static Path writePatchedConfig(final UnaryOperator patch) { + final String base = readBaseConfig(); + final String patched = patch.apply(base); + if (patched.equals(base)) { + throw new IllegalStateException("Patch did not match anything in the base Cassandra config"); + } + + try { + final Path path = Files.createTempFile("cassandra", ".yaml"); + path.toFile().deleteOnExit(); + Files.writeString(path, patched, StandardCharsets.UTF_8); + return path; + } catch (final IOException e) { + throw new UncheckedIOException(e); + } + } + + private static String readBaseConfig() { + try (InputStream in = CassandraConfigOverrides.class.getClassLoader().getResourceAsStream(BASE_CONFIG_RESOURCE)) { + if (in == null) { + throw new IllegalStateException("Missing classpath resource " + BASE_CONFIG_RESOURCE); + } + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } catch (final IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraContainerLimits.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraContainerLimits.java new file mode 100644 index 000000000000..1947e29e8243 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraContainerLimits.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra; + +import org.apache.nifi.service.cql.it.DockerUtils; +import org.testcontainers.cassandra.CassandraContainer; + +/** + * The resource ceiling every Cassandra container in this package runs under, so no container in an + * integration run can size itself against the whole host. This is the Cassandra counterpart to ScyllaDB's + * {@code ScyllaContainerLimits}, and simpler than it: Cassandra needs no equivalent of ScyllaDB's Seastar + * memory and commitlog flags, since the JVM reads the cgroup limit itself. + * + *

The CPU allowance is what this module's integration time is sensitive to, not the memory ceiling. + * Capping containers at 2 cores cost the module roughly 30 seconds a run, most of it in the SSL/auth + * verification suite, which pays Cassandra's CPU-bound startup for a specially-configured container. Measured + * on that suite back when it booted a container per scenario: 57.1s at 2 cores, 36.7s at {@value #CPUS}, + * against 35.7s with no cap at all - so this allowance gives back essentially all of it. Memory turned out + * not to matter over the range tried: at {@value #CPUS} cores the same suite ran 36.8s with this + * {@value #MEMORY_GB} GB ceiling and 36.7s with 3 GB, so the tighter ceiling is kept. + * + *

{@value #CPUS} is an allowance rather than a reservation - it bounds a container that would otherwise + * see every core on the host, and containers here start one at a time. + */ +final class CassandraContainerLimits { + + private static final long CPUS = 3; + + private static final long MEMORY_GB = 2; + + private CassandraContainerLimits() { + } + + /** + * Caps the container's host resources. + * + * @param container the container to constrain, before it is started + * @return the same container, for chaining + */ + static CassandraContainer apply(final CassandraContainer container) { + return container.withCreateContainerCmdModifier(DockerUtils.createMemoryLimits(CPUS, MEMORY_GB)); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraCrudIT.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraCrudIT.java new file mode 100644 index 000000000000..f39159e2a303 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraCrudIT.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra; + +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.service.cql.it.AbstractCqlCrudIT; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.BeforeParameterizedClassInvocation; +import org.junit.jupiter.params.Parameter; +import org.junit.jupiter.params.ParameterizedClass; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * CRUD coverage from {@link AbstractCqlCrudIT} run against a real Cassandra container. Runs against every + * supported Cassandra major version when {@code -DTEST_CASSANDRA_OLDER_VERSIONS=true} is set; otherwise + * only the current major version runs (see {@link CassandraTestVersions}). + * + *

The container itself belongs to {@link SharedCassandraCluster}, not to this class - see there for what + * sharing one costs and requires. This suite owns the {@code testspace} keyspace that {@code init.cql} + * creates at container start. + */ +@ParameterizedClass +@MethodSource("org.apache.nifi.service.cassandra.CassandraTestVersions#allVersions") +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class CassandraCrudIT extends AbstractCqlCrudIT { + + private static final String KEYSPACE = "testspace"; + + // A @ParameterizedClass combined with @TestInstance(PER_CLASS) requires field injection rather than + // constructor injection, so this field is what JUnit populates per invocation (per version value). + @Parameter + private String version; + + // Runs once per invocation of this parameterized class (once per version value), before any @Test + // methods. Also responsible for calling initializeSessionProvider() directly (rather than relying on + // an inherited @BeforeAll) since @BeforeAll - even inherited from AbstractCqlCrudIT - runs BEFORE this + // method on a @ParameterizedClass leaf, not after; see AbstractCqlCrudIT's class javadoc. + @BeforeParameterizedClassInvocation + void attachToCluster(final String version) throws Exception { + initializeSessionProvider(SharedCassandraCluster.forVersion(version).connectionInfo(KEYSPACE)); + } + + @Override + protected CQLExecutionService newSessionProvider() { + return new CassandraCQLExecutionService(); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraRecordFieldTypeIT.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraRecordFieldTypeIT.java new file mode 100644 index 000000000000..6462c4a72c82 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraRecordFieldTypeIT.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra; + +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.service.cql.it.AbstractCqlRecordFieldTypeIT; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestInstance; + +/** + * Type-coverage tests from {@link AbstractCqlRecordFieldTypeIT} run against a real Cassandra container. + * Single fixed version rather than a parameterized matrix, since the behavior under test (record field type + * <-> CQL type conversion) doesn't vary across Cassandra major versions the way connection/auth + * handling can. + * + *

The container itself belongs to {@link SharedCassandraCluster}, not to this class: pinning to + * {@link SharedCassandraCluster#PINNED_VERSION} means this suite lands on the same instance the + * version-parameterized suites use for that version rather than starting one of its own. It owns the + * {@code type_coverage} keyspace, in which it creates a table per type under test. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class CassandraRecordFieldTypeIT extends AbstractCqlRecordFieldTypeIT { + + private static final String KEYSPACE = "type_coverage"; + + @BeforeAll + void attachToCluster() throws Exception { + final SharedCassandraCluster cluster = SharedCassandraCluster.forVersion(SharedCassandraCluster.PINNED_VERSION); + cluster.createKeyspace(KEYSPACE); + + initializeSessionProvider(cluster.connectionInfo(KEYSPACE)); + } + + @Override + protected CQLExecutionService newSessionProvider() { + return new CassandraCQLExecutionService(); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraSecureVerificationIT.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraSecureVerificationIT.java new file mode 100644 index 000000000000..33900eb1995f --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraSecureVerificationIT.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra; + +import com.datastax.oss.driver.api.core.CqlSession; +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.service.cql.it.AbstractCqlSecureVerificationIT; +import org.junit.jupiter.api.TestInstance; +import org.testcontainers.cassandra.CassandraContainer; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.MountableFile; + +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.cert.X509Certificate; +import java.util.function.UnaryOperator; + +/** + * {@link AbstractCqlSecureVerificationIT} against a single Cassandra 5.0 container configured with + * {@code PasswordAuthenticator} and one-way {@code client_encryption_options} in {@code optional: true} + * mode, so the {@code init.cql} bootstrap still runs over plaintext. The {@code admin} role is created once + * afterwards through the built-in {@code cassandra} superuser. A single fixed version rather than a matrix: + * what is exercised here is the driver's auth and SSL wiring, not anything version-specific. + */ +@Testcontainers +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class CassandraSecureVerificationIT extends AbstractCqlSecureVerificationIT { + + private static final String CASSANDRA_VERSION = "5.0"; + + private static final int CQL_PORT = 9042; + + private static final String KEYSTORE_CONTAINER_PATH = "/etc/cassandra/.keystore"; + + // The disabled client_encryption_options block as it appears in cassandra-base-config/cassandra.yaml. + private static final String DISABLED_ENCRYPTION_BLOCK = + "client_encryption_options:\n\n enabled: false\n\n keystore: conf/.keystore\n\n require_client_auth: false\n"; + + @Override + protected CQLExecutionService newSessionProvider() { + return new CassandraCQLExecutionService(); + } + + @Override + protected SecureServer startSecureServer(final KeyPair serverKeyPair, final X509Certificate serverCertificate, + final String adminRole, final String adminPassword) throws Exception { + final SslKeyStoreCredentials serverKeyStore = writeKeyStore(serverKeyPair.getPrivate(), serverCertificate, "server"); + + final CassandraContainer container = CassandraContainerLimits.apply(new CassandraContainer("cassandra:" + CASSANDRA_VERSION)) + .withCopyFileToContainer(MountableFile.forHostPath(writePatchedConfig(serverKeyStore)), + CassandraConfigOverrides.CONTAINER_CASSANDRA_YAML_PATH) + .withCopyFileToContainer(MountableFile.forHostPath(serverKeyStore.path()), KEYSTORE_CONTAINER_PATH) + .withInitScript("init.cql"); + container.withExposedPorts(CQL_PORT); + container.start(); + + // init.cql runs over the optional-plaintext listener as the built-in "cassandra" superuser, which + // Testcontainers also uses; reuse it once to create the real, generated-password admin role. + try (CqlSession bootstrapSession = CqlSession.builder() + .addContactPoint(container.getContactPoint()) + .withLocalDatacenter(LOCAL_DATACENTER) + .withAuthCredentials(container.getUsername(), container.getPassword()) + .build()) { + bootstrapSession.execute(String.format( + "CREATE ROLE %s WITH PASSWORD = '%s' AND LOGIN = true", adminRole, adminPassword)); + } + + final String contactPoint = container.getContainerIpAddress() + ":" + container.getMappedPort(CQL_PORT); + return new SecureServer(contactPoint, container::stop); + } + + /** + * Patches {@code cassandra-base-config/cassandra.yaml} to switch on {@code PasswordAuthenticator} and an + * enabled, one-way, optional {@code client_encryption_options} pointing at the mounted server keystore - + * rather than checking in a near-duplicate of that ~250-line file. + */ + private static Path writePatchedConfig(final SslKeyStoreCredentials serverKeyStore) { + final UnaryOperator enableAuth = requirePatched("authenticator", + config -> config.replace("authenticator: AllowAllAuthenticator", "authenticator: PasswordAuthenticator")); + + final String enabledEncryptionBlock = "client_encryption_options:\n" + + " enabled: true\n" + + " optional: true\n" + + " keystore: " + KEYSTORE_CONTAINER_PATH + "\n" + + " keystore_password: " + serverKeyStore.password() + "\n" + + " require_client_auth: false\n" + + " store_type: " + STORE_TYPE + "\n"; + final UnaryOperator enableEncryption = requirePatched("client_encryption_options", + config -> config.replace(DISABLED_ENCRYPTION_BLOCK, enabledEncryptionBlock)); + + return CassandraConfigOverrides.writePatchedConfig(config -> enableEncryption.apply(enableAuth.apply(config))); + } + + private static UnaryOperator requirePatched(final String what, final UnaryOperator patch) { + return config -> { + final String patched = patch.apply(config); + if (patched.equals(config)) { + throw new IllegalStateException("The " + what + " patch matched nothing in the base Cassandra config"); + } + return patched; + }; + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraTestVersions.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraTestVersions.java new file mode 100644 index 000000000000..b08e7fe1c975 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CassandraTestVersions.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra; + +import java.util.stream.Stream; + +/** + * Supplies the Cassandra major versions that the {@code @ParameterizedClass} integration tests in this + * package run against. Cassandra 3.11 and 4.1 containers are slow to start and are no longer receiving + * upstream support, so they're only exercised when the {@code TEST_CASSANDRA_OLDER_VERSIONS} system + * property is set to {@code true}; otherwise only the current major version (5.0) runs. + */ +final class CassandraTestVersions { + + private static final String RUN_OLDER_VERSIONS_PROPERTY = "TEST_CASSANDRA_OLDER_VERSIONS"; + + /** + * The current major version, and the only one exercised unless {@code TEST_CASSANDRA_OLDER_VERSIONS} is + * set. Suites whose behaviour does not vary by release pin to this rather than declaring a version of + * their own, so that they share {@code SharedCassandraCluster}'s container for it instead of silently + * starting a second one the day this is bumped. + */ + static final String CURRENT_VERSION = "5.0"; + + private CassandraTestVersions() { + } + + static Stream allVersions() { + return Boolean.getBoolean(RUN_OLDER_VERSIONS_PROPERTY) + ? Stream.of("3.11", "4.1", CURRENT_VERSION) + : Stream.of(CURRENT_VERSION); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CqlConsistencyLevelTest.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CqlConsistencyLevelTest.java new file mode 100644 index 000000000000..a6b00d5e4bf8 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CqlConsistencyLevelTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra; + +import com.datastax.oss.driver.api.core.DefaultConsistencyLevel; +import org.apache.nifi.service.cql.api.constants.CqlConsistencyLevel; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.util.Arrays; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; + +/** + * Pins {@link CqlConsistencyLevel}'s values against the DataStax driver's own enum. + * + *

{@code CqlConsistencyLevel} deliberately declares these names itself rather than deriving them from the + * driver, so that {@code nifi-cql-services-api} - the backend-agnostic contract every other module compiles + * against - carries no dependency on a database client library. The cost of that choice is that nothing in the + * type system keeps the two in step, and the selected value is handed to the driver as a plain string: + * {@code CassandraCQLExecutionService} sets it as {@code DefaultDriverOption.REQUEST_CONSISTENCY}, which the + * driver resolves with {@code DefaultConsistencyLevel.valueOf}. A name the driver cannot parse therefore fails + * at runtime on a configured service rather than at build time - which is what this test exists to prevent. + * + *

This test lives here, in the module that already depends on the driver, rather than beside the enum: + * {@code nifi-cql-services-api} is the module the {@code ban-database-client-dependencies} enforcer rule + * guards, and that rule has no scope exemption, so even a test-scoped driver dependency there would fail the + * build. {@code ScyllaConsistencyLevelTest} makes the same assertions against ScyllaDB's fork, which ships its + * own copy of the driver enum. + */ +class CqlConsistencyLevelTest { + + @ParameterizedTest + @EnumSource(CqlConsistencyLevel.class) + @DisplayName("Every declared level's value is a name the DataStax driver can parse, since it is passed straight through as a config string") + void testValueParsesAsDriverConsistencyLevel(final CqlConsistencyLevel level) { + assertDoesNotThrow(() -> DefaultConsistencyLevel.valueOf(level.getValue()), + () -> level.name() + " declares the value '" + level.getValue() + + "', which the driver does not recognise. Known levels: " + + Arrays.stream(DefaultConsistencyLevel.values()) + .map(Enum::name) + .collect(Collectors.joining(", "))); + } + + @ParameterizedTest + @EnumSource(CqlConsistencyLevel.class) + @DisplayName("The declared value matches the driver level it resolves to, so the two enums cannot drift apart silently") + void testValueMatchesTheResolvedDriverLevel(final CqlConsistencyLevel level) { + assertEquals(level.getValue(), DefaultConsistencyLevel.valueOf(level.getValue()).name()); + } + + // Deliberately one-directional: exposing a subset of the driver's levels is a product decision, so a level + // the driver gains but this enum does not offer is not a failure. The direction that matters is that + // everything offered in the UI works. + + @ParameterizedTest + @EnumSource(CqlConsistencyLevel.class) + @DisplayName("Every level carries a display name and a description, which are what the Consistency Level dropdown renders") + void testDescribedValueContractIsPopulated(final CqlConsistencyLevel level) { + assertFalse(level.getDisplayName() == null || level.getDisplayName().isBlank(), + () -> level.name() + " has no display name"); + assertFalse(level.getDescription() == null || level.getDescription().isBlank(), + () -> level.name() + " has no description; the dropdown would show a bare name"); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CqlRowAndCellTest.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CqlRowAndCellTest.java new file mode 100644 index 000000000000..1a2f70d47e64 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/CqlRowAndCellTest.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cassandra; + +import org.apache.nifi.service.cql.api.lookup.CqlCell; +import org.apache.nifi.service.cql.api.lookup.CqlRow; +import org.apache.nifi.service.cql.api.lookup.impl.StandardCqlCell; +import org.apache.nifi.service.cql.api.lookup.impl.StandardCqlRow; +import org.apache.nifi.service.cql.api.lookup.impl.StandardCqlStatementResult; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Covers the schema-free row and cell types: the behaviour a caller relies on that is not simply the record + * components handed back. + */ +class CqlRowAndCellTest { + + private static byte[] utf8(final String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + + // ---- Cell ----------------------------------------------------------------------------------------- + + @Test + @DisplayName("A ByteBuffer value is copied out as bytes without consuming the buffer the backend handed over") + void testByteBufferIsReadWithoutConsumingIt() { + final ByteBuffer shared = ByteBuffer.wrap(utf8("hello")); + final CqlCell cell = new StandardCqlCell("v", shared); + + assertArrayEquals(utf8("hello"), cell.getBytes()); + assertEquals(5, shared.remaining(), "reading the cell must not advance the backend's buffer"); + assertArrayEquals(utf8("hello"), cell.getBytes(), "a second read must return the same bytes"); + } + + @Test + @DisplayName("A byte[] value is copied, so a caller mutating the result cannot corrupt the cell") + void testByteArrayIsCopied() { + final byte[] original = utf8("hello"); + final CqlCell cell = new StandardCqlCell("v", original); + + final byte[] returned = cell.getBytes(); + returned[0] = 'J'; + + assertArrayEquals(utf8("hello"), cell.getBytes()); + } + + @Test + @DisplayName("A null cell reports null rather than an empty array, so absent and empty stay distinguishable") + void testNullValue() { + final CqlCell cell = new StandardCqlCell("v", null); + + assertTrue(cell.isNull()); + assertNull(cell.getBytes()); + assertNull(cell.getObject()); + } + + @Test + @DisplayName("Asking a non-binary cell for bytes fails rather than inventing an encoding") + void testGetBytesOnNonBinaryColumn() { + final CqlCell cell = new StandardCqlCell("n", 42); + + final UnsupportedOperationException thrown = assertThrows(UnsupportedOperationException.class, cell::getBytes); + assertTrue(thrown.getMessage().contains("n"), thrown.getMessage()); + } + + @Test + @DisplayName("A column whose CQL type has no JDK form throws on read, naming the type and the alternative") + void testUnsupportedColumnType() { + final CqlCell cell = StandardCqlCell.unsupported("address", "address_type"); + + assertFalse(cell.isNull(), "an unreadable cell is not the same as a null one"); + final UnsupportedOperationException thrown = assertThrows(UnsupportedOperationException.class, cell::getObject); + assertTrue(thrown.getMessage().contains("address_type"), thrown.getMessage()); + assertTrue(thrown.getMessage().contains("query"), "should point at the API that can read it: " + thrown.getMessage()); + } + + // ---- Row ------------------------------------------------------------------------------------------ + + private static CqlRow row() { + return new StandardCqlRow(List.of( + new StandardCqlCell("k", ByteBuffer.wrap(utf8("key"))), + new StandardCqlCell("writetime(v)", 1234L), + new StandardCqlCell("v", ByteBuffer.wrap(utf8("value"))))); + } + + @Test + @DisplayName("Cells come back in selection order, which is the only ordering information a schema-free caller has") + void testCellsPreserveSelectionOrder() { + assertEquals(List.of("k", "writetime(v)", "v"), row().columnNames()); + } + + @Test + @DisplayName("A column name that is not a legal CQL identifier is carried through unchanged") + void testIllegalIdentifierColumnNameIsPreserved() { + assertEquals(1234L, row().getObject("writetime(v)")); + } + + @Test + @DisplayName("Looking up a column that is not in the row fails and says what the row does have") + void testMissingColumnThrows() { + final IllegalArgumentException thrown = + assertThrows(IllegalArgumentException.class, () -> row().getBytes("nope")); + + assertTrue(thrown.getMessage().contains("nope"), thrown.getMessage()); + assertTrue(thrown.getMessage().contains("writetime(v)"), "should list the actual columns: " + thrown.getMessage()); + } + + @Test + @DisplayName("A duplicated column name keeps both cells, which is why a row is a list rather than a map") + void testDuplicateColumnNamesAreBothRetained() { + final CqlRow duplicated = new StandardCqlRow(List.of( + new StandardCqlCell("v", ByteBuffer.wrap(utf8("first"))), + new StandardCqlCell("v", ByteBuffer.wrap(utf8("second"))))); + + assertEquals(2, duplicated.cells().size()); + assertEquals(List.of("v", "v"), duplicated.columnNames()); + // The by-name shorthand can only answer with one of them, which is precisely why cells() exists. + assertArrayEquals(utf8("first"), duplicated.getBytes("v")); + assertArrayEquals(utf8("second"), duplicated.cells().get(1).getBytes()); + } + + @Test + @DisplayName("findCell distinguishes an absent column from a present one holding null") + void testFindCellSeparatesAbsentFromNull() { + final CqlRow withNull = new StandardCqlRow(List.of(new StandardCqlCell("v", null))); + + assertTrue(withNull.findCell("v").isPresent(), "the column is present"); + assertNull(withNull.getBytes("v"), "...and holds a null"); + assertTrue(withNull.findCell("absent").isEmpty()); + } + + @Test + @DisplayName("A row's cells cannot be changed through the list handed to it or the list handed back") + void testCellsAreImmutable() { + assertThrows(UnsupportedOperationException.class, () -> row().cells().clear()); + } + + // ---- Result --------------------------------------------------------------------------------------- + + @Test + @DisplayName("An applied conditional write reports true and carries no row, since there was nothing to return") + void testAppliedResult() { + final StandardCqlStatementResult result = new StandardCqlStatementResult(true, List.of()); + + assertTrue(result.wasApplied()); + assertTrue(result.rows().isEmpty()); + } + + @Test + @DisplayName("A rejected conditional write reports false and carries the row that is actually stored") + void testRejectedResultCarriesCurrentRow() { + final StandardCqlStatementResult result = new StandardCqlStatementResult(false, List.of(row())); + + assertFalse(result.wasApplied()); + assertArrayEquals(utf8("value"), result.rows().getFirst().getBytes("v")); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/SharedCassandraCluster.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/SharedCassandraCluster.java new file mode 100644 index 000000000000..56b0e602caad --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/SharedCassandraCluster.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra; + +import com.datastax.oss.driver.api.core.CqlSession; +import org.apache.nifi.service.cql.it.CqlConnectionInfo; +import org.apache.nifi.service.cql.it.CqlDdl; +import org.testcontainers.cassandra.CassandraContainer; + +import java.util.HashMap; +import java.util.Map; + +/** + * One Cassandra container per major version, shared by every IT in this package that needs only a plain, + * default-configured server (CRUD, record field types), sparing the integration run a container start + * apiece for suites that otherwise boot an identical server. + * + *

Keyed by version rather than a plain singleton because {@code CassandraCrudIT} is + * {@code @ParameterizedClass} over {@link CassandraTestVersions} while {@code CassandraRecordFieldTypeIT} + * is pinned to one version: the pinned suite asks for {@value #PINNED_VERSION} and lands on the same + * instance the parameterized suite uses for that version. So it is one container per version rather than + * one per suite per version, which matters under {@code -DTEST_CASSANDRA_OLDER_VERSIONS=true}. + * + *

Sharing makes keyspace separation load-bearing: {@code testspace} (from {@code init.cql} at container + * start) belongs to the CRUD suite, {@code type_coverage} to {@code CassandraRecordFieldTypeIT} via + * {@link #createKeyspace}. A suite must not touch another's keyspace, and any suite asserting on an exact + * row or counter value must be that table's only writer. + * + *

The container is deliberately never stopped - Testcontainers' Ryuk sidecar removes it on JVM exit, + * which is the documented way to share one; an explicit {@code stop()} would pull it out from under suites + * that have not run yet. The trade-off for a multi-version run: every version's container stays up once + * touched, so peak memory is their sum. + */ +final class SharedCassandraCluster { + + /** + * The version suites that do not vary by release pin to. Taken from {@link CassandraTestVersions} so + * bumping the current major cannot leave them starting a container nothing else asks for. + */ + static final String PINNED_VERSION = CassandraTestVersions.CURRENT_VERSION; + + private static final String DATACENTER = "datacenter1"; + + private static final int CQL_PORT = 9042; + + private static final Map BY_VERSION = new HashMap<>(); + + private final String contactPoint; + + private final CqlSession session; + + private SharedCassandraCluster(final String contactPoint, final CqlSession session) { + this.contactPoint = contactPoint; + this.session = session; + } + + /** + * The cluster for {@code version}, started on first request and reused thereafter. Synchronized because + * the map operation spans a container start. + */ + static synchronized SharedCassandraCluster forVersion(final String version) { + return BY_VERSION.computeIfAbsent(version, SharedCassandraCluster::start); + } + + private static SharedCassandraCluster start(final String version) { + // init.cql creates the "testspace" keyspace and its tables, which the CRUD and connection + // verification suites both expect to exist before their first test. + final CassandraContainer container = CassandraContainerLimits.apply(new CassandraContainer("cassandra:" + version)) + .withTmpFs(Map.of("/var/lib/cassandra", "rw,size=1g")) + .withInitScript("init.cql"); + container.withExposedPorts(CQL_PORT); + container.start(); + + final CqlSession session = CqlSession.builder() + .addContactPoint(container.getContactPoint()) + .withLocalDatacenter(DATACENTER) + .build(); + + return new SharedCassandraCluster( + container.getContainerIpAddress() + ":" + container.getMappedPort(CQL_PORT), session); + } + + /** + * Creates {@code keyspace} if absent, for a suite that owns one rather than the {@code init.cql} + * keyspace. Idempotent because the container outlives whichever suite got there first. + */ + void createKeyspace(final String keyspace) { + CqlDdl.executeWithRetry(session, "create keyspace if not exists " + keyspace + + " with replication = {'class': 'SimpleStrategy', 'replication_factor': 1}"); + } + + CqlConnectionInfo connectionInfo(final String keyspace) { + return new CqlConnectionInfo(contactPoint, DATACENTER, keyspace, session); + } + + CqlSession session() { + return session; + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/mock/MockCassandraProcessor.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/mock/MockCassandraProcessor.java new file mode 100644 index 000000000000..1e2ac303bc2f --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/java/org/apache/nifi/service/cassandra/mock/MockCassandraProcessor.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cassandra.mock; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.service.cassandra.CassandraCQLExecutionService; + +import java.util.Collections; +import java.util.List; + +/** + * Mock Cassandra processor for testing CassandraSessionProvider + */ +public class MockCassandraProcessor extends AbstractProcessor { + private static final PropertyDescriptor CASSANDRA_SESSION_PROVIDER = new PropertyDescriptor.Builder() + .name("Cassandra Session Provider") + .required(true) + .description("Controller Service to obtain a Cassandra connection session") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .identifiesControllerService(CassandraCQLExecutionService.class) + .build(); + + @Override + public List getSupportedPropertyDescriptors() { + return Collections.singletonList(CASSANDRA_SESSION_PROVIDER); + } + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/resources/cassandra-base-config/cassandra.yaml b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/resources/cassandra-base-config/cassandra.yaml new file mode 100644 index 000000000000..28048e8a66ff --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/resources/cassandra-base-config/cassandra.yaml @@ -0,0 +1,271 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Stock cassandra:5.0 cassandra.yaml (trimmed of its explanatory comments). Read at test runtime by +# CassandraConfigOverrides, which patches in the one or two lines a given IT scenario needs and copies the +# result directly into the container, rather than checking in a full near-duplicate copy of this file per +# scenario. The only deviation from stock: the stock keystore_password/key_password values under the +# disabled transparent_data_encryption_options block are dropped so no credential literal is committed - +# that block is never activated by any scenario. +cluster_name: 'Test Cluster' + +num_tokens: 16 + +allocate_tokens_for_local_replication_factor: 3 + +hinted_handoff_enabled: true + +max_hint_window: 3h + +hinted_handoff_throttle: 1024KiB + +max_hints_delivery_threads: 2 + +hints_flush_period: 10000ms + +max_hints_file_size: 128MiB + +auto_hints_cleanup_enabled: false + +batchlog_replay_throttle: 1024KiB + +authenticator: AllowAllAuthenticator + +authorizer: AllowAllAuthorizer + +role_manager: CassandraRoleManager + +network_authorizer: AllowAllNetworkAuthorizer + +cidr_authorizer: + class_name: AllowAllCIDRAuthorizer + +roles_validity: 2000ms + +permissions_validity: 2000ms + +credentials_validity: 2000ms + +partitioner: org.apache.cassandra.dht.Murmur3Partitioner + +cdc_enabled: false + +disk_failure_policy: stop + +commit_failure_policy: stop + +prepared_statements_cache_size: + +key_cache_size: + +key_cache_save_period: 4h + +row_cache_size: 0MiB + +row_cache_save_period: 0s + +counter_cache_size: + +counter_cache_save_period: 7200s + +commitlog_sync: periodic + +commitlog_sync_period: 10000ms + +commitlog_segment_size: 32MiB + +commitlog_disk_access_mode: legacy + +seed_provider: + + - class_name: org.apache.cassandra.locator.SimpleSeedProvider + parameters: + + - seeds: "127.0.0.1:7000" + +concurrent_reads: 32 +concurrent_writes: 32 +concurrent_counter_writes: 32 + +concurrent_materialized_view_writes: 32 + +memtable: + configurations: + skiplist: + class_name: SkipListMemtable + trie: + class_name: TrieMemtable + default: + inherits: skiplist + +memtable_allocation_type: heap_buffers + +index_summary_capacity: + +index_summary_resize_interval: 60m + +trickle_fsync: false + +trickle_fsync_interval: 10240KiB + +storage_port: 7000 + +ssl_storage_port: 7001 + +listen_address: localhost + +# Must stay present (commented is fine): the official cassandra image's docker-entrypoint.sh templates +# these via sed on startup so Testcontainers can inject the container's reachable address. Deleting them +# breaks that substitution and Cassandra refuses to start with rpc_address bound to a wildcard. +# broadcast_address: 1.2.3.4 +# broadcast_rpc_address: 1.2.3.4 + +start_native_transport: true + +native_transport_port: 9042 + +native_transport_allow_older_protocols: true + +rpc_address: localhost + +rpc_keepalive: true + +incremental_backups: false + +snapshot_before_compaction: false + +auto_snapshot: true + +snapshot_links_per_second: 0 + +column_index_cache_size: 2KiB + +concurrent_materialized_view_builders: 1 + +compaction_throughput: 64MiB/s + +sstable_preemptive_open_interval: 50MiB + +uuid_sstable_identifiers_enabled: false + +read_request_timeout: 5000ms + +range_request_timeout: 10000ms + +write_request_timeout: 2000ms + +counter_write_request_timeout: 5000ms + +cas_contention_timeout: 1000ms + +truncate_request_timeout: 60000ms + +request_timeout: 10000ms + +slow_query_log_timeout: 500ms + +endpoint_snitch: SimpleSnitch + +dynamic_snitch_update_interval: 100ms + +dynamic_snitch_reset_interval: 600000ms + +dynamic_snitch_badness_threshold: 1.0 + +crypto_provider: + - class_name: org.apache.cassandra.security.DefaultCryptoProvider + parameters: + - fail_on_missing_provider: "false" + +server_encryption_options: + + internode_encryption: none + + legacy_ssl_storage_port_enabled: false + + keystore: conf/.keystore + + require_client_auth: false + + truststore: conf/.truststore + + require_endpoint_verification: false + +client_encryption_options: + + enabled: false + + keystore: conf/.keystore + + require_client_auth: false + +internode_compression: dc + +inter_dc_tcp_nodelay: false + +trace_type_query_ttl: 1d + +trace_type_repair_ttl: 7d + +user_defined_functions_enabled: false + +transparent_data_encryption_options: + enabled: false + chunk_length_kb: 64 + cipher: AES/CBC/PKCS5Padding + key_alias: testing:1 + + key_provider: + - class_name: org.apache.cassandra.security.JKSKeyProvider + parameters: + - keystore: conf/.keystore + store_type: JCEKS + +tombstone_warn_threshold: 1000 +tombstone_failure_threshold: 100000 + +replica_filtering_protection: + + cached_rows_warn_threshold: 2000 + cached_rows_fail_threshold: 32000 + +batch_size_warn_threshold: 5KiB + +batch_size_fail_threshold: 50KiB + +unlogged_batch_across_partitions_warn_threshold: 10 + +audit_logging_options: + enabled: false + logger: + - class_name: BinAuditLogger + +diagnostic_events_enabled: false + +repaired_data_tracking_for_range_reads_enabled: false +repaired_data_tracking_for_partition_reads_enabled: false + +report_unconfirmed_repaired_data_mismatches: false + +materialized_views_enabled: false + +sasi_indexes_enabled: false + +transient_replication_enabled: false + +drop_compact_storage_enabled: false + +storage_compatibility_mode: CASSANDRA_4 diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/resources/init.cql b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/resources/init.cql new file mode 100644 index 000000000000..a5b7be221fa2 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cassandra-session-provider-service/src/test/resources/init.cql @@ -0,0 +1,47 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +create keyspace testspace with replication = { 'class': 'SimpleStrategy', 'replication_factor': 1}; + +create table testspace.message +( + sender text, + receiver text, + message text, + when_sent timestamp, + primary key ( sender, receiver, when_sent ) +); + +create table testspace.query_test +( + column_a text, + column_b text, + when timestamp, + primary key ( (column_a), column_b) +); + +create table testspace.counter_test +( + column_a text, + increment_field counter, + primary key ( column_a ) +); + +create table testspace.simple_set_test +( + username text, + is_active boolean, + primary key ( username ) +); + diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/pom.xml b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/pom.xml new file mode 100644 index 000000000000..17ccac4acef5 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/pom.xml @@ -0,0 +1,100 @@ + + + + + nifi-cql-bundle + org.apache.nifi + 2.12.0-SNAPSHOT + + 4.0.0 + + nifi-cql-it-common + jar + + + 4.19.3 + + + + + + org.apache.nifi + nifi-api + + + org.apache.nifi + nifi-cql-services-api + 2.12.0-SNAPSHOT + + + org.apache.nifi + nifi-mock + + + org.apache.nifi + nifi-record + + + org.apache.cassandra + java-driver-core + ${driver.version} + + + org.apache.nifi + nifi-security-cert-builder + 2.12.0-SNAPSHOT + + + org.apache.nifi + nifi-ssl-context-service + 2.12.0-SNAPSHOT + + + + org.testcontainers + testcontainers + ${testcontainers.version} + test + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + test-jar + + test-jar + + + + + + + diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/AbstractCqlCrudIT.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/AbstractCqlCrudIT.java new file mode 100644 index 000000000000..7c9a115ccdaf --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/AbstractCqlCrudIT.java @@ -0,0 +1,595 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.it; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.cql.Row; +import org.apache.nifi.serialization.SimpleRecordSchema; +import org.apache.nifi.serialization.record.MapRecord; +import org.apache.nifi.serialization.record.RecordField; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.serialization.record.RecordSchema; +import org.apache.nifi.service.cql.api.constants.CqlBatchType; +import org.apache.nifi.service.cql.api.constants.UpdateMethod; +import org.apache.nifi.service.cql.api.exception.QueryFailureException; +import org.apache.nifi.service.cql.api.lookup.CqlRow; +import org.apache.nifi.service.cql.api.lookup.CqlStatementResult; +import org.apache.nifi.service.cql.api.metadata.QualifiedTableName; +import org.apache.nifi.service.cql.api.service.AbstractCQLExecutionService; +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.service.cql.api.service.QueryOverrides; +import org.apache.nifi.service.cql.api.service.WriteOverrides; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Exercises the CRUD-oriented public methods of {@link CQLExecutionService} - {@code insert}, {@code update}, + * {@code query}, and {@code execute} - against whichever backend the concrete subclass wires up. Assumes + * the subclass has already bootstrapped a {@code testspace} keyspace containing the {@code message}, + * {@code counter_test}, {@code simple_set_test}, and {@code query_test} tables (see {@code init.cql} in the + * Cassandra module for the canonical schema) and has called {@link #initializeSessionProvider(CqlConnectionInfo)}. + *

+ * Every test here turns on a server- or driver-observable effect a unit test can only assume: a counter's + * running total, a conditional write handing back the row that beat it, an explicit {@code WRITETIME} or + * {@code TTL} taking hold, a LOGGED batch refusing a counter mutation, an unsatisfiable consistency level + * failing on the first {@code execute()}. The driver-metadata-to-{@code PrimaryKey} mapping that backs + * {@code getMetadata} is covered without a cluster in {@code CassandraTableMetadataMappingTest}. + *

+ * Setup is a plain protected method rather than a JUnit lifecycle callback (e.g. {@code @BeforeAll}) + * deliberately: on a {@code @ParameterizedClass} leaf, {@code @BeforeAll} - even inherited from this + * superclass - runs BEFORE the leaf's own {@code @BeforeParameterizedClassInvocation} method, so an + * inherited {@code @BeforeAll} here would run before the container (and connection info) it depends on + * exists. Calling this method explicitly from the leaf's {@code @BeforeParameterizedClassInvocation} avoids + * that ordering trap entirely. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class AbstractCqlCrudIT { + + protected CQLExecutionService sessionProvider; + + private CqlSession session; + + private CqlConnectionInfo connectionInfo; + + private String kvTableName; + + /** + * @return a fresh, unconfigured instance of the backend implementation under test + */ + protected abstract CQLExecutionService newSessionProvider(); + + protected void initializeSessionProvider(final CqlConnectionInfo connectionInfo) throws Exception { + this.connectionInfo = connectionInfo; + this.session = connectionInfo.session(); + + this.kvTableName = connectionInfo.keyspace() + ".execute_kv"; + CqlDdl.executeWithRetry(session, "create table if not exists " + kvTableName + " (k blob primary key, v blob)"); + + this.sessionProvider = CqlServiceRunner.forService(newSessionProvider()) + .withConnection(connectionInfo) + .enable(); + } + + private RecordSchema getMessageSchema() { + List fields = List.of( + new RecordField("sender", RecordFieldType.STRING.getDataType()), + new RecordField("receiver", RecordFieldType.STRING.getDataType()), + new RecordField("message", RecordFieldType.STRING.getDataType()), + new RecordField("when_sent", RecordFieldType.TIMESTAMP.getDataType()) + ); + return new SimpleRecordSchema(fields); + } + + @Test + @DisplayName("Inserting a record writes it to the table using the CQL execution service") + void testInsertRecord() { + RecordSchema schema = getMessageSchema(); + Map rawRecord = new HashMap<>(); + rawRecord.put("sender", "john.smith"); + rawRecord.put("receiver", "jane.smith"); + rawRecord.put("message", "hello"); + rawRecord.put("when_sent", Instant.now()); + + MapRecord record = new MapRecord(schema, rawRecord); + + assertDoesNotThrow(() -> sessionProvider.insert(new QualifiedTableName(null, "message"), record, Map.of(), WriteOverrides.NONE)); + } + + @Test + @DisplayName("Incrementing then decrementing a counter column produces the expected running total") + void testIncrementAndDecrement() throws Exception { + RecordField field1 = new RecordField("column_a", RecordFieldType.STRING.getDataType()); + RecordField field2 = new RecordField("increment_field", RecordFieldType.INT.getDataType()); + RecordSchema schema = new SimpleRecordSchema(List.of(field1, field2)); + + Map map = new HashMap<>(); + map.put("column_a", "abcdef"); + map.put("increment_field", 1); + + MapRecord record = new MapRecord(schema, map); + + List updateKeys = new ArrayList<>(); + updateKeys.add("column_a"); + + //Set the initial value + sessionProvider.update(new QualifiedTableName(null, "counter_test"), record, Map.of(), updateKeys, UpdateMethod.INCREMENT, WriteOverrides.NONE); + + Thread.sleep(1000); + + sessionProvider.update(new QualifiedTableName(null, "counter_test"), record, Map.of(), updateKeys, UpdateMethod.INCREMENT, WriteOverrides.NONE); + + ResultSet results = session.execute("select increment_field from testspace.counter_test where column_a = 'abcdef'"); + + Iterator rowIterator = results.iterator(); + + Row row = rowIterator.next(); + + assertEquals(2, row.getLong("increment_field")); + + sessionProvider.update(new QualifiedTableName(null, "counter_test"), record, Map.of(), updateKeys, UpdateMethod.DECREMENT, WriteOverrides.NONE); + + results = session.execute("select increment_field from testspace.counter_test where column_a = 'abcdef'"); + + rowIterator = results.iterator(); + + row = rowIterator.next(); + + assertEquals(1, row.getLong("increment_field")); + } + + @Test + @DisplayName("Batch-inserting records with a LOGGED batch type writes them all") + void testBatchInsert() { + RecordSchema schema = getMessageSchema(); + List records = new ArrayList<>(); + for (int i = 0; i < 3; i++) { + Map rawRecord = new HashMap<>(); + rawRecord.put("sender", "batch.sender." + i); + rawRecord.put("receiver", "jane.smith"); + rawRecord.put("message", "hello " + i); + rawRecord.put("when_sent", Instant.now()); + records.add(new MapRecord(schema, rawRecord)); + } + + assertDoesNotThrow(() -> sessionProvider.insert(new QualifiedTableName(null, "message"), records, Map.of(), CqlBatchType.LOGGED, WriteOverrides.NONE)); + } + + @Test + @DisplayName("Batch-updating counters requires a COUNTER batch type - Cassandra/ScyllaDB reject counter mutations in a LOGGED batch") + void testBatchCounterUpdateRequiresCounterBatchType() throws Exception { + RecordField field1 = new RecordField("column_a", RecordFieldType.STRING.getDataType()); + RecordField field2 = new RecordField("increment_field", RecordFieldType.INT.getDataType()); + RecordSchema schema = new SimpleRecordSchema(List.of(field1, field2)); + + List records = new ArrayList<>(); + for (String key : List.of("batch-counter-a", "batch-counter-b", "batch-counter-c")) { + Map map = new HashMap<>(); + map.put("column_a", key); + map.put("increment_field", 1); + records.add(new MapRecord(schema, map)); + } + + List updateKeys = List.of("column_a"); + + // A LOGGED (or UNLOGGED) batch cannot contain counter mutations - the server rejects it outright. + // This is exactly the bug PutCQLRecord had: BATCH_STATEMENT_TYPE was validated but never forwarded, + // so every batched counter increment/decrement was silently submitted as LOGGED and would have + // failed here before the fix. + assertThrows(Exception.class, () -> sessionProvider.update(new QualifiedTableName(null, "counter_test"), records, Map.of(), updateKeys, + UpdateMethod.INCREMENT, CqlBatchType.LOGGED, WriteOverrides.NONE)); + + sessionProvider.update(new QualifiedTableName(null, "counter_test"), records, Map.of(), updateKeys, UpdateMethod.INCREMENT, CqlBatchType.COUNTER, WriteOverrides.NONE); + + for (String key : List.of("batch-counter-a", "batch-counter-b", "batch-counter-c")) { + Row row = session.execute("select increment_field from testspace.counter_test where column_a = '" + key + "'").iterator().next(); + assertEquals(1, row.getLong("increment_field")); + } + } + + @Test + @DisplayName("Inserting a record with a TTL override sets an expiration on the written columns") + void testInsertWithTtlOverrideSetsExpiration() { + RecordSchema schema = getMessageSchema(); + Map rawRecord = new HashMap<>(); + rawRecord.put("sender", "ttl.sender"); + rawRecord.put("receiver", "ttl.receiver"); + rawRecord.put("message", "expires soon"); + rawRecord.put("when_sent", Instant.now()); + + MapRecord record = new MapRecord(schema, rawRecord); + + sessionProvider.insert(new QualifiedTableName(null, "message"), record, Map.of(), new WriteOverrides(Duration.ofSeconds(100), null)); + + // "when_sent" (the last clustering column) is deliberately omitted: sender + receiver alone + // uniquely identify the single row this test wrote, and CQL allows a prefix of clustering + // columns in an equality filter without needing ALLOW FILTERING or reconstructing the exact + // timestamp literal that was generated for "when_sent". + Row row = session.execute( + "select TTL(message) as remaining_ttl from testspace.message where sender = 'ttl.sender' and receiver = 'ttl.receiver'") + .iterator().next(); + + int remainingTtl = row.getInt("remaining_ttl"); + assertTrue(remainingTtl > 0 && remainingTtl <= 100, "Expected a remaining TTL between 1 and 100 seconds but was " + remainingTtl); + } + + @Test + @DisplayName("Inserting with a Timestamp Field override uses that record field's value as the CQL write timestamp") + void testInsertWithTimestampFieldOverrideUsesRecordValue() { + RecordSchema schema = getMessageSchema(); + Instant eventTime = Instant.now().minus(Duration.ofDays(30)).truncatedTo(ChronoUnit.MILLIS); + + Map rawRecord = new HashMap<>(); + rawRecord.put("sender", "timestamp.sender"); + rawRecord.put("receiver", "timestamp.receiver"); + rawRecord.put("message", "old event"); + rawRecord.put("when_sent", eventTime); + + MapRecord record = new MapRecord(schema, rawRecord); + + sessionProvider.insert(new QualifiedTableName(null, "message"), record, Map.of(), new WriteOverrides(null, "when_sent")); + + long writeTimeMicros = session.execute( + "select WRITETIME(message) as write_time from testspace.message where sender = 'timestamp.sender' and receiver = 'timestamp.receiver'") + .iterator().next().getLong("write_time"); + + assertEquals(toEpochMicros(eventTime), writeTimeMicros); + } + + @Test + @DisplayName("Each record in a batch insert carries its own write timestamp when a Timestamp Field override is set") + void testBatchInsertWithTimestampFieldOverrideUsesPerRecordValues() { + RecordSchema schema = getMessageSchema(); + Instant olderEventTime = Instant.now().minus(Duration.ofDays(30)).truncatedTo(ChronoUnit.MILLIS); + Instant newerEventTime = Instant.now().minus(Duration.ofDays(1)).truncatedTo(ChronoUnit.MILLIS); + + Map olderRawRecord = new HashMap<>(); + olderRawRecord.put("sender", "batch.timestamp.sender"); + olderRawRecord.put("receiver", "older.receiver"); + olderRawRecord.put("message", "older event"); + olderRawRecord.put("when_sent", olderEventTime); + + Map newerRawRecord = new HashMap<>(); + newerRawRecord.put("sender", "batch.timestamp.sender"); + newerRawRecord.put("receiver", "newer.receiver"); + newerRawRecord.put("message", "newer event"); + newerRawRecord.put("when_sent", newerEventTime); + + List records = List.of( + new MapRecord(schema, olderRawRecord), + new MapRecord(schema, newerRawRecord)); + + sessionProvider.insert(new QualifiedTableName(null, "message"), records, Map.of(), CqlBatchType.LOGGED, new WriteOverrides(null, "when_sent")); + + assertEquals(toEpochMicros(olderEventTime), writeTimeForBatchTimestampSender("older.receiver")); + assertEquals(toEpochMicros(newerEventTime), writeTimeForBatchTimestampSender("newer.receiver")); + } + + @Test + @DisplayName("Updating with a Timestamp Field override uses that record field's value as the CQL write timestamp") + void testUpdateSetWithTimestampFieldOverrideUsesRecordValue() throws Exception { + // "when_sent" doubles as both an update key (part of message's primary key, so it identifies which row + // to update) and the Timestamp Field source - a realistic pattern, since it's a real column already + // being written, not an extra field invented just to carry a timestamp. generateUpdate() only knows + // how to treat a record field as either a SET target or a WHERE key, so the Timestamp Field has to be + // one of those, not an arbitrary extra field. + // + // The row-establishing insert below deliberately writes with an explicit, much older timestamp than + // "when_sent" itself (which only serves as the row's identity here). Without that, this insert's own + // auto-assigned "now" timestamp would land only a fraction of a millisecond away from the override + // under test below, and Cassandra's last-write-wins would resolve that near-tie essentially at random - + // making the test flaky/misleading regardless of whether the override actually works. + Instant whenSent = Instant.now().truncatedTo(ChronoUnit.MILLIS); + long deliberatelyOldWriteTimestamp = toEpochMicros(whenSent.minus(Duration.ofDays(60))); + session.execute(String.format( + "insert into testspace.message (sender, receiver, message, when_sent) values " + + "('update.timestamp.sender', 'update.timestamp.receiver', 'original', %d) using timestamp %d", + whenSent.toEpochMilli(), deliberatelyOldWriteTimestamp)); + Thread.sleep(250); + + RecordSchema schema = getMessageSchema(); + Map map = new HashMap<>(); + map.put("sender", "update.timestamp.sender"); + map.put("receiver", "update.timestamp.receiver"); + map.put("message", "updated"); + map.put("when_sent", whenSent); + + MapRecord record = new MapRecord(schema, map); + List updateKeys = List.of("sender", "receiver", "when_sent"); + + sessionProvider.update(new QualifiedTableName(null, "message"), record, Map.of(), updateKeys, UpdateMethod.SET, new WriteOverrides(null, "when_sent")); + + long writeTimeMicros = session.execute(String.format( + "select WRITETIME(message) as write_time from testspace.message where sender = 'update.timestamp.sender' " + + "and receiver = 'update.timestamp.receiver' and when_sent = %d", whenSent.toEpochMilli())) + .iterator().next().getLong("write_time"); + + assertEquals(toEpochMicros(whenSent), writeTimeMicros); + } + + private long writeTimeForBatchTimestampSender(String receiver) { + return session.execute(String.format( + "select WRITETIME(message) as write_time from testspace.message where sender = 'batch.timestamp.sender' and receiver = '%s'", receiver)) + .iterator().next().getLong("write_time"); + } + + private static long toEpochMicros(Instant instant) { + return instant.getEpochSecond() * 1_000_000L + instant.getNano() / 1_000L; + } + + @Test + @DisplayName("A Timestamp Field override is ignored for counter updates, since Cassandra/ScyllaDB do not support a custom write timestamp on counter columns") + void testCounterUpdateIgnoresTimestampFieldOverride() { + RecordField field1 = new RecordField("column_a", RecordFieldType.STRING.getDataType()); + RecordField field2 = new RecordField("increment_field", RecordFieldType.INT.getDataType()); + RecordSchema schema = new SimpleRecordSchema(List.of(field1, field2)); + + Map map = new HashMap<>(); + map.put("column_a", "timestamp-ignored-for-counters"); + map.put("increment_field", 1); + + MapRecord record = new MapRecord(schema, map); + List updateKeys = List.of("column_a"); + + // "column_a" isn't a timestamp-typed field, but it doesn't matter: if the override were mistakenly + // applied to a counter UPDATE, Cassandra/ScyllaDB would reject the statement outright regardless of + // what value was supplied ("Cannot provide custom timestamp for a BATCH containing counters" / + // similar), so not throwing here proves the override was correctly skipped before ever reading the field. + assertDoesNotThrow(() -> sessionProvider.update(new QualifiedTableName(null, "counter_test"), record, Map.of(), updateKeys, UpdateMethod.INCREMENT, new WriteOverrides(null, "column_a"))); + } + + @Test + @DisplayName("A TTL override is ignored for counter updates, since Cassandra/ScyllaDB do not support a TTL on counter columns") + void testCounterUpdateIgnoresTtlOverride() { + RecordField field1 = new RecordField("column_a", RecordFieldType.STRING.getDataType()); + RecordField field2 = new RecordField("increment_field", RecordFieldType.INT.getDataType()); + RecordSchema schema = new SimpleRecordSchema(List.of(field1, field2)); + + Map map = new HashMap<>(); + map.put("column_a", "ttl-ignored-for-counters"); + map.put("increment_field", 1); + + MapRecord record = new MapRecord(schema, map); + List updateKeys = List.of("column_a"); + + // If the TTL were mistakenly applied to a counter UPDATE, Cassandra/ScyllaDB would reject the + // statement outright ("Cannot set ttl on a counter column"), so simply not throwing here proves + // the override was correctly skipped. + assertDoesNotThrow(() -> sessionProvider.update(new QualifiedTableName(null, "counter_test"), record, Map.of(), updateKeys, + UpdateMethod.INCREMENT, new WriteOverrides(Duration.ofSeconds(100), null))); + } + + @Test + @DisplayName("Updating a record with the SET method overwrites the existing column value") + void testUpdateSet() throws Exception { + session.execute("insert into testspace.simple_set_test(username, is_active) values('john.smith', true)"); + Thread.sleep(250); + + RecordField field1 = new RecordField("username", RecordFieldType.STRING.getDataType()); + RecordField field2 = new RecordField("is_active", RecordFieldType.BOOLEAN.getDataType()); + RecordSchema schema = new SimpleRecordSchema(List.of(field1, field2)); + + Map map = new HashMap<>(); + map.put("username", "john.smith"); + map.put("is_active", false); + + MapRecord record = new MapRecord(schema, map); + + List updateKeys = new ArrayList<>(); + updateKeys.add("username"); + + sessionProvider.update(new QualifiedTableName(null, "simple_set_test"), record, Map.of(), updateKeys, UpdateMethod.SET, WriteOverrides.NONE); + + Iterator iterator = session.execute("select is_active from testspace.simple_set_test where username = 'john.smith'").iterator(); + + Row row = iterator.next(); + + assertFalse(row.getBoolean("is_active")); + } + + @Test + @DisplayName("Querying rows via the CQL execution service returns them without error") + void testQueryRecord() { + String[] statements = """ + insert into testspace.query_test (column_a, column_b, when) + values ('abc', 'def', toTimestamp(now())); + insert into testspace.query_test (column_a, column_b, when) + values ('abc', 'ghi', toTimestamp(now())); + insert into testspace.query_test (column_a, column_b, when) + values ('abc', 'jkl', toTimestamp(now())); + """.trim().split("\\;"); + for (String statement : statements) { + session.execute(statement); + } + + CollectingCqlQueryCallback callback = new CollectingCqlQueryCallback(); + + sessionProvider.query("select * from testspace.query_test", null, callback, QueryOverrides.NONE); + + assertTrue(callback.getRecords().size() >= statements.length, + () -> "Expected at least the " + statements.length + " inserted rows back, got " + callback.getRecords().size()); + } + + @Test + @DisplayName("A query that fails on the driver's initial execute() surfaces as QueryFailureException, not a raw driver exception") + void testQueryExecutionFailureOnInitialExecuteIsWrappedAsQueryFailureException() throws Exception { + // A consistency level of THREE can never be satisfied by this single-node cluster, so the driver + // throws UnavailableException (a QueryExecutionException) synchronously from the very first + // execute() call, before any rows are fetched - exactly the code path that used to bypass the + // QueryFailureException translation. + final CQLExecutionService unsatisfiableConsistencyProvider = CqlServiceRunner.forService(newSessionProvider()) + .withConnection(connectionInfo) + .withProperty(AbstractCQLExecutionService.CONSISTENCY_LEVEL, "THREE") + .enable(); + + final CollectingCqlQueryCallback callback = new CollectingCqlQueryCallback(); + + assertThrows(QueryFailureException.class, () -> unsatisfiableConsistencyProvider.query( + "select * from testspace.query_test", null, callback, QueryOverrides.NONE)); + + assertTrue(callback.getRecords().isEmpty(), "A query that never executed should not have delivered rows"); + } + + // ---- execute(): conditional writes and schema-free reads ------------------------------------------- + // + // These are the paths query() cannot express: a statement-level outcome, and values read without a + // schema imposed on them. Both matter for a caller doing compare-and-set over opaque bytes. + + /** + * The {@code keyspace.table} name the {@code execute()} tests below share. The table itself is created + * once in {@link #initializeSessionProvider(CqlConnectionInfo)}, before the service session exists, so + * that session sees it on connect rather than racing a lazy create against its schema metadata. + */ + private String kvTable() { + return kvTableName; + } + + private static ByteBuffer bytes(final String value) { + return ByteBuffer.wrap(value.getBytes(StandardCharsets.UTF_8)); + } + + @Test + @DisplayName("An unconditional statement reports wasApplied, so a caller that never writes conditionally has no special case") + void testExecuteUnconditionalWriteIsApplied() { + final String table = kvTable(); + + final CqlStatementResult result = sessionProvider.execute("insert into " + table + " (k, v) values (?, ?)", + List.of(bytes("uncond"), bytes("value")), QueryOverrides.NONE); + + assertTrue(result.wasApplied()); + assertTrue(result.rows().isEmpty(), "an unconditional insert returns no rows"); + } + + @Test + @DisplayName("A conditional write that applies reports true and returns no row, since there was no prior value") + void testExecuteConditionalWriteApplied() { + final String table = kvTable(); + + final CqlStatementResult result = sessionProvider.execute( + "insert into " + table + " (k, v) values (?, ?) if not exists", + List.of(bytes("lwt-new"), bytes("first")), QueryOverrides.NONE); + + assertTrue(result.wasApplied()); + assertTrue(result.rows().isEmpty(), + () -> "expected no rows for an applied conditional write, got " + result.rows().size()); + } + + @Test + @DisplayName("A conditional write that is rejected returns the stored row, which is what makes compare-and-set one round trip") + void testExecuteConditionalWriteRejectedCarriesStoredRow() { + final String table = kvTable(); + sessionProvider.execute("insert into " + table + " (k, v) values (?, ?)", + List.of(bytes("lwt-existing"), bytes("original")), QueryOverrides.NONE); + + final CqlStatementResult result = sessionProvider.execute( + "insert into " + table + " (k, v) values (?, ?) if not exists", + List.of(bytes("lwt-existing"), bytes("attempted")), QueryOverrides.NONE); + + assertFalse(result.wasApplied()); + assertEquals(1, result.rows().size()); + + final CqlRow stored = result.rows().getFirst(); + assertArrayEquals("original".getBytes(StandardCharsets.UTF_8), stored.getBytes("v"), + "the losing writer must see the value that is actually stored"); + + // The outcome is reported by wasApplied(), not as data: "[applied]" is not a legal identifier, and + // surfacing it as a column would push that problem onto every caller. + assertFalse(stored.columnNames().contains("[applied]"), + () -> "the outcome column leaked into the row: " + stored.columnNames()); + } + + @Test + @DisplayName("A conditional delete reports whether it applied and returns the row it removed") + void testExecuteConditionalDelete() { + final String table = kvTable(); + sessionProvider.execute("insert into " + table + " (k, v) values (?, ?)", + List.of(bytes("to-delete"), bytes("doomed")), QueryOverrides.NONE); + + final CqlStatementResult deleted = sessionProvider.execute( + "delete from " + table + " where k = ? if exists", List.of(bytes("to-delete")), QueryOverrides.NONE); + assertTrue(deleted.wasApplied()); + + final CqlStatementResult again = sessionProvider.execute( + "delete from " + table + " where k = ? if exists", List.of(bytes("to-delete")), QueryOverrides.NONE); + assertFalse(again.wasApplied(), "deleting an absent row must not report as applied"); + } + + @Test + @DisplayName("Blob values round-trip as bytes, with no schema imposed on an encoding the caller owns") + void testExecuteBlobRoundTrip() { + final String table = kvTable(); + final byte[] payload = "opaquebytes".getBytes(StandardCharsets.UTF_8); + + sessionProvider.execute("insert into " + table + " (k, v) values (?, ?)", + List.of(bytes("roundtrip"), ByteBuffer.wrap(payload)), QueryOverrides.NONE); + + final CqlStatementResult result = sessionProvider.execute("select k, v from " + table + " where k = ?", + List.of(bytes("roundtrip")), QueryOverrides.NONE); + + assertEquals(1, result.rows().size()); + assertArrayEquals(payload, result.rows().getFirst().getBytes("v")); + } + + @Test + @DisplayName("Cells arrive in selection order, and a column name that is not a legal identifier survives intact") + void testExecuteExposesCellsInSelectionOrder() { + final String table = kvTable(); + sessionProvider.execute("insert into " + table + " (k, v) values (?, ?)", + List.of(bytes("cells"), bytes("value")), QueryOverrides.NONE); + + final CqlStatementResult result = sessionProvider.execute( + "select v, writetime(v), k from " + table + " where k = ?", List.of(bytes("cells")), QueryOverrides.NONE); + + final CqlRow row = result.rows().getFirst(); + assertEquals(List.of("v", "writetime(v)", "k"), row.columnNames(), + "selection order is the only ordering information a schema-free caller has"); + assertNotNull(row.getObject("writetime(v)"), + "a projected function's column name is not a legal identifier, and must survive anyway"); + } + + @Test + @DisplayName("A query returning no rows yields an empty result rather than a null one") + void testExecuteEmptyResult() { + final String table = kvTable(); + + final CqlStatementResult result = sessionProvider.execute("select v from " + table + " where k = ?", + List.of(bytes("never-written")), QueryOverrides.NONE); + + assertTrue(result.rows().isEmpty()); + assertTrue(result.wasApplied(), "a plain SELECT is unconditional"); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/AbstractCqlRecordFieldTypeIT.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/AbstractCqlRecordFieldTypeIT.java new file mode 100644 index 000000000000..bae587aff643 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/AbstractCqlRecordFieldTypeIT.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.it; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.data.UdtValue; +import com.datastax.oss.driver.api.core.uuid.Uuids; +import org.apache.nifi.serialization.SimpleRecordSchema; +import org.apache.nifi.serialization.record.DataType; +import org.apache.nifi.serialization.record.MapRecord; +import org.apache.nifi.serialization.record.RecordField; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.serialization.record.RecordSchema; +import org.apache.nifi.service.cql.api.metadata.QualifiedTableName; +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.service.cql.api.service.CQLQueryCallback; +import org.apache.nifi.service.cql.api.service.QueryOverrides; +import org.apache.nifi.service.cql.api.service.WriteOverrides; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.sql.Date; +import java.sql.Time; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.temporal.ChronoUnit; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +/** + * Exercises {@link CQLExecutionService#insert(String, org.apache.nifi.serialization.record.Record, WriteOverrides)} + * and {@link CQLExecutionService#query(String, boolean, List, CQLQueryCallback, QueryOverrides)} for the + * record-field-type to CQL-type pairings that only a real cluster can settle, against whichever backend the + * concrete subclass wires up. Every case follows the same shape: write a record, assert the write succeeds, + * read the row back, and assert the value comes back as the type the column holds. + * + *

What stays here: the UDT cases, whose {@code Record}-to-{@code UdtValue} conversion is driven by the + * driver's live, server-resolved type metadata (a fabricated {@code UserDefinedType} cannot stand in for + * that); one round trip each for a list, a set, and a map; the two {@code java.sql.Date}/{@code Time} rows, + * which bind through a custom codec but read back through the driver's default, so only a real round trip + * shows the write and the read stay consistent across that asymmetry; and one canonical scalar plus one + * representative coercion as an end-to-end smoke test of the write/read path. + * + *

What moved out: the rest of the scalar and coercion grid. Every {@code Flexible*Codec} widening is + * covered byte-for-byte in {@code FlexibleCodecTest} and {@code CharacterCodecTest}, timestamp parsing in + * {@code CassandraCQLExecutionServiceBindValueTest}, and {@code convertForCqlType}'s {@code Object[]} + * handling and the {@code timeuuid} rejection of a non-version-1 UUID in + * {@code CassandraCQLExecutionServiceWritePathTest}. Running those against a container was a slow, + * Docker-gated copy of coverage that already exists without one. + *

+ * Setup is a plain protected method rather than a JUnit lifecycle callback for the same reason documented on + * {@link AbstractCqlCrudIT}. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class AbstractCqlRecordFieldTypeIT { + + protected CQLExecutionService sessionProvider; + + private CqlSession session; + + private String keyspace; + + /** + * @return a fresh, unconfigured instance of the backend implementation under test + */ + protected abstract CQLExecutionService newSessionProvider(); + + protected void initializeSessionProvider(final CqlConnectionInfo connectionInfo) throws Exception { + this.session = connectionInfo.session(); + this.keyspace = connectionInfo.keyspace(); + this.sessionProvider = CqlServiceRunner.forService(newSessionProvider()) + .withConnection(connectionInfo) + .enable(); + } + + /** Idempotent DDL, retried on the failures that say nothing about whether it ran - see {@link CqlDdl}. */ + private void executeDdlWithRetry(final String cql) { + CqlDdl.executeWithRetry(session, cql); + } + + private void createTable(final String tableName, final String cqlColumnType) { + executeDdlWithRetry(String.format("create table if not exists %s.%s (id int primary key, value_field %s)", keyspace, tableName, cqlColumnType)); + } + + private RecordSchema schemaFor(final RecordField valueField) { + return new SimpleRecordSchema(List.of(new RecordField("id", RecordFieldType.INT.getDataType()), valueField)); + } + + /** + * Reads back exactly one row via {@link CQLExecutionService#query}, asserting the query itself doesn't + * throw and that exactly one row came back, then returns it for the caller to assert on. + */ + private org.apache.nifi.serialization.record.Record readBack(final String tableName, final String columns, final int id) { + final CollectingCqlQueryCallback callback = new CollectingCqlQueryCallback(); + + assertDoesNotThrow(() -> sessionProvider.query( + String.format("select %s from %s.%s where id = %d", columns, keyspace, tableName, id), null, callback, QueryOverrides.NONE)); + + final List results = callback.getRecords(); + assertEquals(1, results.size(), () -> "Expected exactly one row back from " + tableName); + return results.getFirst(); + } + + /** + * The single-column type matrix: one row per (record field type, CQL column type) pairing, each writing a + * value and reading it back. + * + *

{@code written} and {@code expected} differ for the rows that exist to prove coercion happens - a + * value handed to the service in one Java type must come back in the type the column actually holds. Where + * they are the same object the row is a plain round-trip. + * + *

Types whose handling needs more than a value and a comparison - collections, UDTs, and the + * timeuuid rejection - keep their own methods below; forcing them into this shape would hide what they + * assert. + */ + static Stream singleColumnTypes() { + final Instant timestamp = Instant.now().truncatedTo(ChronoUnit.MILLIS); + final UUID timeUuid = Uuids.timeBased(); + final Date sqlDate = Date.valueOf(LocalDate.of(2024, 3, 15)); + final Time sqlTime = Time.valueOf(LocalTime.of(13, 45, 30)); + final Map map = Map.of("key1", "value1", "key2", "value2"); + + return Stream.of( + // One canonical scalar round trip, as an end-to-end smoke test of the write/read path. The + // rest of the scalar grid, and every Flexible*Codec widening, is covered without a container + // in FlexibleCodecTest, CharacterCodecTest, and CassandraCQLExecutionServiceBindValueTest. + arguments("STRING -> text", "string_test", "text", RecordFieldType.STRING.getDataType(), "hello world", "hello world"), + + // CQL timestamp has millisecond resolution, so the written value is truncated up front to make + // the round-trip comparison exact. + arguments("TIMESTAMP -> timestamp", "timestamp_test", "timestamp", RecordFieldType.TIMESTAMP.getDataType(), timestamp, timestamp), + + // A timeuuid column requires a genuine version-1 UUID; the rejection of any other version is + // unit-tested in CassandraCQLExecutionServiceWritePathTest. + arguments("UUID (v1) -> timeuuid", "timeuuid_test", "timeuuid", RecordFieldType.UUID.getDataType(), timeUuid, timeUuid), + + // One representative coercion end to end: a value handed in as text must land as - and read + // back as - the type the column actually holds. + arguments("INT <- String", "int_from_string_test", "int", RecordFieldType.INT.getDataType(), "123456", 123456), + + // java.sql.Date/Time bind through JavaSQLDateCodec and JavaSQLTimeCodec, but reads go through the + // driver's default DATE/TIME codecs: registering those codecs adds a way to bind the java.sql + // type, it does not replace the driver's default for an untyped read like Row.getObject(int). + // Only a real round trip shows the bind path and the read path stay consistent. + arguments("DATE <- java.sql.Date", "date_coercion_test", "date", RecordFieldType.DATE.getDataType(), sqlDate, sqlDate.toLocalDate()), + arguments("TIME <- java.sql.Time", "time_coercion_test", "time", RecordFieldType.TIME.getDataType(), sqlTime, sqlTime.toLocalTime()), + + // A collection column: convertForCqlType's element recursion plus the server's own map encoding. + arguments("MAP -> map", "map_test", "map", + RecordFieldType.MAP.getMapDataType(RecordFieldType.STRING.getDataType()), map, map)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("singleColumnTypes") + @DisplayName("A record field writes to its CQL column and reads back as the value the column holds") + void testSingleColumnType(final String description, final String tableName, final String cqlColumnType, + final DataType dataType, final Object written, final Object expected) { + createTable(tableName, cqlColumnType); + final RecordSchema schema = schemaFor(new RecordField("value_field", dataType)); + final MapRecord record = new MapRecord(schema, Map.of("id", 1, "value_field", written)); + + assertDoesNotThrow(() -> sessionProvider.insert(new QualifiedTableName(null, tableName), record, Map.of(), WriteOverrides.NONE)); + + assertEquals(expected, readBack(tableName, "value_field", 1).getValue("value_field")); + } + + @Test + @DisplayName("A record with an ARRAY of STRING field holding a List writes to and reads back from a list column") + void testArray() { + createTable("array_test", "list"); + final RecordSchema schema = schemaFor( + new RecordField("value_field", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.STRING.getDataType()))); + final List expected = List.of("a", "b", "c"); + final MapRecord record = new MapRecord(schema, Map.of("id", 1, "value_field", expected)); + + assertDoesNotThrow(() -> sessionProvider.insert(new QualifiedTableName(null, "array_test"), record, Map.of(), WriteOverrides.NONE)); + + assertEquals(expected, readBack("array_test", "value_field", 1).getValue("value_field")); + } + + @Test + @DisplayName("A record with an ARRAY of STRING field holding a Set writes to and reads back from a set column") + void testSetOfString() { + createTable("set_test", "set"); + final RecordSchema schema = schemaFor( + new RecordField("value_field", RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.STRING.getDataType()))); + final Set expected = Set.of("a", "b", "c"); + final MapRecord record = new MapRecord(schema, Map.of("id", 1, "value_field", expected)); + + assertDoesNotThrow(() -> sessionProvider.insert(new QualifiedTableName(null, "set_test"), record, Map.of(), WriteOverrides.NONE)); + + assertEquals(expected, readBack("set_test", "value_field", 1).getValue("value_field")); + } + + @Test + @DisplayName("A record with a MAP of RECORD field writes a map of UDT values to and reads it back") + void testMapOfAddressUserDefinedType() { + executeDdlWithRetry("create type if not exists " + keyspace + ".address_map_item (street_address text, state text, zip_code int)"); + executeDdlWithRetry(String.format("create table if not exists %s.address_map_test (id int primary key, addresses map>)", keyspace)); + + final RecordSchema addressSchema = new SimpleRecordSchema(List.of( + new RecordField("street_address", RecordFieldType.STRING.getDataType()), + new RecordField("state", RecordFieldType.STRING.getDataType()), + new RecordField("zip_code", RecordFieldType.INT.getDataType()) + )); + final RecordSchema schema = schemaFor(new RecordField("addresses", + RecordFieldType.MAP.getMapDataType(RecordFieldType.RECORD.getRecordDataType(addressSchema)))); + + final MapRecord home = new MapRecord(addressSchema, Map.of("street_address", "123 Main St", "state", "NC", "zip_code", 27601)); + final MapRecord work = new MapRecord(addressSchema, Map.of("street_address", "456 Elm St", "state", "SC", "zip_code", 29401)); + final MapRecord record = new MapRecord(schema, Map.of("id", 1, "addresses", Map.of("home", home, "work", work))); + + assertDoesNotThrow(() -> sessionProvider.insert(new QualifiedTableName(null, "address_map_test"), record, Map.of(), WriteOverrides.NONE)); + + final Object addresses = readBack("address_map_test", "addresses", 1).getValue("addresses"); + assertInstanceOf(Map.class, addresses); + @SuppressWarnings("unchecked") + final Map addressMap = (Map) addresses; + assertEquals("123 Main St", addressMap.get("home").getString("street_address")); + assertEquals("456 Elm St", addressMap.get("work").getString("street_address")); + } + + @Test + @DisplayName("A record with a RECORD field for a Home Address writes a UDT and reads it back") + void testHomeAddressUserDefinedType() { + executeDdlWithRetry("create type if not exists " + keyspace + ".home_address (street_address text, state text, zip_code int)"); + executeDdlWithRetry(String.format("create table if not exists %s.home_address_test (id int primary key, home_address frozen)", keyspace)); + + final RecordSchema addressSchema = new SimpleRecordSchema(List.of( + new RecordField("street_address", RecordFieldType.STRING.getDataType()), + new RecordField("state", RecordFieldType.STRING.getDataType()), + new RecordField("zip_code", RecordFieldType.INT.getDataType()) + )); + final RecordSchema schema = schemaFor(new RecordField("home_address", RecordFieldType.RECORD.getRecordDataType(addressSchema))); + + // A plain nested MapRecord, exactly like what a RecordReader (JSON, Avro, etc.) would produce for a + // nested object - the session provider is responsible for converting this into the UdtValue the + // DataStax driver's codec requires. + final MapRecord homeAddress = new MapRecord(addressSchema, Map.of( + "street_address", "123 Main St", + "state", "NC", + "zip_code", 27601)); + + final MapRecord record = new MapRecord(schema, Map.of("id", 1, "home_address", homeAddress)); + + assertDoesNotThrow(() -> sessionProvider.insert(new QualifiedTableName(null, "home_address_test"), record, Map.of(), WriteOverrides.NONE)); + + final Object readValue = readBack("home_address_test", "home_address", 1).getValue("home_address"); + assertInstanceOf(UdtValue.class, readValue); + final UdtValue udtValue = (UdtValue) readValue; + assertEquals("123 Main St", udtValue.getString("street_address")); + assertEquals("NC", udtValue.getString("state")); + assertEquals(27601, udtValue.getInt("zip_code")); + } + + @Test + @DisplayName("A record with a UDT nested inside another UDT writes to and reads back without any UDT-specific code") + void testPersonWithNestedAddressUserDefinedType() { + // Deliberately different names, field count, casing, and nesting depth than testHomeAddressUserDefinedType: + // this proves the RECORD-to-UdtValue conversion is driven entirely by the driver's live UDT metadata + // rather than any UDT-specific code, since it also has to recurse into a UDT nested inside another UDT. + // Quoting "firstName"/"lastName"/"zipCode" preserves their camelCase spelling; CQL would otherwise fold + // unquoted identifiers to lowercase. + executeDdlWithRetry("create type if not exists " + keyspace + ".address (street text, state text, \"zipCode\" int)"); + executeDdlWithRetry("create type if not exists " + keyspace + ".person (\"firstName\" text, \"lastName\" text, address frozen

)"); + executeDdlWithRetry(String.format("create table if not exists %s.person_test (id int primary key, person frozen)", keyspace)); + + final RecordSchema addressSchema = new SimpleRecordSchema(List.of( + new RecordField("street", RecordFieldType.STRING.getDataType()), + new RecordField("state", RecordFieldType.STRING.getDataType()), + new RecordField("zipCode", RecordFieldType.INT.getDataType()) + )); + final RecordSchema personSchema = new SimpleRecordSchema(List.of( + new RecordField("firstName", RecordFieldType.STRING.getDataType()), + new RecordField("lastName", RecordFieldType.STRING.getDataType()), + new RecordField("address", RecordFieldType.RECORD.getRecordDataType(addressSchema)) + )); + final RecordSchema schema = schemaFor(new RecordField("person", RecordFieldType.RECORD.getRecordDataType(personSchema))); + + final MapRecord address = new MapRecord(addressSchema, Map.of( + "street", "123 Main St", + "state", "NC", + "zipCode", 27601)); + final MapRecord person = new MapRecord(personSchema, Map.of( + "firstName", "John", + "lastName", "Doe", + "address", address)); + final MapRecord record = new MapRecord(schema, Map.of("id", 1, "person", person)); + + assertDoesNotThrow(() -> sessionProvider.insert(new QualifiedTableName(null, "person_test"), record, Map.of(), WriteOverrides.NONE)); + + final Object readValue = readBack("person_test", "person", 1).getValue("person"); + assertInstanceOf(UdtValue.class, readValue); + final UdtValue personValue = (UdtValue) readValue; + assertEquals("John", personValue.getString("firstName")); + assertEquals("Doe", personValue.getString("lastName")); + final UdtValue addressValue = personValue.getUdtValue("address"); + assertEquals("123 Main St", addressValue.getString("street")); + assertEquals("NC", addressValue.getString("state")); + assertEquals(27601, addressValue.getInt("zipCode")); + } + + @Test + @DisplayName("A record with an ARRAY of RECORD field writes a list of UDTs to and reads it back") + void testArrayOfAddressUserDefinedType() { + executeDdlWithRetry("create type if not exists " + keyspace + ".address_item (street_address text, state text, zip_code int)"); + executeDdlWithRetry(String.format("create table if not exists %s.address_array_test (id int primary key, addresses list>)", keyspace)); + + final RecordSchema addressSchema = new SimpleRecordSchema(List.of( + new RecordField("street_address", RecordFieldType.STRING.getDataType()), + new RecordField("state", RecordFieldType.STRING.getDataType()), + new RecordField("zip_code", RecordFieldType.INT.getDataType()) + )); + final RecordSchema schema = schemaFor(new RecordField("addresses", + RecordFieldType.ARRAY.getArrayDataType(RecordFieldType.RECORD.getRecordDataType(addressSchema)))); + + final MapRecord addressOne = new MapRecord(addressSchema, Map.of("street_address", "123 Main St", "state", "NC", "zip_code", 27601)); + final MapRecord addressTwo = new MapRecord(addressSchema, Map.of("street_address", "456 Elm St", "state", "SC", "zip_code", 29401)); + final MapRecord record = new MapRecord(schema, Map.of("id", 1, "addresses", List.of(addressOne, addressTwo))); + + assertDoesNotThrow(() -> sessionProvider.insert(new QualifiedTableName(null, "address_array_test"), record, Map.of(), WriteOverrides.NONE)); + + final Object addresses = readBack("address_array_test", "addresses", 1).getValue("addresses"); + assertInstanceOf(List.class, addresses); + final List addressList = (List) addresses; + assertEquals(2, addressList.size()); + assertInstanceOf(UdtValue.class, addressList.get(0)); + assertEquals("123 Main St", ((UdtValue) addressList.get(0)).getString("street_address")); + assertEquals("456 Elm St", ((UdtValue) addressList.get(1)).getString("street_address")); + } + +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/AbstractCqlSecureVerificationIT.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/AbstractCqlSecureVerificationIT.java new file mode 100644 index 000000000000..61421f38037a --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/AbstractCqlSecureVerificationIT.java @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.it; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.security.cert.builder.StandardCertificateBuilder; +import org.apache.nifi.service.cql.api.service.AbstractCQLExecutionService; +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.ssl.StandardSSLContextService; +import org.apache.nifi.util.TestRunner; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.util.HexFormat; +import java.util.List; +import java.util.UUID; +import java.util.function.Consumer; +import javax.security.auth.x500.X500Principal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Every {@code verify()} scenario that needs a specially-provisioned server, against one container per + * backend: {@code PasswordAuthenticator} plus one-way TLS, holding an {@value #ADMIN_ROLE} role with a + * generated password and the {@value #KEYSPACE} keyspace. This replaces three separate suites - connection + * verification, authentication, SSL - that each booted their own container, because the session provider + * runs them all through the same {@code verify()} path and the only thing that differed was server config. + * + *

The scenarios: a fully valid secure config succeeds every step; a wrong password and an untrusted + * server certificate each fail {@code Establish Connection}; a bogus datacenter fails {@code Verify + * Datacenter} (the driver builds a session with an unknown local datacenter and only fails when a statement + * forces node selection, which {@code verify()} does deliberately); an unknown keyspace fails {@code Verify + * Keyspace}; and a syntactically valid driver configuration file still connects. The one-way-vs-two-way TLS + * distinction is not exercised - the provider hands the driver a single {@code SSLContext} and never + * inspects it - and the {@code buildConfigLoader} composition itself is unit-tested without a container. + * + *

Gated behind {@link ConnectionTest}: runs only when {@value ConnectionTest#ENABLED_PROPERTY} is + * {@code true}, since a specially-configured container is a high price for coverage that a change to query + * or write behaviour cannot affect. + */ +@ConnectionTest +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public abstract class AbstractCqlSecureVerificationIT { + + protected static final String ADMIN_ROLE = "admin"; + + protected static final String KEYSPACE = "testspace"; + + protected static final String LOCAL_DATACENTER = "datacenter1"; + + protected static final String STORE_TYPE = "PKCS12"; + + private static final String SSL_CONTEXT_SERVICE_ID = "ssl-context-service"; + + private static final Path SSL_ARTIFACT_DIRECTORY = createSslArtifactDirectory(); + + private SecureServer server; + + private X509Certificate serverCertificate; + + private String adminPassword; + + /** + * @return a fresh, unconfigured instance of the backend implementation under test + */ + protected abstract CQLExecutionService newSessionProvider(); + + /** + * Boots one container with {@code PasswordAuthenticator} and one-way TLS enabled using the given + * already-generated server certificate/key, creates an {@value #ADMIN_ROLE} role with {@code adminPassword} + * and the {@value #KEYSPACE} keyspace, and returns its contact point once it is ready for the test + * connections this class will make. + */ + protected abstract SecureServer startSecureServer(KeyPair serverKeyPair, X509Certificate serverCertificate, + String adminRole, String adminPassword) throws Exception; + + /** + * A running container's connection coordinates plus how to stop it, so the shared test methods never + * need to know the concrete container type. + */ + public record SecureServer(String contactPoint, Runnable shutdown) { + } + + /** + * A generated PKCS12 file together with the random password it was protected with. Reusable by concrete + * subclasses for the server's own keystore where the backend's server-side TLS is also keystore-based. + */ + public record SslKeyStoreCredentials(Path path, String password) { + } + + @BeforeAll + void startServer() throws Exception { + final KeyPair serverKeyPair = generateKeyPair(); + serverCertificate = buildSelfSignedCertificate(serverKeyPair, "CN=cql-secure-server"); + adminPassword = UUID.randomUUID().toString(); + server = startSecureServer(serverKeyPair, serverCertificate, ADMIN_ROLE, adminPassword); + } + + @AfterAll + void stopServer() { + if (server != null) { + server.shutdown().run(); + } + } + + @Test + @DisplayName("A fully valid secure configuration reports success for every verification step") + void testValidSecureConfigurationSucceeds() throws Exception { + final List results = verify(adminPassword, trustStoreFor(serverCertificate), runner -> { }); + + assertAllSuccessful(results); + assertTrue(results.stream().anyMatch(result -> "Establish Connection".equals(result.getVerificationStepName()))); + assertTrue(results.stream().anyMatch(result -> "Verify Datacenter".equals(result.getVerificationStepName()))); + assertTrue(results.stream().anyMatch(result -> "Verify Keyspace".equals(result.getVerificationStepName()))); + } + + @Test + @DisplayName("An incorrect password fails the connection step against PasswordAuthenticator") + void testIncorrectPasswordFailsConnection() throws Exception { + final List results = + verify(UUID.randomUUID().toString(), trustStoreFor(serverCertificate), runner -> { }); + + assertStepFailed(results, "Establish Connection"); + } + + @Test + @DisplayName("A truststore that does not trust the server certificate fails the connection step") + void testUntrustedServerCertificateFailsConnection() throws Exception { + // A certificate the server never presents - the client's trust anchor is simply wrong. + final X509Certificate unrelated = buildSelfSignedCertificate(generateKeyPair(), "CN=untrusted-server"); + + final List results = verify(adminPassword, trustStoreFor(unrelated), runner -> { }); + + assertStepFailed(results, "Establish Connection"); + } + + @Test + @DisplayName("A nonexistent datacenter fails the datacenter step, caught by the statement verify() forces") + void testInvalidDatacenterFailsDatacenterStep() throws Exception { + final List results = verify(adminPassword, trustStoreFor(serverCertificate), + runner -> runner.withProperty(AbstractCQLExecutionService.DATACENTER, "not-a-real-datacenter")); + + assertStepFailed(results, "Verify Datacenter"); + } + + @Test + @DisplayName("A keyspace that does not exist fails the keyspace step") + void testNonexistentKeyspaceFailsKeyspaceStep() throws Exception { + final List results = verify(adminPassword, trustStoreFor(serverCertificate), + runner -> runner.withProperty(AbstractCQLExecutionService.KEYSPACE, "keyspace_that_does_not_exist")); + + assertStepFailed(results, "Verify Keyspace"); + } + + @Test + @DisplayName("A syntactically valid driver configuration file still connects on top of auth and TLS") + void testDriverConfigurationFileSucceeds() throws Exception { + final Path configFile = Files.createTempFile("CqlDriverConfig", ".conf"); + Files.writeString(configFile, """ + datastax-java-driver { + basic.request.timeout = 15 seconds + } + """, StandardCharsets.UTF_8); + configFile.toFile().deleteOnExit(); + + final List results = verify(adminPassword, trustStoreFor(serverCertificate), + runner -> runner.withProperty(AbstractCQLExecutionService.DRIVER_CONFIGURATION_FILE, configFile.toString())); + + assertAllSuccessful(results); + } + + /** + * Runs {@code verify()} with the {@value #ADMIN_ROLE} credentials, the given password, and an SSL + * context service configured with the given truststore, applying {@code customizer} last so a scenario + * can override one property before verification. + */ + private List verify(final String password, final SslKeyStoreCredentials trustStore, + final Consumer customizer) throws Exception { + final CqlServiceRunner serviceRunner = CqlServiceRunner.forService(newSessionProvider()) + .withConnection(server.contactPoint(), LOCAL_DATACENTER, KEYSPACE) + .withProperty(AbstractCQLExecutionService.USERNAME, ADMIN_ROLE) + .withProperty(AbstractCQLExecutionService.PASSWORD, password); + + final TestRunner runner = serviceRunner.runner(); + final StandardSSLContextService sslContextService = new StandardSSLContextService(); + runner.addControllerService(SSL_CONTEXT_SERVICE_ID, sslContextService); + runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE, trustStore.path().toString()); + runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_PASSWORD, trustStore.password()); + runner.setProperty(sslContextService, StandardSSLContextService.TRUSTSTORE_TYPE, STORE_TYPE); + runner.enableControllerService(sslContextService); + serviceRunner.withProperty(AbstractCQLExecutionService.PROP_SSL_CONTEXT_SERVICE, SSL_CONTEXT_SERVICE_ID); + + customizer.accept(serviceRunner); + return serviceRunner.verify(); + } + + private SslKeyStoreCredentials trustStoreFor(final X509Certificate certificate) throws Exception { + return writeTrustStore(certificate, "trusted"); + } + + private static void assertAllSuccessful(final List results) { + assertFalse(results.isEmpty()); + for (final ConfigVerificationResult result : results) { + assertEquals(ConfigVerificationResult.Outcome.SUCCESSFUL, result.getOutcome(), result.getExplanation()); + } + } + + private static void assertStepFailed(final List results, final String stepName) { + final ConfigVerificationResult stepResult = results.stream() + .filter(result -> stepName.equals(result.getVerificationStepName())) + .findFirst() + .orElseThrow(() -> new AssertionError("Expected a '" + stepName + "' verification result in " + results)); + + assertEquals(ConfigVerificationResult.Outcome.FAILED, stepResult.getOutcome(), stepResult.getExplanation()); + } + + // ---- TLS material helpers, shared with the concrete subclasses' server provisioning ---------------- + + protected static KeyPair generateKeyPair() throws Exception { + final KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance("RSA"); + keyPairGenerator.initialize(2048); + return keyPairGenerator.generateKeyPair(); + } + + protected static X509Certificate buildSelfSignedCertificate(final KeyPair keyPair, final String distinguishedName) { + return new StandardCertificateBuilder(keyPair, new X500Principal(distinguishedName), Duration.ofDays(1)) + .setDnsSubjectAlternativeNames(List.of("localhost")) + .build(); + } + + protected static SslKeyStoreCredentials writeKeyStore(final PrivateKey privateKey, final X509Certificate certificate, final String alias) throws Exception { + final String password = generateStorePassword(); + final KeyStore keyStore = KeyStore.getInstance(STORE_TYPE); + keyStore.load(null); + keyStore.setKeyEntry(alias, privateKey, password.toCharArray(), new X509Certificate[]{certificate}); + + final Path path = Files.createTempFile(SSL_ARTIFACT_DIRECTORY, alias + "-keystore", ".p12"); + try (OutputStream outputStream = Files.newOutputStream(path)) { + keyStore.store(outputStream, password.toCharArray()); + } + return new SslKeyStoreCredentials(path, password); + } + + protected static SslKeyStoreCredentials writeTrustStore(final X509Certificate certificate, final String alias) throws Exception { + final String password = generateStorePassword(); + final KeyStore trustStore = KeyStore.getInstance(STORE_TYPE); + trustStore.load(null); + trustStore.setCertificateEntry(alias, certificate); + + final Path path = Files.createTempFile(SSL_ARTIFACT_DIRECTORY, alias + "-truststore", ".p12"); + try (OutputStream outputStream = Files.newOutputStream(path)) { + trustStore.store(outputStream, password.toCharArray()); + } + return new SslKeyStoreCredentials(path, password); + } + + private static String generateStorePassword() { + final SecureRandom secureRandom = new SecureRandom(); + final byte[] bytes = new byte[24]; + secureRandom.nextBytes(bytes); + return HexFormat.of().formatHex(bytes); + } + + // Written under the module's own target/ directory so "mvn clean" collects the generated stores rather + // than relying on deleteOnExit(). + private static Path createSslArtifactDirectory() { + try { + return Files.createDirectories(Path.of("target", "cql-secure-it")); + } catch (final IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/CollectingCqlQueryCallback.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/CollectingCqlQueryCallback.java new file mode 100644 index 000000000000..47708e4204d7 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/CollectingCqlQueryCallback.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.it; + +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.service.cql.api.service.CQLQueryCallback; + +import java.util.ArrayList; +import java.util.List; + +/** + * The {@link CQLQueryCallback} the integration tests read result sets through: it accumulates every row the + * service hands back, so a test can assert on what a query actually returned. + * + *

{@link #clear()} discards the accumulated rows, mirroring what a real callback does when the service + * signals that a fatal error means the batch it has built up so far must be thrown away. The number of times + * it was called is recorded rather than swallowed, so a test can assert that a failing query cleaned up after + * itself - a no-op {@code clear()} would look identical to one that was never called at all. + */ +public class CollectingCqlQueryCallback implements CQLQueryCallback { + private final List records = new ArrayList<>(); + private int clearCount; + + @Override + public void receive(final long rowNumber, final Record result, final boolean hasMore) { + records.add(result); + } + + @Override + public void clear() { + records.clear(); + clearCount++; + } + + /** + * Always {@code false}: these tests call the service directly, so there is no incoming FlowFile for this + * callback to have routed to {@code original}. + */ + @Override + public boolean hasSentOriginal() { + return false; + } + + /** + * @return the rows received so far, in the order the service delivered them + */ + public List getRecords() { + return List.copyOf(records); + } + + /** + * @return how many times {@link #clear()} has been called + */ + public int getClearCount() { + return clearCount; + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/ConnectionTest.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/ConnectionTest.java new file mode 100644 index 000000000000..d0b18099a71f --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/ConnectionTest.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.it; + +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marks an integration suite as testing how a connection is established - authentication, TLS, and + * {@code verify()} - rather than what can be done once one exists. Suites so marked run only when + * {@value #ENABLED_PROPERTY} is set to {@code true}. + * + *

These are the expensive suites and the ones least likely to be affected by a change to query or write + * behaviour: authentication and TLS each provision a dedicated container per scenario, because the server + * settings they exercise are baked into {@code cassandra.yaml}/{@code scylla.yaml} at startup and cannot be + * toggled on a running one. Skipping them by default keeps the routine integration run to the suites that + * exercise the code most changes actually touch. + * + *

Enable with: + * 

{@code mvn verify -Pintegration-tests -DTEST_CQL_CONNECTION_TESTS=true}
+ * + *

The condition is applied at class level, so a disabled suite never reaches {@code @BeforeAll} (or, on a + * {@code @ParameterizedClass}, {@code @BeforeParameterizedClassInvocation}) and therefore never starts a + * container. Being skipped rather than excluded also means the run still reports that they did not execute, + * instead of quietly showing a smaller test count. + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +@Documented +@EnabledIfSystemProperty(named = ConnectionTest.ENABLED_PROPERTY, matches = "(?i)true") +public @interface ConnectionTest { + + /** + * The system property that opts these suites in. Named to match the {@code TEST_*} convention the other + * integration-test switches in this bundle already use ({@code TEST_CASSANDRA_OLDER_VERSIONS}, + * {@code TEST_SCYLLA_DDL_TIMEOUT_SECONDS}). + */ + String ENABLED_PROPERTY = "TEST_CQL_CONNECTION_TESTS"; +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/CqlConnectionInfo.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/CqlConnectionInfo.java new file mode 100644 index 000000000000..1567602de297 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/CqlConnectionInfo.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.it; + +import com.datastax.oss.driver.api.core.CqlSession; + +/** + * Connection coordinates for a running backend container, resolved by a backend-specific JUnit + * extension and injected into {@code @BeforeAll}/{@code @Test} methods as a single parameter (a plain + * data holder, not a factory - each test still builds its own {@code TestRunner} inline as before). + * + * @param contactPoint "host:port" of the running container + * @param datacenter local datacenter name to use with the driver + * @param keyspace keyspace bootstrapped by the extension before tests run + * @param session a raw driver session, already connected, for out-of-band setup/verification + */ +public record CqlConnectionInfo(String contactPoint, String datacenter, String keyspace, CqlSession session) { +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/CqlDdl.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/CqlDdl.java new file mode 100644 index 000000000000..5a094bb0d064 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/CqlDdl.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.it; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.DriverTimeoutException; +import com.datastax.oss.driver.api.core.connection.HeartbeatException; + +/** + * Executes schema-modifying statements against a real container, retrying the two failures that say nothing + * about whether the statement actually ran. + * + *

A {@link DriverTimeoutException} means the client stopped waiting for schema agreement, not that the + * server rejected the statement - ScyllaDB's Raft-based schema management (and, less often, Cassandra's own + * agreement protocol) can take longer than the configured request timeout on an otherwise healthy cluster. A + * {@link HeartbeatException} is the same story one layer down: the connection carrying the statement died + * mid-flight (observed on ScyllaDB under the load of a full-reactor IT run). Either way the statement may + * still land server-side moments later. + * + *

Every statement passed here must be idempotent - an "if not exists" form - since a + * retry after one of those false negatives must not fail with "already exists". + * + *

This lives in the shared IT module rather than beside any one suite because all three former copies of + * this loop wanted the same behaviour, and had drifted: two caught only the timeout, so they would flake on + * exactly the heartbeat failure the third already survived. + */ +public final class CqlDdl { + + private static final int MAX_ATTEMPTS = 3; + + private CqlDdl() { + } + + public static void executeWithRetry(final CqlSession session, final String cql) { + RuntimeException lastFailure = null; + + for (int attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) { + try { + session.execute(cql); + return; + } catch (final DriverTimeoutException | HeartbeatException e) { + lastFailure = e; + } + } + + throw lastFailure; + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/CqlServiceRunner.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/CqlServiceRunner.java new file mode 100644 index 000000000000..7c0d36208025 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/CqlServiceRunner.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.it; + +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.reporting.InitializationException; +import org.apache.nifi.service.cql.api.service.AbstractCQLExecutionService; +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; + +import java.util.List; +import java.util.Map; + +/** + * Wires a {@link CQLExecutionService} onto a {@link TestRunner} for an integration test. + * + *

Every IT in this suite needs the same four lines - a runner over {@link MockCqlProcessor}, the service + * added to it, and contact points/datacenter/keyspace set from a running container - before it can do + * anything interesting, and then diverges: some enable the service and use it, others only call + * {@code verify()} against a deliberately broken configuration. This collects the common part and leaves the + * divergence to {@link #enable()} versus {@link #verify()}. + * + *

Properties are set in call order and later calls overwrite earlier ones, so a test wanting all the + * usual connection settings except one writes {@code withConnection(info).withProperty(DATACENTER, "bogus")} + * rather than restating the other two. + */ +public final class CqlServiceRunner { + + private static final String SERVICE_ID = "cql-session-provider"; + + private final TestRunner runner = TestRunners.newTestRunner(new MockCqlProcessor()); + + private final CQLExecutionService service; + + private CqlServiceRunner(final CQLExecutionService service) throws InitializationException { + this.service = service; + runner.addControllerService(SERVICE_ID, service); + } + + public static CqlServiceRunner forService(final CQLExecutionService service) throws InitializationException { + return new CqlServiceRunner(service); + } + + /** + * Sets contact points, datacenter and keyspace from a running container. + */ + public CqlServiceRunner withConnection(final CqlConnectionInfo connectionInfo) { + return withConnection(connectionInfo.contactPoint(), connectionInfo.datacenter(), connectionInfo.keyspace()); + } + + /** + * As {@link #withConnection(CqlConnectionInfo)}, for a test holding the coordinates loose rather than as + * a {@link CqlConnectionInfo} - the SSL and authentication suites provision their own containers and + * never build one. + */ + public CqlServiceRunner withConnection(final String contactPoint, final String datacenter, final String keyspace) { + return withProperty(AbstractCQLExecutionService.CONTACT_POINTS, contactPoint) + .withProperty(AbstractCQLExecutionService.DATACENTER, datacenter) + .withProperty(AbstractCQLExecutionService.KEYSPACE, keyspace); + } + + public CqlServiceRunner withProperty(final PropertyDescriptor descriptor, final String value) { + runner.setProperty(service, descriptor, value); + return this; + } + + /** + * The underlying runner, for the occasional test that has to wire a second controller service the CQL + * service refers to (an SSL context, say) with setup too conditional to express here. + */ + public TestRunner runner() { + return runner; + } + + /** + * Enables the service and hands it back, ready to use. + */ + public CQLExecutionService enable() { + runner.enableControllerService(service); + return service; + } + + /** + * Runs {@code verify()} against the configuration built so far, without enabling the service - the point + * being that the configuration may be one that cannot work. + */ + public List verify() { + return runner.verify(service, Map.of()); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/DockerUtils.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/DockerUtils.java new file mode 100644 index 000000000000..4d302b005f12 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/DockerUtils.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cql.it; + +import com.github.dockerjava.api.command.CreateContainerCmd; + +import java.util.Objects; +import java.util.function.Consumer; + +/** + * Docker-level container tuning that Testcontainers does not expose directly. + */ +public final class DockerUtils { + + private DockerUtils() { + } + + /** + * Caps what a test container may take from the host, so a database container cannot size itself against + * all of the machine's memory and cores. + * + * @param cpus CPU cores the container may use + * @param memoryGb memory ceiling in gibibytes + * @return a modifier to hand to {@code withCreateContainerCmdModifier} + */ + public static Consumer createMemoryLimits(final long cpus, final long memoryGb) { + final long memoryLimitAsBytes = memoryGb * (1024L * 1024L * 1024L); + final long nanoCpus = cpus * 1_000_000_000L; + return cmd -> + Objects.requireNonNull(cmd.getHostConfig(), "HostConfig unexpectedly null") + .withMemory(memoryLimitAsBytes) + .withNanoCPUs(nanoCpus); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/MockCqlProcessor.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/MockCqlProcessor.java new file mode 100644 index 000000000000..ed6aeb00a8a1 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-it-common/src/test/java/org/apache/nifi/service/cql/it/MockCqlProcessor.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.it; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.service.cql.api.service.CQLExecutionService; + +import java.util.Collections; +import java.util.List; + +/** + * Mock processor for exercising a {@link CQLExecutionService} controller service via {@code TestRunner}, + * independent of which backend implementation (Cassandra, ScyllaDB) is under test. + */ +public class MockCqlProcessor extends AbstractProcessor { + private static final PropertyDescriptor CQL_SESSION_PROVIDER = new PropertyDescriptor.Builder() + .name("CQL Session Provider") + .required(true) + .description("Controller Service to obtain a CQL connection session") + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .identifiesControllerService(CQLExecutionService.class) + .build(); + + @Override + public List getSupportedPropertyDescriptors() { + return Collections.singletonList(CQL_SESSION_PROVIDER); + } + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-nar/pom.xml b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-nar/pom.xml new file mode 100644 index 000000000000..eac5154cfa78 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-nar/pom.xml @@ -0,0 +1,51 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-cql-bundle + 2.12.0-SNAPSHOT + + + nifi-cql-nar + nar + + + + + + com.google.guava + guava + provided + + + + + + + org.apache.nifi + nifi-cql-services-api-nar + 2.12.0-SNAPSHOT + nar + + + org.apache.nifi + nifi-cql-processors + + + diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-nar/src/main/resources/META-INF/LICENSE b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-nar/src/main/resources/META-INF/LICENSE new file mode 100644 index 000000000000..2672f6e36a1d --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-nar/src/main/resources/META-INF/LICENSE @@ -0,0 +1,342 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +APACHE NIFI SUBCOMPONENTS: + +The Apache NiFi project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +This product bundles 'libffi' which is available under an MIT style license. + libffi - Copyright (c) 1996-2014 Anthony Green, Red Hat, Inc and others. + see https://github.com/java-native-access/jna/blob/master/native/libffi/LICENSE + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + ``Software''), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to + permit persons to whom the Software is furnished to do so, subject to + the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +This product bundles 'asm' which is available under a 3-Clause BSD style license. +For details see http://asm.ow2.org/asmdex-license.html + + Copyright (c) 2012 France Télécom + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + + The binary distribution of this product bundles 'Bouncy Castle JDK 1.5' + under an MIT style license. + + Copyright (c) 2000 - 2015 The Legion of the Bouncy Castle Inc. (http://www.bouncycastle.org) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +The binary distribution of this product bundles 'JNR x86asm' under an MIT +style license. + + Copyright (C) 2010 Wayne Meissner + Copyright (c) 2008-2009, Petr Kobalicek + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + The binary distribution of this product bundles 'ParaNamer' and 'Paranamer Core' + which is available under a BSD style license. + + Copyright (c) 2006 Paul Hammant & ThoughtWorks Inc + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-nar/src/main/resources/META-INF/NOTICE b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-nar/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..6bd6684a5156 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-nar/src/main/resources/META-INF/NOTICE @@ -0,0 +1,328 @@ +nifi-cassandra-nar +Copyright 2016-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +****************** +Apache Software License v2 +****************** + +The following binary components are provided under the Apache Software License v2 + + (ASLv2) DataStax Java Driver for Apache Cassandra - Core + The following NOTICE information applies: + DataStax Java Driver for Apache Cassandra - Core + Copyright (C) 2012-2017 DataStax Inc. + + (ASLv2) Apache Avro + The following NOTICE information applies: + Apache Avro + Copyright 2009-2017 The Apache Software Foundation + + (ASLv2) Jackson JSON processor + The following NOTICE information applies: + # Jackson JSON processor + + Jackson is a high-performance, Free/Open Source JSON processing library. + It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has + been in development since 2007. + It is currently developed by a community of developers, as well as supported + commercially by FasterXML.com. + + ## Licensing + + Jackson core and extension components may licensed under different licenses. + To find the details that apply to this artifact see the accompanying LICENSE file. + For more information, including possible other licensing options, contact + FasterXML.com (http://fasterxml.com). + + ## Credits + + A list of contributors may be found from CREDITS file, which is included + in some artifacts (usually source distributions); but is always available + from the source code management (SCM) system project uses. + + (ASLv2) Apache Commons Codec + The following NOTICE information applies: + Apache Commons Codec + Copyright 2002-2014 The Apache Software Foundation + + src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java + contains test data from http://aspell.net/test/orig/batch0.tab. + Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + + =============================================================================== + + The content of package org.apache.commons.codec.language.bm has been translated + from the original php source code available at http://stevemorse.org/phoneticinfo.htm + with permission from the original authors. + Original source copyright: + Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + + (ASLv2) Apache Commons Compress + The following NOTICE information applies: + Apache Commons Compress + Copyright 2002-2017 The Apache Software Foundation + + The files in the package org.apache.commons.compress.archivers.sevenz + were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), + which has been placed in the public domain: + + "LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) + + (ASLv2) Apache Commons IO + The following NOTICE information applies: + Apache Commons IO + Copyright 2002-2016 The Apache Software Foundation + + (ASLv2) Apache Commons Lang + The following NOTICE information applies: + Apache Commons Lang + Copyright 2001-2017 The Apache Software Foundation + + This product includes software from the Spring Framework, + under the Apache License 2.0 (see: StringUtils.containsWhitespace()) + + (ASLv2) Guava + The following NOTICE information applies: + Guava + Copyright 2015 The Guava Authors + + (ASLv2) Dropwizard Metrics + The following NOTICE information applies: + Copyright (c) 2010-2013 Coda Hale, Yammer.com + + This product includes software developed by Coda Hale and Yammer, Inc. + + This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, + LongAdder), which was released with the following comments: + + Written by Doug Lea with assistance from members of JCP JSR-166 + Expert Group and released to the public domain, as explained at + http://creativecommons.org/publicdomain/zero/1.0/ + + (ASLv2) The Netty Project + The following NOTICE information applies: + Copyright 2014 The Netty Project + ------------------------------------------------------------------------------- + This product contains the extensions to Java Collections Framework which has + been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + + This product contains a modified version of Robert Harder's Public Domain + Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + + This product contains a modified portion of 'Webbit', an event based + WebSocket and HTTP server, which can be obtained at: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + + This product contains a modified portion of 'SLF4J', a simple logging + facade for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + + This product contains a modified portion of 'Apache Harmony', an open source + Java SE, which can be obtained at: + + * LICENSE: + * license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: + * http://archive.apache.org/dist/harmony/ + + This product contains a modified portion of 'jbzip2', a Java bzip2 compression + and decompression library written by Matthew J. Francis. It can be obtained at: + + * LICENSE: + * license/LICENSE.jbzip2.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jbzip2/ + + This product contains a modified portion of 'libdivsufsort', a C API library to construct + the suffix array and the Burrows-Wheeler transformed string for any input string of + a constant-size alphabet written by Yuta Mori. It can be obtained at: + + * LICENSE: + * license/LICENSE.libdivsufsort.txt (MIT License) + * HOMEPAGE: + * https://github.com/y-256/libdivsufsort + + This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, + which can be obtained at: + + * LICENSE: + * license/LICENSE.jctools.txt (ASL2 License) + * HOMEPAGE: + * https://github.com/JCTools/JCTools + + This product optionally depends on 'JZlib', a re-implementation of zlib in + pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + + This product optionally depends on 'Compress-LZF', a Java library for encoding and + decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: + + * LICENSE: + * license/LICENSE.compress-lzf.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/ning/compress + + This product optionally depends on 'lz4', a LZ4 Java compression + and decompression library written by Adrien Grand. It can be obtained at: + + * LICENSE: + * license/LICENSE.lz4.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jpountz/lz4-java + + This product optionally depends on 'lzma-java', a LZMA Java compression + and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.lzma-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jponge/lzma-java + + This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression + and decompression library written by William Kinney. It can be obtained at: + + * LICENSE: + * license/LICENSE.jfastlz.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jfastlz/ + + This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data + interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/protobuf + + This product optionally depends on 'Bouncy Castle Crypto APIs' to generate + a temporary self-signed X.509 certificate when the JVM does not provide the + equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + + This product optionally depends on 'Snappy', a compression library produced + by Google Inc, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/snappy + + This product optionally depends on 'JBoss Marshalling', an alternative Java + serialization API, which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://www.jboss.org/jbossmarshalling + + This product optionally depends on 'Caliper', Google's micro- + benchmarking framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.caliper.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/google/caliper + + This product optionally depends on 'Apache Log4J', a logging framework, which + can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + + This product optionally depends on 'Aalto XML', an ultra-high performance + non-blocking XML processor, which can be obtained at: + + * LICENSE: + * license/LICENSE.aalto-xml.txt (Apache License 2.0) + * HOMEPAGE: + * http://wiki.fasterxml.com/AaltoHome + + This product contains a modified version of 'HPACK', a Java implementation of + the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: + + * LICENSE: + * license/LICENSE.hpack.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/twitter/hpack + + This product contains a modified portion of 'Apache Commons Lang', a Java library + provides utilities for the java.lang API, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-lang.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/proper/commons-lang/ + + This product contains a forked and modified version of Tomcat Native + + * LICENSE: + * ASL2 + * HOMEPAGE: + * http://tomcat.apache.org/native-doc/ + * https://svn.apache.org/repos/asf/tomcat/native/ + + (ASLv2) Objenesis + The following NOTICE information applies: + Objenesis + Copyright 2006-2013 Joe Walnes, Henri Tremblay, Leonardo Mesquita + + (ASLv2) Snappy Java + The following NOTICE information applies: + This product includes software developed by Google + Snappy: http://code.google.com/p/snappy/ (New BSD License) + + This product includes software developed by Apache + PureJavaCrc32C from apache-hadoop-common http://hadoop.apache.org/ + (Apache 2.0 license) + + This library containd statically linked libstdc++. This inclusion is allowed by + "GCC RUntime Library Exception" + http://gcc.gnu.org/onlinedocs/libstdc++/manual/license.html + +************************ +Eclipse Public License 1.0 +************************ + +The following binary components are provided under the Eclipse Public License 1.0. See project link for details. + + (EPL 1.0) JNR Posix ( jnr.posix ) https://github.com/jnr/jnr-posix/blob/master/LICENSE.txt + +***************** +Public Domain +***************** + +The following binary components are provided to the 'Public Domain'. See project link for details. + + (Public Domain) XZ for Java (org.tukaani:xz:jar:1.5 - http://tukaani.org/xz/java.html diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/pom.xml b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/pom.xml new file mode 100644 index 000000000000..b6077d1f7681 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/pom.xml @@ -0,0 +1,126 @@ + + + + 4.0.0 + + + org.apache.nifi + nifi-cql-bundle + 2.12.0-SNAPSHOT + + + nifi-cql-processors + jar + + + + org.apache.nifi + nifi-api + + + org.apache.nifi + nifi-utils + + + org.apache.nifi + nifi-properties + + + org.apache.nifi + nifi-ssl-context-service-api + + + org.apache.avro + avro + + + org.apache.nifi + nifi-cql-services-api + 2.12.0-SNAPSHOT + provided + + + org.apache.nifi + nifi-record-serialization-service-api + compile + + + org.apache.nifi + nifi-record + compile + + + + org.apache.nifi + nifi-mock + + + org.apache.nifi + nifi-mock-record-utils + + + org.apache.commons + commons-text + + + + org.apache.nifi + nifi-record-path + 2.12.0-SNAPSHOT + compile + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + ban-database-client-dependencies + + enforce + + + + + + The processors talk to a database only through the CQLExecutionService + abstraction, never through a driver directly, so no database client + library belongs on this module's classpath at any scope. If an + integration test here genuinely needs a live driver, add it to a module + that owns a backend instead. + + true + + + org.apache.cassandra:java-driver-* + com.datastax.oss:* + + com.datastax.cassandra:* + + com.scylladb:* + + + + + + + + + + diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/AbstractCQLProcessor.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/AbstractCQLProcessor.java new file mode 100644 index 000000000000..7a8790877848 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/AbstractCQLProcessor.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.cql; + +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.AbstractProcessor; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.service.cql.api.service.CQLExecutionService; + +import java.util.concurrent.atomic.AtomicReference; + +/** + * AbstractCQLProcessor is a base class for Cassandra processors and contains logic and variables common to most + * processors integrating with Apache Cassandra. + */ +public abstract class AbstractCQLProcessor extends AbstractProcessor { + + // Common descriptors + static final PropertyDescriptor CONNECTION_PROVIDER_SERVICE = new PropertyDescriptor.Builder() + .name("Cassandra Connection Provider") + .description("Specifies the Cassandra connection providing controller service to be used to connect to Cassandra cluster.") + .required(true) + .identifiesControllerService(CQLExecutionService.class) + .build(); + + static final Relationship REL_SUCCESS = new Relationship.Builder() + .name("success") + .description("A FlowFile is transferred to this relationship if the operation completed successfully.") + .build(); + + static final Relationship REL_FAILURE = new Relationship.Builder() + .name("failure") + .description("A FlowFile is transferred to this relationship if the operation failed.") + .build(); + + static final Relationship REL_RETRY = new Relationship.Builder().name("retry") + .description("A FlowFile is transferred to this relationship if the operation cannot be completed but attempting " + + "it again may succeed.") + .build(); + + protected final AtomicReference cqlSessionService = new AtomicReference<>(null); + + @OnScheduled + public void onScheduled(ProcessContext context) { + CQLExecutionService sessionProvider = context.getProperty(CONNECTION_PROVIDER_SERVICE).asControllerService(CQLExecutionService.class); + cqlSessionService.set(sessionProvider); + } + + public void stop(ProcessContext context) { + + } +} + diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/ExecuteCQLQueryCallback.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/ExecuteCQLQueryCallback.java new file mode 100644 index 000000000000..621be99922b9 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/ExecuteCQLQueryCallback.java @@ -0,0 +1,230 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.cql; + +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.flowfile.attributes.FragmentAttributes; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.serialization.RecordSetWriter; +import org.apache.nifi.serialization.RecordSetWriterFactory; +import org.apache.nifi.serialization.record.RecordSchema; +import org.apache.nifi.service.cql.api.service.CQLQueryCallback; + +import java.io.Closeable; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import static org.apache.nifi.processors.cql.AbstractCQLProcessor.REL_SUCCESS; +import static org.apache.nifi.processors.cql.ExecuteCQLQueryRecord.REL_ORIGINAL; + +public class ExecuteCQLQueryCallback implements CQLQueryCallback { + private final ProcessSession session; + private RecordSetWriterFactory writerFactory; + private RecordSetWriter recordWriter; + private ComponentLog logger; + + private long currentIndex = 0; + private long rowsPerFlowFile; + private long flowFilesPerBatch; + private FlowFile parentFlowFile; + + private boolean commitImmediately; + private final List flowFileBatch; + private FlowFile currentFlowFile; + + private int fragmentIndex; + private long currentRecordCount; + private UUID fragmentId; + + public ExecuteCQLQueryCallback(FlowFile parentFlowFile, + RecordSetWriterFactory writerFactory, + ProcessSession session, + ComponentLog logger, + long rowsPerFlowfile, + long flowFilesPerBatch) { + this.parentFlowFile = parentFlowFile; + this.writerFactory = writerFactory; + this.session = session; + this.logger = logger; + + this.commitImmediately = flowFilesPerBatch > 0; + this.rowsPerFlowFile = rowsPerFlowfile; + this.flowFilesPerBatch = flowFilesPerBatch; + + this.flowFileBatch = new ArrayList<>(); + this.fragmentIndex = 0; + this.currentRecordCount = 0L; + this.fragmentId = UUID.randomUUID(); + } + + /** + * @return true if no rows were ever received, meaning no FlowFiles were created. The caller is responsible + * for routing the incoming FlowFile (if any) in that case, since this callback never gets a chance to. + */ + public boolean isEmpty() { + return recordWriter == null; + } + + private void updateFlowFileAttributes() { + Map attributes = Map.of( + FragmentAttributes.FRAGMENT_ID.key(), fragmentId.toString(), + FragmentAttributes.FRAGMENT_INDEX.key(), String.valueOf(fragmentIndex++), + "mime.type", recordWriter.getMimeType(), + "record.count", String.valueOf(currentRecordCount)); + + this.currentFlowFile = session.putAllAttributes(currentFlowFile, attributes); + flowFileBatch.add(currentFlowFile); + } + + /** + * fragment.count can only be known once the whole result set has been processed. When Output Batch Size + * commits FlowFiles early, earlier FlowFiles are already released to REL_SUCCESS before the total is known, + * so fragment.count is intentionally left unset in that mode, matching the OUTPUT_BATCH_SIZE property + * description. Otherwise, every FlowFile in flowFileBatch is still held here, so the true total is known + * and applied to all of them. + */ + private void finalizeFragmentCounts() { + if (!commitImmediately) { + final String totalFragmentCount = String.valueOf(flowFileBatch.size()); + flowFileBatch.replaceAll(flowFile -> session.putAttribute(flowFile, FragmentAttributes.FRAGMENT_COUNT.key(), totalFragmentCount)); + } + } + + /** + * session.remove(FlowFile) refuses to remove a FlowFile with an open OutputStream/callback from + * session.write(FlowFile), so any writer or stream tied to a FlowFile being discarded on an error path + * must be closed first. + */ + private void closeQuietly(Closeable closeable) { + if (closeable != null) { + try { + closeable.close(); + } catch (Exception ex) { + logger.error("Error closing {}", closeable, ex); + } + } + } + + private void removeUnfinishedFlowFiles() { + flowFileBatch.remove(currentFlowFile); + if (currentFlowFile != null) { + session.remove(currentFlowFile); + } + flowFileBatch.forEach(session::remove); + flowFileBatch.clear(); + } + + private void initWriter(RecordSchema schema) { + if (recordWriter != null) { + try { + recordWriter.finishRecordSet(); + recordWriter.close(); + + updateFlowFileAttributes(); + currentRecordCount = 0L; + + if (commitImmediately && flowFileBatch.size() == flowFilesPerBatch) { + session.transfer(flowFileBatch, REL_SUCCESS); + + if (parentFlowFile != null) { + session.transfer(parentFlowFile, REL_ORIGINAL); + parentFlowFile = null; + } + + session.commitAsync(); + flowFileBatch.clear(); + } + } catch (Exception ex) { + closeQuietly(recordWriter); + removeUnfinishedFlowFiles(); + + throw new ProcessException("Error closing record writer", ex); + } + } + + final FlowFile newFlowFile = session.create(); + final OutputStream out = session.write(newFlowFile); + try { + recordWriter = writerFactory.createWriter(logger, schema, out, newFlowFile); + recordWriter.beginRecordSet(); + currentFlowFile = newFlowFile; + } catch (Exception ex) { + closeQuietly(out); + + session.remove(newFlowFile); + flowFileBatch.forEach(session::remove); + flowFileBatch.clear(); + + throw new ProcessException("Error creating record writer", ex); + } + } + + @Override + public void receive(long rowNumber, + org.apache.nifi.serialization.record.Record result, boolean hasMore) { + final boolean shouldRotateWriter = recordWriter == null + || (rowsPerFlowFile > 0 && ++currentIndex % rowsPerFlowFile == 0); + + if (shouldRotateWriter) { + initWriter(result.getSchema()); + } + + try { + recordWriter.write(result); + this.currentRecordCount++; + + if (!hasMore) { + recordWriter.finishRecordSet(); + recordWriter.close(); + + updateFlowFileAttributes(); + finalizeFragmentCounts(); + + if (parentFlowFile != null) { + session.transfer(parentFlowFile, REL_ORIGINAL); + parentFlowFile = null; + } + + session.transfer(flowFileBatch, REL_SUCCESS); + flowFileBatch.clear(); + } + + } catch (Exception ex) { + closeQuietly(recordWriter); + removeUnfinishedFlowFiles(); + + throw new ProcessException("Error writing record", ex); + } + } + + @Override + public void clear() { + closeQuietly(recordWriter); + removeUnfinishedFlowFiles(); + } + + @Override + public boolean hasSentOriginal() { + return parentFlowFile == null; + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/ExecuteCQLQueryRecord.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/ExecuteCQLQueryRecord.java new file mode 100644 index 000000000000..ccb751c377b3 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/ExecuteCQLQueryRecord.java @@ -0,0 +1,402 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.cql; + +import org.apache.nifi.annotation.behavior.DynamicProperty; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.SystemResource; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.SeeAlso; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.documentation.UseCase; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.serialization.RecordSetWriterFactory; +import org.apache.nifi.service.cql.api.exception.QueryFailureException; +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.service.cql.api.service.QueryOverrides; +import org.apache.nifi.util.StopWatch; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Tags({"cassandra", "scylladb", "cql", "select"}) +@InputRequirement(InputRequirement.Requirement.INPUT_ALLOWED) +@CapabilityDescription("Execute provided Cassandra Query Language (CQL) select query on a data store that supports CQL (Cassandra or ScyllaDB primarily). Using a" + + " configured record writer service, it will convert result rows into any output format supported by NiFi's record API.") +@WritesAttributes({ + @WritesAttribute(attribute = "fragment.identifier", description = "If 'Max Rows Per Flow File' is set then all FlowFiles from the same query result set " + + "will have the same value for the fragment.identifier attribute. This can then be used to correlate the results."), + @WritesAttribute(attribute = "fragment.count", description = "If 'Max Rows Per Flow File' is set then this is the total number of " + + "FlowFiles produced by a single ResultSet. This can be used in conjunction with the " + + "fragment.identifier attribute in order to know how many FlowFiles belonged to the same incoming ResultSet. If Output Batch Size is set, then this " + + "attribute will not be populated."), + @WritesAttribute(attribute = "fragment.index", description = "If 'Max Rows Per Flow File' is set then the position of this FlowFile in the list of " + + "outgoing FlowFiles that were all derived from the same result set FlowFile. This can be " + + "used in conjunction with the fragment.identifier attribute to know which FlowFiles originated from the same query result set and in what order " + + "FlowFiles were produced") +}) +@DynamicProperty(name = "cql.arg.", value = "The value to bind to that bind marker", + expressionLanguageScope = ExpressionLanguageScope.FLOWFILE_ATTRIBUTES, + description = "Supplies the value for one '?' bind marker in the CQL select query, where is the marker's " + + "1-based position in the query - cql.arg.1 for the first, cql.arg.2 for the second, and so on. Positions " + + "must run consecutively from 1, and the number of parameters must match the number of bind markers in the " + + "query. Each value is sent to the cluster as data and is never parsed as CQL, so this is the safe way to " + + "build a query around a value taken from a FlowFile attribute. See 'Additional Details'.") +@SystemResourceConsideration(resource = SystemResource.MEMORY, + description = "With the default 'Max Rows Per Flow File' of 0, an entire result set is written to a single " + + "FlowFile; with the default 'Output Batch Size' of 0, every output FlowFile is held in the session " + + "until the whole result set has been read. Set both when querying large tables.") +@SeeAlso( + value = {PutCQLRecord.class}, + // The session provider services cannot be referenced by class: this module is barred from depending on either + // of them - and so on the database drivers they carry - by the ban-database-client-dependencies enforcer rule. + classNames = { + "org.apache.nifi.service.cassandra.CassandraCQLExecutionService", + "org.apache.nifi.service.scylladb.ScyllaDBCQLExecutionService" + }) +@UseCase( + description = "Run a fixed CQL query on a schedule and emit the results as records.", + inputRequirement = InputRequirement.Requirement.INPUT_FORBIDDEN, + keywords = {"cassandra", "scylladb", "cql", "select", "query", "source"}, + notes = "A scheduled processor runs on every node of a NiFi cluster, so the query is executed once per node and " + + "each node emits its own copy of the result. Set the processor's Execution to 'Primary node only' if " + + "a single copy is wanted. Note also that every run re-executes the whole query: this processor keeps " + + "no state, so there is no built-in way to fetch only rows that are new since the last run.", + configuration = """ + Give the processor no incoming connection and schedule it on a timer. + + Set "CQL select query" to the query to run and "Result Set Output Writer" to a record writer for the \ + desired output format. + + Set "Max Rows Per Flow File" to split a large result set across several FlowFiles, and "Output Batch \ + Size" to release those FlowFiles downstream as the result set is read rather than all at once when it \ + completes. + """) +@UseCase( + description = "Query a table using values taken from an incoming FlowFile, without exposing the query to CQL injection.", + inputRequirement = InputRequirement.Requirement.INPUT_REQUIRED, + keywords = {"cassandra", "scylladb", "cql", "select", "query", "parameter", "bind"}, + notes = "Anything interpolated into the query text with Expression Language is parsed as CQL, so a query built " + + "that way from FlowFile attributes is injectable. Bind markers are not: each cql.arg. value " + + "is sent to the cluster as data. Prefer bind markers whenever a value originates outside the flow's " + + "own configuration.", + configuration = """ + Write the query with '?' bind markers in place of the values, for example: \ + SELECT * FROM my_keyspace.events WHERE id = ? + + Add one dynamic property per marker, named cql.arg.1, cql.arg.2 and so on in the order the markers \ + appear, with each value supplied by Expression Language against the FlowFile's attributes. The \ + positions must run consecutively from 1, and the count must match the number of markers, or the \ + processor is invalid. + + The incoming FlowFile is routed to 'original' once the query completes; result records leave via \ + 'success'. + """) +public class ExecuteCQLQueryRecord extends AbstractCQLProcessor { + + /** + * Matches the dynamic property name for a positional query parameter - {@code cql.arg.1}, {@code cql.arg.2}, + * and so on - capturing the 1-based position of the bind marker the property supplies a value for. + */ + private static final Pattern QUERY_PARAMETER_PATTERN = Pattern.compile("^cql\\.arg\\.(?[1-9]\\d*)$"); + + static final PropertyDescriptor CQL_SELECT_QUERY = new PropertyDescriptor.Builder() + .name("CQL select query") + .description(""" + CQL select query. Values that come from outside the flow's own configuration - a FlowFile \ + attribute, for example - should be supplied as '?' bind markers with matching cql.arg. \ + dynamic properties rather than interpolated into this query text, since anything interpolated here \ + is parsed as CQL.""") + .required(true) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .build(); + + static final PropertyDescriptor QUERY_TIMEOUT = new PropertyDescriptor.Builder() + .name("Max Wait Time") + .description(""" + The maximum amount of time allowed for this query to run, overriding the Read Timeout configured on the \ + connection service for this query only. Must be of format where is a \ + non-negative integer and TimeUnit is a supported Time Unit, such as: nanos, millis, secs, mins, hrs, days. \ + If not set, the connection service's configured Read Timeout is used.""") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .build(); + + static final PropertyDescriptor FETCH_SIZE = new PropertyDescriptor.Builder() + .name("Fetch Size") + .description(""" + The number of result rows to be fetched from the result set at a time, overriding the Fetch Size \ + configured on the connection service for this query only. If not set, the connection service's \ + configured Fetch Size is used.""") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.INTEGER_VALIDATOR) + .build(); + + static final PropertyDescriptor MAX_ROWS_PER_FLOW_FILE = new PropertyDescriptor.Builder() + .name("Max Rows Per Flow File") + .description(""" + The maximum number of result rows that will be included in a single FlowFile. This will allow you to break up very large \ + result sets into multiple FlowFiles. If the value specified is zero, then all rows are returned in a single FlowFile.""") + .defaultValue("0") + .required(true) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .addValidator(StandardValidators.INTEGER_VALIDATOR) + .build(); + + static final PropertyDescriptor OUTPUT_BATCH_SIZE = new PropertyDescriptor.Builder() + .name("Output Batch Size") + .description(""" + The number of output FlowFiles to queue before committing the process session. When set to zero, the session will be committed when all result set rows \ + have been processed and the output FlowFiles are ready for transfer to the downstream relationship. For large result sets, this can cause a large burst of FlowFiles \ + to be transferred at the end of processor execution. If this property is set, then when the specified number of FlowFiles are ready for transfer, then the session will \ + be committed, thus releasing the FlowFiles to the downstream relationship. NOTE: The maxvalue.* and fragment.count attributes will not be set on FlowFiles when this \ + property is set.""") + .defaultValue("0") + .required(true) + .addValidator(StandardValidators.NON_NEGATIVE_INTEGER_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .build(); + + static final PropertyDescriptor OUTPUT_WRITER = new PropertyDescriptor.Builder() + .name("Result Set Output Writer") + .identifiesControllerService(RecordSetWriterFactory.class) + .required(true) + .description("The controller service to use for writing the results to a flowfile") + .build(); + + static final Relationship REL_ORIGINAL = new Relationship.Builder() + .autoTerminateDefault(true) + .name("original") + .description("The incoming FlowFile that triggered the query is routed here once every resulting " + + "FlowFile has been transferred to success, or immediately if the query returned no rows. " + + "On a failed query the incoming FlowFile goes to failure or retry instead, never here.") + .build(); + + static final List PROPERTY_DESCRIPTORS = List.of( + CONNECTION_PROVIDER_SERVICE, + OUTPUT_WRITER, + CQL_SELECT_QUERY, + FETCH_SIZE, + QUERY_TIMEOUT, + MAX_ROWS_PER_FLOW_FILE, + OUTPUT_BATCH_SIZE + ); + + static final Set RELATIONSHIPS = Set.of(REL_SUCCESS, REL_ORIGINAL, REL_FAILURE, REL_RETRY); + + @Override + public Set getRelationships() { + return RELATIONSHIPS; + } + + @Override + public final List getSupportedPropertyDescriptors() { + return PROPERTY_DESCRIPTORS; + } + + @Override + protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(final String propertyDescriptorName) { + final Matcher matcher = QUERY_PARAMETER_PATTERN.matcher(propertyDescriptorName); + + if (!matcher.matches()) { + throw new IllegalArgumentException(String.format( + "'%s' is not a valid query parameter name; positional parameters are named cql.arg.1, cql.arg.2, and so on", + propertyDescriptorName)); + } + + return new PropertyDescriptor.Builder() + .dynamic(true) + .name(propertyDescriptorName) + .description(String.format("The value bound to bind marker %s of the CQL select query.", matcher.group("position"))) + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR) + .build(); + } + + /** + * Rejects a gap or a non-1 start in the positional parameter numbering. The parameters are bound by + * position, so {@code cql.arg.1} plus {@code cql.arg.3} is ambiguous rather than merely unusual - it would + * otherwise silently bind the second marker with the third parameter's value. + */ + @Override + protected Collection customValidate(final ValidationContext context) { + final List positions = getParameterPositions(context.getProperties().keySet()); + + for (int index = 0; index < positions.size(); index++) { + final int expected = index + 1; + + if (positions.get(index) != expected) { + return List.of(new ValidationResult.Builder() + .subject("Query parameters") + .valid(false) + .explanation(String.format( + "positional query parameters must be numbered consecutively starting at 1, but cql.arg.%d is missing", + expected)) + .build()); + } + } + + return List.of(); + } + + private static List getParameterPositions(final Collection descriptors) { + return descriptors.stream() + .filter(PropertyDescriptor::isDynamic) + .map(descriptor -> QUERY_PARAMETER_PATTERN.matcher(descriptor.getName())) + .filter(Matcher::matches) + .map(matcher -> Integer.valueOf(matcher.group("position"))) + .sorted() + .toList(); + } + + /** + * Collects the {@code cql.arg.} dynamic properties into the positional order the bind markers + * expect, evaluating each against {@code flowFile}'s attributes. Numbering is already known to be + * consecutive from 1 by {@link #customValidate(ValidationContext)}, so sorting by position is enough. + */ + private List getQueryParameters(final ProcessContext context, final FlowFile flowFile) { + final SortedMap parametersByPosition = new TreeMap<>(); + + for (final PropertyDescriptor descriptor : context.getProperties().keySet()) { + if (!descriptor.isDynamic()) { + continue; + } + + final Matcher matcher = QUERY_PARAMETER_PATTERN.matcher(descriptor.getName()); + + if (matcher.matches()) { + parametersByPosition.put(Integer.valueOf(matcher.group("position")), + context.getProperty(descriptor).evaluateAttributeExpressions(flowFile).getValue()); + } + } + + return new ArrayList<>(parametersByPosition.values()); + } + + @OnScheduled + @Override + public void onScheduled(final ProcessContext context) { + super.onScheduled(context); + } + + @Override + public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException { + FlowFile fileToProcess = null; + + if (context.hasIncomingConnection()) { + fileToProcess = session.get(); + + // If we have no FlowFile, and all incoming connections are self-loops then we can continue on. + // However, if we have no FlowFile and we have connections coming from other Processors, then + // we know that we should run only if we have a FlowFile. + if (fileToProcess == null && context.hasNonLoopConnection()) { + return; + } + } + + final ComponentLog logger = getLogger(); + final String selectQuery = context.getProperty(CQL_SELECT_QUERY).evaluateAttributeExpressions(fileToProcess).getValue(); + final long maxRowsPerFlowFile = context.getProperty(MAX_ROWS_PER_FLOW_FILE).evaluateAttributeExpressions().asInteger(); + final long outputBatchSize = context.getProperty(OUTPUT_BATCH_SIZE).evaluateAttributeExpressions().asInteger(); + + final PropertyValue fetchSizeProperty = context.getProperty(FETCH_SIZE).evaluateAttributeExpressions(fileToProcess); + final Integer fetchSizeOverride = fetchSizeProperty.isSet() ? fetchSizeProperty.asInteger() : null; + + final PropertyValue queryTimeoutProperty = context.getProperty(QUERY_TIMEOUT).evaluateAttributeExpressions(fileToProcess); + final Duration queryTimeoutOverride = queryTimeoutProperty.isSet() ? queryTimeoutProperty.asDuration() : null; + + final QueryOverrides queryOverrides = new QueryOverrides(fetchSizeOverride, queryTimeoutOverride); + final List queryParameters = getQueryParameters(context, fileToProcess); + + final StopWatch stopWatch = new StopWatch(true); + + final RecordSetWriterFactory writerFactory = context.getProperty(OUTPUT_WRITER).asControllerService(RecordSetWriterFactory.class); + final CQLExecutionService cqlExecutionService = context.getProperty(CONNECTION_PROVIDER_SERVICE) + .asControllerService(CQLExecutionService.class); + + final ExecuteCQLQueryCallback callback = new ExecuteCQLQueryCallback(fileToProcess, writerFactory, session, + getLogger(), maxRowsPerFlowFile, outputBatchSize); + + try { + stopWatch.start(); + + cqlExecutionService.query(selectQuery, queryParameters, callback, queryOverrides); + + if (callback.isEmpty() && fileToProcess != null) { + session.transfer(fileToProcess, REL_ORIGINAL); + } + + stopWatch.stop(); + + getLogger().debug("The query took {} seconds.", stopWatch.getDuration(TimeUnit.SECONDS)); + } catch (final QueryFailureException qee) { + //The logger is called in the client service + if (context.hasIncomingConnection()) { + if (fileToProcess == null || callback.hasSentOriginal()) { + fileToProcess = session.create(); + } + fileToProcess = session.penalize(fileToProcess); + session.transfer(fileToProcess, REL_RETRY); + } else { + context.yield(); + } + } catch (final ProcessException e) { + if (context.hasIncomingConnection()) { + logger.error(String.format("Unable to execute CQL select query %s for %s routing to failure", + selectQuery, fileToProcess), e); + if (fileToProcess == null || callback.hasSentOriginal()) { + fileToProcess = session.create(); + } + + fileToProcess = session.penalize(fileToProcess); + session.transfer(fileToProcess, REL_FAILURE); + + } else { + logger.error(String.format("Unable to execute CQL select query %s", + selectQuery), e); + context.yield(); + } + } + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/PutCQLRecord.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/PutCQLRecord.java new file mode 100644 index 000000000000..e10dff70e57c --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/PutCQLRecord.java @@ -0,0 +1,624 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.cql; + +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.behavior.DynamicProperty; +import org.apache.nifi.annotation.behavior.InputRequirement; +import org.apache.nifi.annotation.behavior.ReadsAttribute; +import org.apache.nifi.annotation.behavior.ReadsAttributes; +import org.apache.nifi.annotation.behavior.SupportsBatching; +import org.apache.nifi.annotation.behavior.SystemResource; +import org.apache.nifi.annotation.behavior.SystemResourceConsideration; +import org.apache.nifi.annotation.behavior.WritesAttribute; +import org.apache.nifi.annotation.behavior.WritesAttributes; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.SeeAlso; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.documentation.UseCase; +import org.apache.nifi.annotation.lifecycle.OnScheduled; +import org.apache.nifi.annotation.lifecycle.OnShutdown; +import org.apache.nifi.annotation.lifecycle.OnUnscheduled; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; +import org.apache.nifi.components.ValidationContext; +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.flowfile.FlowFile; +import org.apache.nifi.processor.ProcessContext; +import org.apache.nifi.processor.ProcessSession; +import org.apache.nifi.processor.Relationship; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.processors.cql.constants.BatchStatementType; +import org.apache.nifi.processors.cql.constants.StatementType; +import org.apache.nifi.processors.cql.constants.UpdateType; +import org.apache.nifi.record.path.RecordPath; +import org.apache.nifi.record.path.util.RecordPathCache; +import org.apache.nifi.record.path.validation.RecordPathValidator; +import org.apache.nifi.serialization.RecordReader; +import org.apache.nifi.serialization.RecordReaderFactory; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordSchema; +import org.apache.nifi.service.cql.api.constants.CqlBatchType; +import org.apache.nifi.service.cql.api.constants.UpdateMethod; +import org.apache.nifi.service.cql.api.exception.QueryFailureException; +import org.apache.nifi.service.cql.api.metadata.PrimaryKeyIdentifier; +import org.apache.nifi.service.cql.api.metadata.QualifiedTableName; +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.service.cql.api.service.WriteOverrides; +import org.apache.nifi.util.StopWatch; + +import java.io.InputStream; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static java.lang.String.format; +import static org.apache.nifi.processors.cql.constants.BatchStatementType.BATCH_STATEMENT_TYPE_USE_ATTR_TYPE; +import static org.apache.nifi.processors.cql.constants.BatchStatementType.COUNTER_TYPE; +import static org.apache.nifi.processors.cql.constants.BatchStatementType.UNLOGGED_TYPE; +import static org.apache.nifi.processors.cql.constants.StatementType.INSERT_TYPE; +import static org.apache.nifi.processors.cql.constants.StatementType.STATEMENT_TYPE_USE_ATTR_TYPE; +import static org.apache.nifi.processors.cql.constants.StatementType.UPDATE_TYPE; +import static org.apache.nifi.processors.cql.constants.UpdateType.DECR_TYPE; +import static org.apache.nifi.processors.cql.constants.UpdateType.INCR_TYPE; + +@Tags({"cassandra", "scylladb", "cql", "put", "insert", "update", "set", "record"}) +@InputRequirement(InputRequirement.Requirement.INPUT_REQUIRED) +@CapabilityDescription("This is a record aware processor that reads the content of the incoming FlowFile as individual records using the " + + "configured 'Record Reader' and writes them to a data store that supports CQL (Cassandra or ScyllaDB primarily), as inserts or " + + "updates, individually or in batches.") +@ReadsAttributes({ + @ReadsAttribute(attribute = "cql.statement.type", description = "If 'Use cql.statement.type Attribute' is selected for the Statement " + + "Type property, the value of the cql.statement.type Attribute will be used to determine which type of statement (UPDATE, INSERT) " + + "will be generated and executed"), + @ReadsAttribute(attribute = "cql.update.method", description = "If 'Use cql.update.method Attribute' is selected for the Update " + + "Method property, the value of the cql.update.method Attribute will be used to determine which operation (Set, Increment, Decrement) " + + "will be used to generate and execute the Update statement. Ignored if the Statement Type property is not set to UPDATE"), + @ReadsAttribute(attribute = "cql.batch.statement.type", description = "If 'Use cql.batch.statement.type Attribute' is selected for the Batch " + + "Statement Type property, the value of the cql.batch.statement.type Attribute will be used to determine which type of batch statement " + + "(LOGGED, UNLOGGED, COUNTER) will be generated and executed") +}) +@WritesAttributes({ + @WritesAttribute(attribute = "cql.records.written", description = "On failure or retry, the number of records that were already successfully " + + "written to Cassandra/ScyllaDB before the error occurred. Since records are written in batches as they're read, earlier batches " + + "may have already been committed even though the FlowFile as a whole did not succeed.") +}) +@DynamicProperty(name = "..", value = "A RecordPath expression", + expressionLanguageScope = ExpressionLanguageScope.NONE, + description = "Overrides how the named primary key column's value is resolved for records written to the given " + + "keyspace-qualified table, in place of the default behavior of matching a record field with the same " + + "name as the column. The RecordPath is evaluated once per record and must resolve to exactly one " + + "value; zero or more than one is a configuration error for that record. See 'Additional Details' for " + + "the full name format and examples, including how to supply a valid version-1 (time-based) UUID for " + + "a timeuuid primary key column.") +@SupportsBatching +@SystemResourceConsideration(resource = SystemResource.MEMORY, + description = "Up to 'Batch size' records are held in memory at once, per concurrent task, in their parsed form " + + "rather than as the FlowFile's serialized bytes. Raising Batch size or the number of concurrent tasks " + + "raises heap use proportionally.") +@SeeAlso( + value = {ExecuteCQLQueryRecord.class}, + // The session provider services cannot be referenced by class: this module is barred from depending on either + // of them - and so on the database drivers they carry - by the ban-database-client-dependencies enforcer rule. + classNames = { + "org.apache.nifi.service.cassandra.CassandraCQLExecutionService", + "org.apache.nifi.service.scylladb.ScyllaDBCQLExecutionService" + }) +@UseCase( + description = "Insert records from a FlowFile into a Cassandra or ScyllaDB table.", + keywords = {"cassandra", "scylladb", "cql", "insert", "record"}, + configuration = """ + Configure a Record Reader appropriate to the incoming data and point "Cassandra Connection Provider" at \ + the session provider service for the cluster. + + Set "Table name" to the target table, either as .
or as an unqualified
if the \ + connection service already names a keyspace. + + Leave "Statement Type" at INSERT. Each record field is matched to the column of the same name; use a \ + .
. dynamic property to resolve a primary key column from somewhere else in the \ + record instead. + + "Batch size" controls how many records are grouped into a single batch statement. It is a throughput \ + and heap trade-off, not a correctness one - a FlowFile larger than the batch size is written across \ + several batches. + """) +@UseCase( + description = "Update existing rows in a Cassandra or ScyllaDB table from the records in a FlowFile.", + keywords = {"cassandra", "scylladb", "cql", "update", "record"}, + notes = "Cassandra and ScyllaDB do not distinguish an update from an insert: an UPDATE against a primary key " + + "that does not exist creates the row. Statement Type UPDATE differs from INSERT in that it writes " + + "only the columns present in the record, rather than replacing the whole row.", + configuration = """ + Set "Statement Type" to UPDATE and list the primary key columns in "Update Keys" as a comma-separated \ + list. Both are required together - an UPDATE with no update keys fails the FlowFile at runtime rather \ + than failing validation, because the statement type may itself come from a FlowFile attribute. + + Leave "Update Method" at SET to assign the record's values to the columns. + + "Time To Live" and "Timestamp Field" both apply to SET-method updates. Setting "Timestamp Field" to a \ + record field holding a stable timestamp makes reprocessing the same record a true no-op rather than a \ + write that races whatever else has touched the row since. + """) +@UseCase( + description = "Increment or decrement counter columns in a Cassandra or ScyllaDB counter table.", + keywords = {"cassandra", "scylladb", "cql", "counter", "increment", "decrement"}, + notes = "Counter mutations are the one CQL write that is not idempotent: applying the same increment twice " + + "counts twice. Leave Run Duration at 0 for counter flows, since a longer Run Duration allows the " + + "framework to batch session commits, and a rolled-back batch is reprocessed from the queue. For the " + + "same reason, prefer counter flows that can tolerate at-least-once delivery.", + configuration = """ + Set "Statement Type" to UPDATE, list the counter table's primary key columns in "Update Keys", and set \ + "Update Method" to Increment or Decrement. Each record field matching a counter column supplies the \ + amount to add or subtract. + + Set "Batch Statement Type" to COUNTER, which is the type Cassandra and ScyllaDB require for counter \ + mutations. UNLOGGED is also accepted; LOGGED is rejected. + + Statement Type INSERT cannot be used against a counter table and is rejected. + + "Time To Live" and "Timestamp Field" are ignored here - neither is supported on counter columns. + """) +public class PutCQLRecord extends AbstractCQLProcessor { + static final String STATEMENT_TYPE_ATTRIBUTE = "cql.statement.type"; + + static final String UPDATE_METHOD_ATTRIBUTE = "cql.update.method"; + + static final String BATCH_STATEMENT_TYPE_ATTRIBUTE = "cql.batch.statement.type"; + + static final String RECORDS_WRITTEN_ATTRIBUTE = "cql.records.written"; + + static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder() + .name("Record Reader") + .description(""" + Specifies the type of Record Reader controller service to use for parsing the incoming data \ + and determining the schema""") + .identifiesControllerService(RecordReaderFactory.class) + .required(true) + .build(); + + static final PropertyDescriptor STATEMENT_TYPE = new PropertyDescriptor.Builder() + .name("Statement Type") + .description("Specifies the type of CQL Statement to generate.") + .required(true) + .defaultValue(INSERT_TYPE.getValue()) + .allowableValues(StatementType.class) + .build(); + + static final PropertyDescriptor UPDATE_METHOD = new PropertyDescriptor.Builder() + .name("Update Method") + .description(""" + Specifies the method to use to SET the values. This property is used if the Statement Type is \ + UPDATE and ignored otherwise.""") + .required(false) + .defaultValue(UpdateType.SET_TYPE.getValue()) + .allowableValues(UpdateType.class) + .build(); + + static final PropertyDescriptor UPDATE_KEYS = new PropertyDescriptor.Builder() + .name("Update Keys") + .description(""" + A comma-separated list of column names that uniquely identifies a row in the database for UPDATE statements. \ + If the Statement Type is UPDATE and this property is not set, the conversion to CQL will fail. \ + This property is ignored if the Statement Type is not UPDATE.""") + .addValidator(StandardValidators.createListValidator(true, false, StandardValidators.NON_EMPTY_VALIDATOR)) + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .build(); + + static final PropertyDescriptor TABLE = new PropertyDescriptor.Builder() + .name("Table name") + .description(""" + The name of the Cassandra table to which the records have to be written. This can be expressed \ + as either a raw table name or a qualified table name (ex. .). Due to the dynamic \ + nature of this property, it will be validated at runtime by this processor and raise an error if \ + it is neither nor . when the value is retrieved.""") + .required(true) + .addValidator(StandardValidators.NON_EMPTY_EL_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .build(); + + static final PropertyDescriptor BATCH_SIZE = new PropertyDescriptor.Builder() + .name("Batch size") + .description("Specifies the number of 'Insert statements' to be grouped together to execute as a batch (BatchStatement)") + .defaultValue("100") + .addValidator(StandardValidators.POSITIVE_INTEGER_VALIDATOR) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .required(true) + .build(); + + static final PropertyDescriptor BATCH_STATEMENT_TYPE = new PropertyDescriptor.Builder() + .name("Batch Statement Type") + .description("Specifies the type of 'Batch Statement' to be used.") + .allowableValues(BatchStatementType.class) + .defaultValue(UNLOGGED_TYPE.getValue()) + .required(false) + .build(); + + static final PropertyDescriptor TTL = new PropertyDescriptor.Builder() + .name("Time To Live") + .description(""" + Overrides the connection service's configured Default Time To Live (TTL) for records written by this processor. \ + Applies to INSERT statements and to UPDATE statements using the SET method; ignored for Increment/Decrement updates, \ + since Cassandra/ScyllaDB do not support a TTL on counter columns. If not set, the connection service's configured \ + default (if any) is used.""") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .build(); + + static final PropertyDescriptor TIMESTAMP_FIELD = new PropertyDescriptor.Builder() + .name("Timestamp Field") + .description(""" + The name of a field in each record whose value supplies the CQL write timestamp for that record's INSERT or \ + SET-method UPDATE statement, instead of the time the statement executes. Useful for safe retries/reprocessing: \ + resubmitting the same record with the same timestamp is a true no-op rather than a write that could win a \ + last-write-wins race against different data written to the same row in the meantime. Ignored for Increment/Decrement \ + updates, since Cassandra/ScyllaDB do not support a custom write timestamp on counter columns. If not set, the current \ + time is used, as usual.""") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + private static final List PROPERTY_DESCRIPTORS = Collections.unmodifiableList(Arrays.asList( + CONNECTION_PROVIDER_SERVICE, TABLE, STATEMENT_TYPE, UPDATE_KEYS, UPDATE_METHOD, + RECORD_READER_FACTORY, BATCH_SIZE, BATCH_STATEMENT_TYPE, TTL, TIMESTAMP_FIELD)); + + private static final Set RELATIONSHIPS = Collections.unmodifiableSet( + new HashSet<>(Arrays.asList(REL_SUCCESS, REL_FAILURE, REL_RETRY))); + + private static final Pattern QUALIFIED_TABLE_PATTERN = Pattern.compile( + "^(?[a-zA-Z][a-zA-Z0-9_]{0,47})\\.(?
[a-zA-Z][a-zA-Z0-9_]{0,47})\\.(?[a-zA-Z][a-zA-Z0-9_]{0,47})$"); + private static final Pattern TABLE_REGEX = Pattern.compile( + "^(?:(?[a-zA-Z][a-zA-Z0-9_]{0,47})\\.)?(?
[a-zA-Z][a-zA-Z0-9_]{0,47})$"); + + private RecordPathCache recordPathCache; + + @Override + protected List getSupportedPropertyDescriptors() { + return PROPERTY_DESCRIPTORS; + } + + @Override + protected PropertyDescriptor getSupportedDynamicPropertyDescriptor(String propertyDescriptorName) { + Matcher matcher = QUALIFIED_TABLE_PATTERN.matcher(propertyDescriptorName); + + if (!matcher.matches()) { + throw new IllegalArgumentException(format("%s is not a valid qualified table name and field name", propertyDescriptorName)); + } + + String keyspace = matcher.group("keyspace"); + String table = matcher.group("table"); + String field = matcher.group("field"); + + return new PropertyDescriptor.Builder() + .dynamic(true) + .name(propertyDescriptorName) + .description(""" + This property sets a record path to be evaluated for field %s on table %s.%s + """.formatted(field, keyspace, table).trim()) + .defaultValue("") + .addValidator(new RecordPathValidator()) + .build(); + } + + @OnScheduled + @Override + public void onScheduled(ProcessContext context) { + super.onScheduled(context); + + this.recordPathCache = new RecordPathCache(50); + this.createRecordPathOverrides(context); + } + + private Map recordPathOverrides; + + private void createRecordPathOverrides(ProcessContext context) { + final Map overrides = new HashMap<>(); + + context + .getProperties() + .keySet() + .stream() + .filter(PropertyDescriptor::isDynamic) + .forEach(p -> { + String rawPath = context.getProperty(p).getValue(); + RecordPath compiledPath = recordPathCache.getCompiled(rawPath); + Matcher matcher = QUALIFIED_TABLE_PATTERN.matcher(p.getName()); + if (matcher.matches()) { + PrimaryKeyIdentifier identifier = new PrimaryKeyIdentifier(matcher.group("keyspace"), + matcher.group("table"), matcher.group("field")); + overrides.put(identifier, compiledPath); + } + }); + recordPathOverrides = new ConcurrentHashMap<>(overrides); + } + + private Map resolveSpecificOverrides(QualifiedTableName tableName) { + return new HashMap<>(recordPathOverrides + .entrySet() + .stream() + .filter(e -> e.getKey().keyspace().equals(tableName.keyspace()) + && e.getKey().tableName().equals(tableName.table())) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + } + + @Override + public Set getRelationships() { + return RELATIONSHIPS; + } + + @Override + public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException { + FlowFile inputFlowFile = session.get(); + + if (inputFlowFile == null) { + return; + } + + CQLExecutionService sessionProviderService = super.cqlSessionService.get(); + + final String cassandraTable = context.getProperty(TABLE).evaluateAttributeExpressions(inputFlowFile).getValue(); + final QualifiedTableName qualifiedTableName; + + try { + qualifiedTableName = qualifyTable(cassandraTable, sessionProviderService.getDefaultKeyspace()); + } catch (IllegalArgumentException e) { + getLogger().error("Error processing", e); + // Nothing has been read yet, but cql.records.written is promised on every failure, so say 0 explicitly. + session.transfer(session.putAttribute(inputFlowFile, RECORDS_WRITTEN_ATTRIBUTE, "0"), REL_FAILURE); + return; + } + + Map specificOverrides = resolveSpecificOverrides(qualifiedTableName); + + final RecordReaderFactory recordParserFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class); + final int batchSize = context.getProperty(BATCH_SIZE).evaluateAttributeExpressions().asInteger(); + final String updateKeys = context.getProperty(UPDATE_KEYS).evaluateAttributeExpressions(inputFlowFile).getValue(); + + final String statementTypeProperty = context.getProperty(STATEMENT_TYPE).getValue(); + String statementType = statementTypeProperty; + if (STATEMENT_TYPE_USE_ATTR_TYPE.getValue().equals(statementTypeProperty)) { + statementType = inputFlowFile.getAttribute(STATEMENT_TYPE_ATTRIBUTE); + } + + final String updateMethodProperty = context.getProperty(UPDATE_METHOD).getValue(); + String updateMethod = updateMethodProperty; + if (UpdateType.UPDATE_METHOD_USE_ATTR_TYPE.getValue().equals(updateMethodProperty)) { + final String updateMethodAttr = inputFlowFile.getAttribute(UPDATE_METHOD_ATTRIBUTE); + updateMethod = StringUtils.isBlank(updateMethodAttr) ? null : updateMethodAttr.toUpperCase(); + } + + + final String batchStatementTypeProperty = context.getProperty(BATCH_STATEMENT_TYPE).getValue(); + String batchStatementType = batchStatementTypeProperty; + + if (BATCH_STATEMENT_TYPE_USE_ATTR_TYPE.getValue().equals(batchStatementTypeProperty)) { + final String batchStatementTypeAttr = inputFlowFile.getAttribute(BATCH_STATEMENT_TYPE_ATTRIBUTE); + + if (StringUtils.isBlank(batchStatementTypeAttr)) { + getLogger().error("Batch Statement Type is not specified, FlowFile {}", inputFlowFile); + session.penalize(inputFlowFile); + session.transfer(inputFlowFile, REL_FAILURE); + return; + } + + batchStatementType = StringUtils.isBlank(batchStatementTypeAttr) ? null : batchStatementTypeAttr.toUpperCase(); + } + + final PropertyValue ttlProperty = context.getProperty(TTL).evaluateAttributeExpressions(inputFlowFile); + final Duration ttlOverride = ttlProperty.isSet() ? ttlProperty.asDuration() : null; + + final PropertyValue timestampFieldProperty = context.getProperty(TIMESTAMP_FIELD).evaluateAttributeExpressions(inputFlowFile); + final String timestampField = timestampFieldProperty.isSet() ? timestampFieldProperty.getValue() : null; + + final WriteOverrides writeOverrides = new WriteOverrides(ttlOverride, timestampField); + + final AtomicInteger recordsAdded = new AtomicInteger(0); + final StopWatch stopWatch = new StopWatch(true); + + boolean error = false; + + try (final InputStream inputStream = session.read(inputFlowFile); + final RecordReader reader = recordParserFactory.createRecordReader(inputFlowFile, inputStream, getLogger())) { + + if (StringUtils.isEmpty(statementType)) { + throw new IllegalArgumentException(format("Statement Type is not specified, FlowFile %s", inputFlowFile)); + } + + if (UPDATE_TYPE.getValue().equalsIgnoreCase(statementType) && StringUtils.isEmpty(updateKeys)) { + throw new IllegalArgumentException(format("Update Keys are not specified, FlowFile %s", inputFlowFile)); + } + + if (UPDATE_TYPE.getValue().equalsIgnoreCase(statementType) && StringUtils.isBlank(updateMethod)) { + throw new IllegalArgumentException(format("Update Method is not specified, FlowFile %s", inputFlowFile)); + } + + final List updateKeyNames = UPDATE_TYPE.getValue().equalsIgnoreCase(statementType) ? Stream.of(updateKeys + .split(",")) + .map(key -> key.trim()) + .filter(key -> StringUtils.isNotEmpty(key)) + .toList() : List.of(); + + if (INCR_TYPE.getValue().equalsIgnoreCase(updateMethod) || DECR_TYPE.getValue().equalsIgnoreCase(updateMethod)) { + if (!(UNLOGGED_TYPE.getValue().equalsIgnoreCase(batchStatementType) || COUNTER_TYPE.getValue().equalsIgnoreCase(batchStatementType))) { + throw new IllegalArgumentException(format("Increment/Decrement Update Method can only be used with COUNTER " + + "or UNLOGGED Batch Statement Type, FlowFile %s", inputFlowFile)); + } + } + + // throw an exception if the statement type is INSERT and the batch statement type is COUNTER: Cassandra/ScyllaDB + // do not allow INSERT statements against counter tables, only UPDATE + if (INSERT_TYPE.getValue().equalsIgnoreCase(statementType) && COUNTER_TYPE.getValue().equalsIgnoreCase(batchStatementType)) { + throw new IllegalArgumentException(format("Statement Type INSERT cannot be used with COUNTER Batch Statement Type, FlowFile %s", inputFlowFile)); + } + + // throw an exception if a Timestamp Field is configured but isn't actually a field in the incoming + // records - checked unconditionally (regardless of whether the resolved statement/update method + // would even use it) so a mistyped field name always fails the same way, rather than only + // sometimes, depending on which attribute-driven statement type a given FlowFile happens to take. + if (StringUtils.isNotEmpty(timestampField)) { + final RecordSchema recordSchema = reader.getSchema(); + if (!recordSchema.getFieldNames().contains(timestampField)) { + throw new IllegalArgumentException(format("Timestamp Field '%s' is not present in the record schema, FlowFile %s", timestampField, inputFlowFile)); + } + } + + Record record; + List recordsBatch = new ArrayList<>(); + + final boolean isUpdate = UPDATE_TYPE.getValue().equalsIgnoreCase(statementType); + final UpdateMethod resolvedUpdateMethod = isUpdate ? UpdateMethod.valueOf(updateMethod) : null; + final CqlBatchType resolvedBatchType = CqlBatchType.valueOf(batchStatementType); + + // Writes and clears the current batch, tracking how many records have been written so far so that + // a mid-stream failure can report an accurate count instead of leaving it at 0, and emits a + // provenance event per batch so a partial failure still leaves a trail of what was actually written. + final Runnable flushBatch = () -> { + final int batchRecordCount = recordsBatch.size(); + + if (isUpdate) { + sessionProviderService.update(qualifiedTableName, recordsBatch, specificOverrides, updateKeyNames, resolvedUpdateMethod, resolvedBatchType, writeOverrides); + } else { + sessionProviderService.insert(qualifiedTableName, recordsBatch, specificOverrides, resolvedBatchType, writeOverrides); + } + + recordsAdded.addAndGet(batchRecordCount); + session.getProvenanceReporter().send(inputFlowFile, sessionProviderService.getTransitUrl(qualifiedTableName), + format("Wrote %d records (%d total)", batchRecordCount, recordsAdded.get()), stopWatch.getElapsed(TimeUnit.MILLISECONDS)); + recordsBatch.clear(); + }; + + while ((record = reader.nextRecord()) != null) { + recordsBatch.add(record); + + if (recordsBatch.size() == batchSize) { + flushBatch.run(); + } + } + + if (!recordsBatch.isEmpty()) { + flushBatch.run(); + } + + } catch (QueryFailureException qfe) { + error = true; + final FlowFile flowFileToRetry = session.putAttribute(inputFlowFile, RECORDS_WRITTEN_ATTRIBUTE, String.valueOf(recordsAdded.get())); + getLogger().error("Cassandra query failure writing records for {}, {} records already written this attempt, routing to retry", + inputFlowFile, recordsAdded.get(), qfe); + session.transfer(session.penalize(flowFileToRetry), REL_RETRY); + } catch (IllegalArgumentException e) { + error = true; + // Also reachable mid-stream (an override RecordPath resolving to zero values on a later record), + // so earlier batches may already be committed - report them, as the failure paths below do. + final FlowFile failedFlowFile = session.putAttribute(inputFlowFile, RECORDS_WRITTEN_ATTRIBUTE, String.valueOf(recordsAdded.get())); + getLogger().error("Invalid PutCQLRecord configuration for {}", inputFlowFile, e); + session.transfer(failedFlowFile, REL_FAILURE); + } catch (Exception e) { + error = true; + final FlowFile failedFlowFile = session.putAttribute(inputFlowFile, RECORDS_WRITTEN_ATTRIBUTE, String.valueOf(recordsAdded.get())); + getLogger().error("Unable to write the records into Cassandra table, {} records already written this attempt", recordsAdded.get(), e); + session.transfer(failedFlowFile, REL_FAILURE); + } finally { + if (!error) { + session.transfer(inputFlowFile, REL_SUCCESS); + } + } + + } + + @Override + protected Collection customValidate(ValidationContext validationContext) { + Set results = new HashSet<>(super.customValidate(validationContext)); + + String statementType = validationContext.getProperty(STATEMENT_TYPE).getValue(); + String batchStatementType = validationContext.getProperty(BATCH_STATEMENT_TYPE).getValue(); + + if (UPDATE_TYPE.getValue().equalsIgnoreCase(statementType)) { + // Check that update keys are set + String updateKeys = validationContext.getProperty(UPDATE_KEYS).getValue(); + if (StringUtils.isEmpty(updateKeys)) { + results.add(new ValidationResult.Builder().subject("Update statement configuration").valid(false).explanation( + "if the Statement Type is set to Update, then the Update Keys must be specified as well").build()); + } + + // Check that if the update method is set to increment or decrement that the batch statement type is set to + // unlogged or counter (or USE_ATTR_TYPE, which we cannot check at this point). + String updateMethod = validationContext.getProperty(UPDATE_METHOD).getValue(); + if (INCR_TYPE.getValue().equalsIgnoreCase(updateMethod) + || DECR_TYPE.getValue().equalsIgnoreCase(updateMethod)) { + if (!(COUNTER_TYPE.getValue().equalsIgnoreCase(batchStatementType) + || UNLOGGED_TYPE.getValue().equalsIgnoreCase(batchStatementType) + || BATCH_STATEMENT_TYPE_USE_ATTR_TYPE.getValue().equalsIgnoreCase(batchStatementType))) { + results.add(new ValidationResult.Builder().subject("Update method configuration").valid(false).explanation( + "if the Update Method is set to Increment or Decrement, then the Batch Statement Type must be set " + + "to either COUNTER or UNLOGGED").build()); + } + } + } else if (INSERT_TYPE.getValue().equalsIgnoreCase(statementType) + && COUNTER_TYPE.getValue().equalsIgnoreCase(batchStatementType)) { + // Cassandra/ScyllaDB do not allow INSERT statements against counter tables, only UPDATE, so this + // combination can never succeed (USE_ATTR_TYPE for either property cannot be checked at this point). + results.add(new ValidationResult.Builder().subject("Batch Statement Type configuration").valid(false).explanation( + "if the Statement Type is set to INSERT, then the Batch Statement Type cannot be set to COUNTER, since " + + "Cassandra/ScyllaDB only allow COUNTER batches against UPDATE statements").build()); + } + + return results; + } + + + @OnUnscheduled + @Override + public void stop(ProcessContext context) { + super.stop(context); + } + + @OnShutdown + public void shutdown(ProcessContext context) { + super.stop(context); + } + + private QualifiedTableName qualifyTable(String tableName, String providerDefault) { + Matcher matcher = TABLE_REGEX.matcher(tableName); + if (!matcher.matches()) { + throw new IllegalArgumentException(format("%s is not a valid table name", tableName)); + } + + String keyspace = matcher.group("keyspace"); + String table = matcher.group("table"); + + return new QualifiedTableName(StringUtils.isNotBlank(keyspace) ? keyspace : providerDefault, table); + } + +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/constants/BatchStatementType.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/constants/BatchStatementType.java new file mode 100644 index 000000000000..1c5205c0f465 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/constants/BatchStatementType.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.cql.constants; + +import org.apache.nifi.components.DescribedValue; + +public enum BatchStatementType implements DescribedValue { + LOGGED_TYPE("LOGGED", "LOGGED", + "Use a LOGGED batch statement"), + UNLOGGED_TYPE("UNLOGGED", "UNLOGGED", + "Use an UNLOGGED batch statement"), + COUNTER_TYPE("COUNTER", "COUNTER", + "Use a COUNTER batch statement"), + BATCH_STATEMENT_TYPE_USE_ATTR_TYPE("USE_ATTR", "Use cql.batch.statement.type Attribute", + "The value of the cql.batch.statement.type Attribute will be used to determine which type of batch statement (LOGGED, UNLOGGED or COUNTER) " + + "will be used to generate and execute the Update statement."); + + private String value; + private String displayName; + private String description; + + BatchStatementType(String value, String displayName, String description) { + this.value = value; + this.displayName = displayName; + this.description = description; + } + + @Override + public String getValue() { + return value; + } + + @Override + public String getDisplayName() { + return displayName; + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/constants/StatementType.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/constants/StatementType.java new file mode 100644 index 000000000000..889114fb0c68 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/constants/StatementType.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.cql.constants; + +import org.apache.nifi.components.DescribedValue; + +public enum StatementType implements DescribedValue { + UPDATE_TYPE("UPDATE", "UPDATE", + "Use an UPDATE statement."), + INSERT_TYPE("INSERT", "INSERT", + "Use an INSERT statement."), + STATEMENT_TYPE_USE_ATTR_TYPE("USE_ATTR", "Use cql.statement.type Attribute", + "The value of the cql.statement.type Attribute will be used to determine which type of statement (UPDATE, INSERT) " + + "will be generated and executed"); + + private String value; + private String displayName; + private String description; + + StatementType(String value, String displayName, String description) { + this.value = value; + this.displayName = displayName; + this.description = description; + } + + @Override + public String getValue() { + return value; + } + + @Override + public String getDisplayName() { + return displayName; + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/constants/UpdateType.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/constants/UpdateType.java new file mode 100644 index 000000000000..c09a93fa285b --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/java/org/apache/nifi/processors/cql/constants/UpdateType.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.cql.constants; + +import org.apache.nifi.components.DescribedValue; + +public enum UpdateType implements DescribedValue { + INCR_TYPE("INCREMENT", "Increment", + "Use an increment operation (+=) for the Update statement."), + SET_TYPE("SET", "Set", + "Use a set operation (=) for the Update statement."), + DECR_TYPE("DECREMENT", "Decrement", + "Use a decrement operation (-=) for the Update statement."), + UPDATE_METHOD_USE_ATTR_TYPE("USE_ATTR", "Use cql.update.method Attribute", + "The value of the cql.update.method Attribute will be used to determine which operation (Set, Increment, Decrement) " + + "will be used to generate and execute the Update statement."); + private String value; + private String displayName; + private String description; + + UpdateType(String value, String displayName, String description) { + this.value = value; + this.displayName = displayName; + this.description = description; + } + + @Override + public String getValue() { + return value; + } + + @Override + public String getDisplayName() { + return displayName; + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor new file mode 100644 index 000000000000..de9057cf2797 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/resources/META-INF/services/org.apache.nifi.processor.Processor @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +org.apache.nifi.processors.cql.ExecuteCQLQueryRecord +org.apache.nifi.processors.cql.PutCQLRecord \ No newline at end of file diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/resources/docs/org.apache.nifi.processors.cql.ExecuteCQLQueryRecord/additionalDetails.md b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/resources/docs/org.apache.nifi.processors.cql.ExecuteCQLQueryRecord/additionalDetails.md new file mode 100644 index 000000000000..36a42120ca85 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/resources/docs/org.apache.nifi.processors.cql.ExecuteCQLQueryRecord/additionalDetails.md @@ -0,0 +1,181 @@ + + +# ExecuteCQLQueryRecord + +## Description + +_ExecuteCQLQueryRecord_ runs a CQL `SELECT` query against Apache Cassandra or ScyllaDB, through the +connection provided by the configured _Cassandra Connection Provider_ controller service (either +`CassandraCQLExecutionService` or `ScyllaDBCQLExecutionService`), and writes the result set using the +configured _Result Set Output Writer_ (any `RecordSetWriterFactory`, so JSON, Avro, CSV, etc. are all valid +targets). Results stream row-by-row rather than buffering the whole result set in memory, so arbitrarily +large result sets are supported. + +The processor can run two ways: + +* **Triggered by an incoming FlowFile** - the query, and any property using + [Expression Language](https://nifi.apache.org/docs/nifi-docs/html/expression-language-guide.html), is + evaluated against that FlowFile's attributes. The incoming FlowFile is routed to `original` on success, + `retry` on a query failure, or `failure` on any other processing error. +* **Triggered on a schedule (timer or cron)**, with no incoming connection - useful for a polling query that + isn't driven by upstream flow content. In this mode there is no FlowFile to evaluate Expression Language + against, so property values must already be resolvable without one, and a processing error yields the + processor rather than routing anything (there's no FlowFile to route). + +The `executecql.row.count` attribute on the routed FlowFile(s) is not written directly by this processor; +row-count reporting comes from the fragmentation attributes described below, which is the accurate signal +when a single query's results are split across more than one FlowFile. + +## Query parameters + +A query can carry `?` bind markers, each supplied by a dynamic property named `cql.arg.`, where +`` is the marker's 1-based position in the query text: + +| Property | Value | +|---|---| +| _CQL select query_ | `SELECT * FROM sensors.readings WHERE sensor_id = ? AND taken_at > ?` | +| `cql.arg.1` | `${sensor.id}` | +| `cql.arg.2` | `${reading.since}` | + +Positions must run consecutively from 1 - a gap such as `cql.arg.1` plus `cql.arg.3` is a configuration error +and fails validation, since the parameters are matched to markers by position. The number of parameters must +also match the number of markers in the query; a mismatch fails the query with a message giving both counts. + +Each value is evaluated as Expression Language against the incoming FlowFile, then **bound** - sent to the +cluster as a data value, separately from the statement text. Parameter values are therefore never parsed as +CQL: a value of `1' OR '1'='1` is looked up as that literal string, not treated as query syntax. + +Because Expression Language always produces text, each parameter is converted to whatever Java type its bind +marker's declared CQL type requires, using the same conversion applied to record values on the write path. +Text works directly for `text`/`ascii` columns, for the boolean, numeric, `date` and `time` types, for `uuid`, +for `timeuuid` (which must still be a genuine version-1 UUID), and for `timestamp`, which accepts either an +ISO-8601 instant such as `2026-08-07T14:30:00Z` or a count of milliseconds since the epoch. A type outside +that set - `blob`, `inet`, `decimal`, `varint` or a collection - is passed to the driver as-is and will fail +the query unless the value is already of a type the driver's codec for that column accepts. + +### Interpolating a value into the query text instead + +The _CQL select query_ property also supports Expression Language, so a FlowFile attribute can be interpolated +directly into the query text. That is a deliberate capability - it is the only way to vary the parts of a +statement that cannot be parameterized, such as a table name or a column list - but it is **not** a substitute +for parameters, and it carries a real risk: + +> Anything interpolated into the query text is parsed as CQL. An attribute whose value is influenced by +> upstream data - the contents of a file, a value from an external system, an HTTP request - can therefore +> alter the structure of the statement, not just a value within it. This is CQL injection, and it is the same +> exposure that string-concatenated SQL has. + +Use `?` markers with `cql.arg.` properties for every value that comes from outside the flow's own +configuration. Reserve interpolation into the query text for values you control - and where an identifier +genuinely must be dynamic, constrain the attribute upstream to a known set rather than passing it through +unchecked. + +## Data types + +Result set values are converted to an Avro schema (and from there, whatever the configured Record Writer +produces) using the same type mapping `PutCQLRecord` uses for writes, in reverse. The value placed into each +record field is whatever the DataStax driver's own codec decoded the column into, unchanged. + +* `BOOLEAN` → boolean; `TINYINT`/`SMALLINT`/`INT` → int (Avro has no byte/short primitive, so these + widen); `BIGINT`/`COUNTER` → long; `FLOAT` → float; `DOUBLE` → double; `BLOB` → bytes; + `ASCII`/`TEXT` → string. +* `UUID` and `TIMEUUID` become the record API's native `UUID` field type, holding the `java.util.UUID` the + driver decoded. A `timeuuid` column's value is always a genuine version-1 (time-based) UUID by the time + it's read back - Cassandra/ScyllaDB do not accept a non-version-1 value being written to a `timeuuid` + column in the first place (see `PutCQLRecord`'s documentation for what happens on that write-side + rejection) - so there is nothing extra to validate on the read side. +* `TIMESTAMP`, `DATE`, `TIME`, `DECIMAL`, `VARINT` and `INET` are declared as a string field, while the value + itself stays the driver's decoded object - a `java.time.Instant`, `LocalDate`, `LocalTime`, + `java.math.BigDecimal`, `java.math.BigInteger` or `java.net.InetAddress` respectively. See + [String-declared fields and their values](#string-declared-fields-and-their-values) below, which explains + why and what it means for a downstream consumer. +* `LIST`/`SET` become an Avro array, `MAP` becomes an Avro map, and a Cassandra User Defined Type (UDT) + becomes a nested named Avro record, all recursively, using the same per-element type mapping described + above - a UDT nested inside another UDT, or inside a `LIST`/`SET`/`MAP`, is resolved the same way with no + special-casing. Note that Avro maps require string keys: a CQL `map` whose key type isn't already + string-like is represented with its real (non-string) key objects in the underlying data even though the + declared schema implies string keys. +* A CQL type with no mapping defined above - `DURATION`, `TUPLE`, `VECTOR`, or any future/custom type - is + represented as a string field holding the driver's own decoded object. This keeps one unusual column from + failing the entire query; it does not attempt to give the column's structure (a tuple's individual + elements, a vector's components) back as anything richer. + +### String-declared fields and their values + +For the types listed above as string-declared, the field's declared type (`string`) and the value's runtime +class (`Instant`, `BigDecimal`, ...) are deliberately not the same class. This is a supported combination +rather than a defect: NiFi's `STRING` is a **widening** record type over `DATE`, `TIME`, `TIMESTAMP`, +`DECIMAL`, `BIGINT` and others, so `DataTypeUtils` accepts each of these values for a string field and +coerces it **losslessly** — via `toString()` — if and when a Record Writer asks for a string. A nanosecond +`LocalTime` renders in full as `15:14:54.899676065`; a `BigDecimal` renders as its exact decimal text. + +Leaving the driver's object in place is what makes both readings available: a writer that wants text gets a +faithful rendering, and a script or custom writer that wants the real `BigDecimal`, `Instant` or +`InetAddress` can take it straight from `getValue(...)` without reparsing a string. + +Declaring NiFi's native types instead is not workable for these columns: + +* NiFi's native `TIMESTAMP`/`DATE`/`TIME` types are backed by `java.sql.Timestamp`/`Date`/`Time`, not + `java.time.*`. Pairing them with the driver's values makes conversion **fail outright** for `TIMESTAMP`, + and converting the values to `java.sql.*` to suit instead **truncates** — CQL `time` is + nanosecond-resolution and `java.sql.Time` is not, so `15:14:54.899676065` would become `15:14:54`. +* A native `DECIMAL` field needs a fixed precision and scale declared up front, which CQL's + arbitrary-precision `decimal` cannot supply; a native `BIGINT` is not reachable through the Avro schema + round trip at all. +* There is no native NiFi record type for an IP address. + +One consequence worth knowing: `InetAddress.toString()` renders as `/127.0.0.1`, with a leading slash, so an +`inet` column coerced to text carries that slash. Use the `InetAddress` object directly if that matters. + +## Pagination and query overrides + +* _Fetch Size_ overrides how many rows the driver requests per page from the server for this query only, + without changing the connection service's own configured default. +* _Max Wait Time_ overrides the connection service's configured Read Timeout for this query only. + +Both are optional; when unset, the connection service's own configured values apply. + +## Splitting results across FlowFiles + +* _Max Rows Per Flow File_ (default `0`, meaning "one FlowFile for the whole result set") caps how many rows + go into a single output FlowFile, splitting a large result set into several. When this produces more than + one FlowFile, all of them share a `fragment.identifier` attribute, each gets a `fragment.index` giving its + position among the set, and (unless _Output Batch Size_ is also set - see below) a `fragment.count` + attribute giving the total, so they can be correlated and reassembled downstream. +* _Output Batch Size_ (default `0`, meaning "commit once at the end") commits the session, releasing output + FlowFiles downstream, every time this many are ready, instead of waiting for the entire result set to + finish. This bounds memory/queue growth for very large result sets at the cost of releasing FlowFiles + before the final row count is known - which is exactly why `fragment.count` is intentionally left unset + whenever _Output Batch Size_ is configured: earlier FlowFiles may already have been committed downstream + before the true total could be determined. One further consequence: if the query fails after early commits + have already released the incoming FlowFile to `original`, that FlowFile cannot be recalled, so a new, + empty FlowFile is created to carry the retry signal to `retry` in its place - and the results committed + before the failure stay downstream, so a retried query delivers those rows a second time. + +If the incoming FlowFile's query returns zero rows, no output FlowFile is created at all, and the incoming +FlowFile (if any) is routed directly to `original`. + +## Relationships + +* `success` - one or more FlowFiles containing query results. +* `original` - the incoming FlowFile that triggered the query (only present when triggered by an incoming + FlowFile), routed here once every resulting output FlowFile has been transferred to `success` - or + immediately, if the query returned no rows at all. Auto-terminated by default, since most flows only need + the query results themselves, not the triggering FlowFile. +* `failure` - the query could not be executed (for example, invalid CQL, or a schema mismatch between the + result set and the configured Record Writer). +* `retry` - the query failed in a way that may succeed if attempted again (for example, a transient + connectivity or consistency-level failure), penalizing the FlowFile before routing it here. diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/resources/docs/org.apache.nifi.processors.cql.PutCQLRecord/additionalDetails.md b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/resources/docs/org.apache.nifi.processors.cql.PutCQLRecord/additionalDetails.md new file mode 100644 index 000000000000..3ba02b395a0a --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/main/resources/docs/org.apache.nifi.processors.cql.PutCQLRecord/additionalDetails.md @@ -0,0 +1,183 @@ + + +# PutCQLRecord + +## Description + +_PutCQLRecord_ is a record-aware processor that reads each FlowFile using the configured _Record Reader_ and +writes the resulting records to an Apache Cassandra or ScyllaDB table, using either a CQL `INSERT` or `UPDATE` +statement, via the connection provided by the configured _Cassandra Connection Provider_ controller service +(either `CassandraCQLExecutionService` or `ScyllaDBCQLExecutionService`). Records are grouped into batches (see +_Batch size_) and written as a single Cassandra `BatchStatement` per batch, one prepared statement per +FlowFile. + +## Data types + +CQL column types are matched against record field values using the driver's own codec registry wherever +possible, with the following notable behaviors: + +* `BOOLEAN`, the integer types (`TINYINT`/`SMALLINT`/`INT`/`BIGINT`/`COUNTER`), `FLOAT`, and `DOUBLE` accept a + compatible-but-non-exact value (for example a numeric `String`) via custom codecs registered by the + connection service, not just an exact Java type match. +* `VARINT` is the natural fit for a record's `BIGINT` field (arbitrary-precision integer). +* `CHAR` has no dedicated CQL equivalent and is written to a `TEXT`/`VARCHAR` column. +* `UUID` and `TIMEUUID` both bind from a record's `UUID` field type - see + [Writing to a timeuuid column](#writing-to-a-timeuuid-column) below for the extra requirement `TIMEUUID` + imposes. +* `DATE`/`TIME`/`TIMESTAMP` accept `java.sql.Date`/`java.sql.Time`/`Instant`-family values. +* `LIST`/`SET` accept either a record `ARRAY` or an `Object[]`; `MAP` accepts a record `MAP`. Elements are + converted recursively using these same rules, so a `list` or `set` is subject to the same + requirements as a scalar column of that type. +* A Cassandra User Defined Type (UDT) column accepts either a nested `RECORD` or a raw `Map`, with fields + converted recursively the same way, including UDTs nested inside UDTs, or inside a `LIST`/`SET`/`MAP`. + +## Statement types + +The _Statement Type_ property (or the `cql.statement.type` FlowFile attribute, when _Statement Type_ is set +to use it) selects between two CQL statement shapes: + +* **INSERT** - every field in each record is written to the column of the same name. There is no separate + concept of "which fields are the primary key" for an INSERT: Cassandra requires all primary key columns to + be supplied as ordinary column values in the statement, the same as any other column. +* **UPDATE** - requires _Update Keys_, a comma-separated list of column names that identify the row to + update (these must be present as fields in every record, with matching names, unless overridden - see + [Primary Key Dynamic Properties](#primary-key-dynamic-properties)). Every other field in the record is + applied via the method selected by _Update Method_: + * `SET` - overwrites the column's value. + * `INCREMENT` / `DECREMENT` - adjusts a counter column, and requires _Batch Statement Type_ to be + `COUNTER` or `UNLOGGED`, since Cassandra/ScyllaDB reject counter mutations in any other batch type. + +`INSERT` cannot be combined with a `COUNTER` _Batch Statement Type_, since Cassandra/ScyllaDB do not allow +`INSERT` statements against counter tables - only `UPDATE`. + +## Batching + +_Batch size_ controls how many records are grouped into a single `BatchStatement` (default 100). _Batch +Statement Type_ selects `UNLOGGED` (the default - higher throughput, no atomicity guarantee), `LOGGED` +(atomic across partitions, at the cost of writing every batch to the distributed batchlog first, so reserve +it for the cases that genuinely need that atomicity), or `COUNTER` (required for counter mutations); it can +also be resolved per FlowFile from the `cql.batch.statement.type` attribute. If a batch fails partway +through a FlowFile with multiple batches, the FlowFile is routed to `retry` (on a query failure) or +`failure` (on any other error) with the +`cql.records.written` attribute set to how many records were already committed in prior batches within that +attempt, since earlier batches may have succeeded even though the FlowFile as a whole did not. + +### Run Duration and counters + +This processor supports batching in the framework sense as well, so _Run Duration_ can be raised above `0` to +let NiFi batch session commits and trade latency for throughput. That is safe for `INSERT` and `SET`-method +`UPDATE` because a CQL write against a full primary key is an upsert: if a batch is rolled back and the +FlowFiles are reprocessed, the same rows are written again to the same values. Setting _Timestamp Field_ makes +that guarantee exact rather than merely usually-true. + +**It is not safe for counters.** `INCREMENT` and `DECREMENT` are the one CQL write that is not idempotent - +reprocessing a FlowFile applies its increments a second time. Leave _Run Duration_ at `0` on any flow using a +counter _Update Method_, and prefer counter flows that can tolerate at-least-once delivery in any case, since +a rollback can also be triggered by a failure that has nothing to do with Run Duration. + +## TTL and write timestamp overrides + +* _Time To Live_ overrides the connection service's own default TTL for `INSERT` and `SET`-method `UPDATE` + statements. Ignored for `INCREMENT`/`DECREMENT`, since Cassandra/ScyllaDB do not support a TTL on counter + columns. +* _Timestamp Field_ names a field in each record whose value supplies the CQL write timestamp for that + record's statement, instead of the time the statement executes. This makes retries/reprocessing safe: + resubmitting the same record with the same timestamp is a true no-op rather than a write that could win a + last-write-wins race against different data written to the same row in the meantime. Also ignored for + `INCREMENT`/`DECREMENT`, since Cassandra/ScyllaDB do not support a custom write timestamp on counter + columns. + +## Primary Key Dynamic Properties + +By default, a column's value comes from the record field with the same name. A dynamic property overrides +that for one specific column of one specific table, sourcing its value from an arbitrary location in the +record instead of requiring a same-named top-level field. + +**Property name** must be exactly `.
.` - three unquoted CQL-identifier segments +(each starting with a letter, followed by letters/digits/underscores), separated by dots. All three segments +are required; there is no unqualified-table form for dynamic properties (unlike the _Table name_ property +itself, which does accept an unqualified table name and resolves it against the connection service's default +keyspace). + +**Property value** is a [RecordPath](https://nifi.apache.org/docs/nifi-docs/html/record-path-guide.html) +expression, evaluated once per record. It must resolve to **exactly one value** - a RecordPath that matches +zero fields, or more than one (for example, one written against an array without narrowing to a single +element), causes that record's write to fail rather than silently picking an arbitrary match. This matters +for a partition key specifically: a value that isn't deterministic per record can route logically identical +records to different partitions. + +An override only applies to writes against the keyspace-qualified table named in the property; the same +column name in a different keyspace or table is unaffected, and an unrelated dynamic property never +accidentally applies to the wrong table. + +### Example + +Given records shaped like: + +```json +{ + "customer": { + "id": "9f2c1e2a-1111-4a11-8a11-abcdefabcdef" + }, + "event_type": "login" +} +``` + +being written to `analytics.customer_events`, where the table's partition key column `customer_id` has no +matching top-level record field, add a dynamic property: + +* Name: `analytics.customer_events.customer_id` +* Value: `/customer/id` + +Every record written to `analytics.customer_events` now resolves `customer_id` from the nested +`customer/id` field instead of requiring a top-level `customer_id` field. + +## Writing to a timeuuid column + +A CQL `timeuuid` column requires a genuine **version 1 (time-based)** UUID - this is part of the CQL type's +own definition (`now()`, the idiomatic way to populate one in CQL, generates a version-1 value), not +something specific to this processor. Supplying a version-4 (random) UUID - which is what `UUID.randomUUID()` +and most record-generation logic produce by default - is rejected, not silently accepted: + +* If the value's version is not 1, `PutCQLRecord` fails that record with a message naming the actual value + and its version, for example: + `Value '87ccc9c5-c4fc-4073-98bc-ee6e7e538237' is not a valid timeuuid: version 4, but timeuuid columns + require a version 1 (time-based) UUID.` +* This check happens before the value is ever sent to the driver. Without it, the same bad value would still + be rejected, but with a much less helpful driver-level error + (`CodecNotFoundException: Codec not found for requested operation: [TIMEUUID <-> java.util.UUID]`) that + reads like a configuration problem rather than a data problem. + +**To supply a valid value**, the record field targeting a `timeuuid` column must already carry a real +version-1 UUID by the time it reaches this processor - for example, one generated by an upstream system, or +by the DataStax Java Driver's `Uuids.timeBased()` (always uses the current time, with a randomized node +identifier and an internal clock sequence to keep concurrently-generated values from colliding). A record's +own `TIMESTAMP` field is **not** sufficient on its own: it supplies only the time component of a version-1 +UUID's 128 bits, not the clock-sequence/node bits that make the value collision-resistant, so constructing a +"valid-looking" UUID from just a timestamp (for example via the driver's `Uuids.startOf(millis)`/ +`Uuids.endOf(millis)`, which are intended for building CQL range-query bounds, not per-record identifiers) +can produce identical values for two different records that happen to share a timestamp. + +## Relationships + +* `success` - every record in the FlowFile was written. +* `failure` - a configuration or data problem (for example, a mistyped _Timestamp Field_, a missing _Update + Keys_ for an `UPDATE`, or a `timeuuid` value that isn't version 1) prevented the write; `cql.records.written` + reflects any batches that were already committed before the failure, if the FlowFile spanned more than one + batch. +* `retry` - the query itself failed against Cassandra/ScyllaDB (for example, a transient connectivity or + consistency-level failure). Since `INSERT`/`UPDATE` are idempotent per-row upserts, retrying is normally + safe; combine with _Timestamp Field_ if last-write-wins ordering against concurrent writers matters. diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/test/java/org/apache/nifi/processors/cql/ExecuteCQLQueryRecordTest.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/test/java/org/apache/nifi/processors/cql/ExecuteCQLQueryRecordTest.java new file mode 100644 index 000000000000..77059d580851 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/test/java/org/apache/nifi/processors/cql/ExecuteCQLQueryRecordTest.java @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.cql; + +import org.apache.nifi.flowfile.attributes.FragmentAttributes; +import org.apache.nifi.processors.cql.mock.FailingRecordSetWriterFactory; +import org.apache.nifi.processors.cql.mock.MockCQLQueryExecutionService; +import org.apache.nifi.serialization.SimpleRecordSchema; +import org.apache.nifi.serialization.record.MapRecord; +import org.apache.nifi.serialization.record.MockRecordWriter; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordField; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.service.cql.api.service.QueryOverrides; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ExecuteCQLQueryRecordTest { + private TestRunner testRunner; + + @BeforeEach + public void setup() { + testRunner = TestRunners.newTestRunner(ExecuteCQLQueryRecord.class); + testRunner.setProperty(ExecuteCQLQueryRecord.CQL_SELECT_QUERY, "SELECT * FROM cql_table"); + } + + @DisplayName("Verify the normal write behavior with different record counts and expected batch sizes") + @ParameterizedTest(name = "rows per file={0},generated records={1},expected ff count={2},output batch size={3}") + @CsvSource({ + "5,4,1,0", + "5,100,20,0", + "5,101,21,0", + "5,100,20,1", + "50,3000,60,5", + "0,100,1,0" + }) + public void testSimpleWriteScenario(int rowsPerFlowFile, int recordCount, + int expectedFlowFileCount, + int outputBatchSize) throws Exception { + testRunner.setProperty(ExecuteCQLQueryRecord.MAX_ROWS_PER_FLOW_FILE, String.valueOf(rowsPerFlowFile)); + testRunner.setProperty(ExecuteCQLQueryRecord.OUTPUT_BATCH_SIZE, String.valueOf(outputBatchSize)); + + MockRecordWriter parser = new MockRecordWriter(); + RecordField field1 = new RecordField("a", RecordFieldType.STRING.getDataType()); + RecordField field2 = new RecordField("b", RecordFieldType.STRING.getDataType()); + SimpleRecordSchema schema = new SimpleRecordSchema(List.of(field1, field2)); + + List data = new ArrayList<>(); + + for (int i = 0; i < recordCount; i++) { + Map rec = Map.of("a", UUID.randomUUID().toString(), "b", UUID.randomUUID().toString()); + MapRecord testData = new MapRecord(schema, rec); + data.add(testData); + } + + MockCQLQueryExecutionService service = new MockCQLQueryExecutionService(data.iterator()); + + testRunner.setProperty(ExecuteCQLQueryRecord.CONNECTION_PROVIDER_SERVICE, "connection"); + testRunner.addControllerService("connection", service); + testRunner.enableControllerService(service); + testRunner.setProperty(ExecuteCQLQueryRecord.OUTPUT_WRITER, "writer"); + testRunner.addControllerService("writer", parser); + testRunner.enableControllerService(parser); + testRunner.assertValid(); + + testRunner.enqueue("parent_file"); + testRunner.run(); + + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_ORIGINAL, 1); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_SUCCESS, expectedFlowFileCount); + + List flowFiles = testRunner.getFlowFilesForRelationship(ExecuteCQLQueryRecord.REL_SUCCESS); + assertEquals(expectedFlowFileCount, flowFiles.size()); + flowFiles.sort(Comparator.comparingInt(ff -> Integer.parseInt(ff.getAttribute(FragmentAttributes.FRAGMENT_INDEX.key())))); + + final boolean batching = outputBatchSize > 0; + final Set fragmentIds = new HashSet<>(); + int recordsRemaining = recordCount; + + for (MockFlowFile ff : flowFiles) { + String fragmentId = ff.getAttribute(FragmentAttributes.FRAGMENT_ID.key()); + assertNotNull(fragmentId); + assertDoesNotThrow(() -> UUID.fromString(fragmentId)); + fragmentIds.add(fragmentId); + + assertNotNull(ff.getAttribute(FragmentAttributes.FRAGMENT_INDEX.key())); + int fragmentIndex = Integer.parseInt(ff.getAttribute(FragmentAttributes.FRAGMENT_INDEX.key())); + assertTrue(fragmentIndex < expectedFlowFileCount); + + // fragment.count is only knowable once the whole result set has been processed, which isn't the + // case once Output Batch Size starts releasing FlowFiles early - see OUTPUT_BATCH_SIZE's description. + if (batching) { + assertNull(ff.getAttribute(FragmentAttributes.FRAGMENT_COUNT.key())); + } else { + assertEquals(String.valueOf(expectedFlowFileCount), ff.getAttribute(FragmentAttributes.FRAGMENT_COUNT.key())); + } + + int rowsInFile = ff.getContent().isEmpty() ? 0 : ff.getContent().split("\n").length; + int expectedRowsInFile = rowsPerFlowFile == 0 ? recordCount : Math.min(rowsPerFlowFile, recordsRemaining); + assertEquals(expectedRowsInFile, rowsInFile); + recordsRemaining -= rowsInFile; + } + + assertEquals(1, fragmentIds.size()); + assertEquals(0, recordsRemaining); + } + + private MockCQLQueryExecutionService configureServices() throws Exception { + MockRecordWriter parser = new MockRecordWriter(); + RecordField field1 = new RecordField("a", RecordFieldType.STRING.getDataType()); + SimpleRecordSchema schema = new SimpleRecordSchema(List.of(field1)); + + List data = new ArrayList<>(); + data.add(new MapRecord(schema, Map.of("a", "value"))); + + MockCQLQueryExecutionService service = new MockCQLQueryExecutionService(data.iterator()); + + testRunner.setProperty(ExecuteCQLQueryRecord.CONNECTION_PROVIDER_SERVICE, "connection"); + testRunner.addControllerService("connection", service); + testRunner.enableControllerService(service); + testRunner.setProperty(ExecuteCQLQueryRecord.OUTPUT_WRITER, "writer"); + testRunner.addControllerService("writer", parser); + testRunner.enableControllerService(parser); + + return service; + } + + private MockCQLQueryExecutionService runWithSingleRecord() throws Exception { + return runWithSingleRecord(Map.of()); + } + + private MockCQLQueryExecutionService runWithSingleRecord(final Map attributes) throws Exception { + testRunner.setProperty(ExecuteCQLQueryRecord.MAX_ROWS_PER_FLOW_FILE, "10"); + + MockCQLQueryExecutionService service = configureServices(); + testRunner.assertValid(); + + testRunner.enqueue("parent_file", attributes); + testRunner.run(); + + return service; + } + + /** + * Runs a query that delivers {@code rowsBeforeFailure} rows and then fails partway through the result set, + * against a writer that may itself be set to fail. Returns the runner for the caller to assert routing on. + */ + private void runWithMidStreamFailure(final int rowsBeforeFailure, final int maxRowsPerFlowFile, + final int outputBatchSize, final MockRecordWriter writer) throws Exception { + testRunner.setProperty(ExecuteCQLQueryRecord.MAX_ROWS_PER_FLOW_FILE, String.valueOf(maxRowsPerFlowFile)); + testRunner.setProperty(ExecuteCQLQueryRecord.OUTPUT_BATCH_SIZE, String.valueOf(outputBatchSize)); + + final RecordField field = new RecordField("a", RecordFieldType.STRING.getDataType()); + final SimpleRecordSchema schema = new SimpleRecordSchema(List.of(field)); + + final List data = new ArrayList<>(); + for (int i = 0; i < rowsBeforeFailure; i++) { + data.add(new MapRecord(schema, Map.of("a", "value-" + i))); + } + + final MockCQLQueryExecutionService service = + new MockCQLQueryExecutionService(data.iterator()).failAfter(rowsBeforeFailure); + + testRunner.setProperty(ExecuteCQLQueryRecord.CONNECTION_PROVIDER_SERVICE, "connection"); + testRunner.addControllerService("connection", service); + testRunner.enableControllerService(service); + testRunner.setProperty(ExecuteCQLQueryRecord.OUTPUT_WRITER, "writer"); + testRunner.addControllerService("writer", writer); + testRunner.enableControllerService(writer); + testRunner.assertValid(); + + testRunner.enqueue("parent_file"); + testRunner.run(); + } + + @DisplayName("Verify that a failure partway through a result set leaves no FlowFile stranded, so the session still commits and the retry takes effect") + @ParameterizedTest(name = "rows before failure={0}, rows per flow file={1}") + @CsvSource({ + "5,2", + "3,0", + "0,2" + }) + public void testMidStreamFailureRoutesParentToRetry(final int rowsBeforeFailure, final int maxRowsPerFlowFile) throws Exception { + runWithMidStreamFailure(rowsBeforeFailure, maxRowsPerFlowFile, 0, new MockRecordWriter()); + + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_RETRY, 1); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_SUCCESS, 0); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_ORIGINAL, 0); + assertTrue(testRunner.getFlowFilesForRelationship(ExecuteCQLQueryRecord.REL_RETRY).getFirst().isPenalized()); + } + + @Test + @DisplayName("A query failure with no incoming connection yields the processor instead of routing an empty FlowFile to retry") + public void testSourceModeQueryFailureYieldsInsteadOfRouting() throws Exception { + final MockCQLQueryExecutionService service = + new MockCQLQueryExecutionService(Collections.emptyList().iterator()).failAfter(0); + + testRunner.setProperty(ExecuteCQLQueryRecord.CONNECTION_PROVIDER_SERVICE, "connection"); + testRunner.addControllerService("connection", service); + testRunner.enableControllerService(service); + testRunner.setProperty(ExecuteCQLQueryRecord.OUTPUT_WRITER, "writer"); + final MockRecordWriter writer = new MockRecordWriter(); + testRunner.addControllerService("writer", writer); + testRunner.enableControllerService(writer); + testRunner.assertValid(); + + testRunner.setIncomingConnection(false); + testRunner.run(); + + // Nothing routed anywhere - a created-from-nothing FlowFile has no attributes to re-drive the query, + // so the retry is the next scheduled run, signalled by yielding. + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_RETRY, 0); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_FAILURE, 0); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_SUCCESS, 0); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_ORIGINAL, 0); + assertTrue(((org.apache.nifi.util.MockProcessContext) testRunner.getProcessContext()).isYieldCalled()); + } + + @Test + @DisplayName("Verify that a failure after Output Batch Size has already committed FlowFiles retries on a new FlowFile rather than one the session released") + public void testMidStreamFailureAfterOutputBatchCommitStillRetries() throws Exception { + runWithMidStreamFailure(9, 2, 2, new MockRecordWriter()); + + // The four FlowFiles committed before the failure stay downstream - they cannot be recalled - and the + // parent went to 'original' with them, so the retry has to be carried by a FlowFile created afterwards. + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_SUCCESS, 4); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_ORIGINAL, 1); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_RETRY, 1); + } + + @Test + @DisplayName("Verify that a record writer failing mid-result-set routes to failure with no FlowFile left stranded") + public void testWriterFailureMidResultSetRoutesToFailure() throws Exception { + // failAfterN counts per writer instance and the callback builds a fresh writer per FlowFile, so this + // only reaches the failure with a single writer covering the whole result set - hence 0 (unlimited) + // rows per FlowFile. The writer throws at row 5, before the query's own failure at row 9 would have + // fired, so this is the writer-failure path rather than the query-failure one. + runWithMidStreamFailure(9, 0, 0, new MockRecordWriter(null, false, 5)); + + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_FAILURE, 1); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_SUCCESS, 0); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_ORIGINAL, 0); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_RETRY, 0); + } + + @Test + @DisplayName("Verify that cql.arg.N properties are forwarded as positional parameters in order, leaving the query text untouched") + public void testPositionalParametersForwardedInOrder() throws Exception { + testRunner.setProperty(ExecuteCQLQueryRecord.CQL_SELECT_QUERY, "SELECT * FROM cql_table WHERE id = ? AND region = ?"); + testRunner.setProperty("cql.arg.1", "${row.id}"); + testRunner.setProperty("cql.arg.2", "${row.region}"); + + MockCQLQueryExecutionService service = runWithSingleRecord(Map.of("row.id", "42", "row.region", "us-east")); + + assertEquals(List.of("42", "us-east"), service.getLastParameters()); + assertEquals("SELECT * FROM cql_table WHERE id = ? AND region = ?", service.getLastCql()); + } + + @Test + @DisplayName("Verify that CQL inside a parameter value stays data - it is bound, never concatenated into the query text") + public void testParameterValueIsNotInterpolatedIntoQueryText() throws Exception { + final String injected = "1' OR '1'='1"; + + testRunner.setProperty(ExecuteCQLQueryRecord.CQL_SELECT_QUERY, "SELECT * FROM cql_table WHERE id = ?"); + testRunner.setProperty("cql.arg.1", "${row.id}"); + + MockCQLQueryExecutionService service = runWithSingleRecord(Map.of("row.id", injected)); + + assertEquals(List.of(injected), service.getLastParameters()); + assertEquals("SELECT * FROM cql_table WHERE id = ?", service.getLastCql()); + } + + @Test + @DisplayName("Verify that a query with no parameters forwards an empty parameter list") + public void testNoParametersForwardsEmptyList() throws Exception { + MockCQLQueryExecutionService service = runWithSingleRecord(); + + assertEquals(List.of(), service.getLastParameters()); + } + + @Test + @DisplayName("Verify that a gap in the positional parameter numbering fails validation") + public void testNonConsecutiveParameterNumberingIsInvalid() throws Exception { + configureServices(); + testRunner.setProperty("cql.arg.1", "first"); + testRunner.setProperty("cql.arg.3", "third"); + + testRunner.assertNotValid(); + } + + @Test + @DisplayName("Verify that positional parameter numbering must start at 1") + public void testParameterNumberingNotStartingAtOneIsInvalid() throws Exception { + configureServices(); + testRunner.setProperty("cql.arg.2", "second"); + + testRunner.assertNotValid(); + } + + @Test + @DisplayName("Verify that a dynamic property which is not a positional parameter is rejected outright") + public void testUnrecognizedDynamicPropertyNameIsRejected() { + assertThrows(IllegalArgumentException.class, () -> testRunner.setProperty("cql.parameter.1", "value")); + } + + @Test + @DisplayName("Verify that fetch size and query timeout overrides are unset by default, deferring to the connection service") + public void testQueryOverridesDefaultToUnset() throws Exception { + MockCQLQueryExecutionService service = runWithSingleRecord(); + + QueryOverrides overrides = service.getLastQueryOverrides(); + assertNotNull(overrides); + assertNull(overrides.fetchSize()); + assertNull(overrides.timeout()); + } + + @Test + @DisplayName("Verify that fetch size and query timeout overrides are forwarded to the connection service when set") + public void testQueryOverridesForwardedWhenSet() throws Exception { + testRunner.setProperty(ExecuteCQLQueryRecord.FETCH_SIZE, "250"); + testRunner.setProperty(ExecuteCQLQueryRecord.QUERY_TIMEOUT, "5 sec"); + + MockCQLQueryExecutionService service = runWithSingleRecord(); + + QueryOverrides overrides = service.getLastQueryOverrides(); + assertNotNull(overrides); + assertEquals(250, overrides.fetchSize()); + assertEquals(Duration.ofSeconds(5), overrides.timeout()); + } + + @Test + @DisplayName("Verify that an incoming FlowFile is routed to 'original' when the query returns no rows") + public void testEmptyResultSetRoutesParentToOriginal() throws Exception { + testRunner.setProperty(ExecuteCQLQueryRecord.MAX_ROWS_PER_FLOW_FILE, "5"); + + MockRecordWriter parser = new MockRecordWriter(); + MockCQLQueryExecutionService service = new MockCQLQueryExecutionService(Collections.emptyList().iterator()); + + testRunner.setProperty(ExecuteCQLQueryRecord.CONNECTION_PROVIDER_SERVICE, "connection"); + testRunner.addControllerService("connection", service); + testRunner.enableControllerService(service); + testRunner.setProperty(ExecuteCQLQueryRecord.OUTPUT_WRITER, "writer"); + testRunner.addControllerService("writer", parser); + testRunner.enableControllerService(parser); + testRunner.assertValid(); + + testRunner.enqueue("parent_file"); + testRunner.run(); + + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_ORIGINAL, 1); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_SUCCESS, 0); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_FAILURE, 0); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_RETRY, 0); + } + + @Test + @DisplayName("Verify that a record writer creation failure routes to failure without leaking the newly-created FlowFile") + public void testWriterCreationFailureRoutesToFailureWithoutLeakingFlowFiles() throws Exception { + testRunner.setProperty(ExecuteCQLQueryRecord.MAX_ROWS_PER_FLOW_FILE, "5"); + + RecordField field1 = new RecordField("a", RecordFieldType.STRING.getDataType()); + SimpleRecordSchema schema = new SimpleRecordSchema(List.of(field1)); + + List data = new ArrayList<>(); + data.add(new MapRecord(schema, Map.of("a", "value"))); + + MockCQLQueryExecutionService service = new MockCQLQueryExecutionService(data.iterator()); + FailingRecordSetWriterFactory writer = new FailingRecordSetWriterFactory(); + + testRunner.setProperty(ExecuteCQLQueryRecord.CONNECTION_PROVIDER_SERVICE, "connection"); + testRunner.addControllerService("connection", service); + testRunner.enableControllerService(service); + testRunner.setProperty(ExecuteCQLQueryRecord.OUTPUT_WRITER, "writer"); + testRunner.addControllerService("writer", writer); + testRunner.enableControllerService(writer); + testRunner.assertValid(); + + testRunner.enqueue("parent_file"); + testRunner.run(); + + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_FAILURE, 1); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_SUCCESS, 0); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_ORIGINAL, 0); + testRunner.assertTransferCount(ExecuteCQLQueryRecord.REL_RETRY, 0); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/test/java/org/apache/nifi/processors/cql/PutCQLRecordTest.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/test/java/org/apache/nifi/processors/cql/PutCQLRecordTest.java new file mode 100644 index 000000000000..4dd13dfd5921 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/test/java/org/apache/nifi/processors/cql/PutCQLRecordTest.java @@ -0,0 +1,549 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.cql; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.provenance.ProvenanceEventRecord; +import org.apache.nifi.record.path.RecordPath; +import org.apache.nifi.serialization.record.MockRecordParser; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.service.cql.api.constants.CqlBatchType; +import org.apache.nifi.service.cql.api.constants.UpdateMethod; +import org.apache.nifi.service.cql.api.exception.QueryFailureException; +import org.apache.nifi.service.cql.api.metadata.PrimaryKeyIdentifier; +import org.apache.nifi.service.cql.api.metadata.QualifiedTableName; +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.service.cql.api.service.WriteOverrides; +import org.apache.nifi.util.MockFlowFile; +import org.apache.nifi.util.TestRunner; +import org.apache.nifi.util.TestRunners; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import org.mockito.stubbing.Stubber; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static org.apache.nifi.processors.cql.constants.StatementType.UPDATE_TYPE; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.params.provider.Arguments.arguments; +import static org.mockito.Mockito.any; +import static org.mockito.Mockito.anyList; +import static org.mockito.Mockito.anyMap; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class PutCQLRecordTest { + // The batch type the processor resolves when Batch Statement Type is left alone. Most tests below are + // about something else entirely - batching, TTL, primary key overrides - and only need to name the batch + // type because it is part of the service call they verify, so they refer to this rather than repeating a + // literal. testDefaultBatchStatementType is the one test that actually asserts what the default is. + private static final CqlBatchType DEFAULT_BATCH_TYPE = CqlBatchType.UNLOGGED; + + private TestRunner runner; + private CQLExecutionService service; + private MockRecordParser mockReader; + + @BeforeEach + void setUp() throws Exception { + runner = TestRunners.newTestRunner(PutCQLRecord.class); + service = Mockito.mock(CQLExecutionService.class); + mockReader = new MockRecordParser(); + mockReader.addSchemaField("message", RecordFieldType.STRING); + mockReader.addSchemaField("sender", RecordFieldType.STRING); + + when(service.getIdentifier()).thenReturn("executionService"); + when(service.getTransitUrl(any(QualifiedTableName.class))).thenReturn("cassandra://message"); + + runner.setProperty(PutCQLRecord.CONNECTION_PROVIDER_SERVICE, service.getIdentifier()); + runner.setProperty(PutCQLRecord.TABLE, "message"); + runner.setProperty(PutCQLRecord.RECORD_READER_FACTORY, "reader"); + + runner.addControllerService("reader", mockReader); + runner.enableControllerService(mockReader); + + runner.addControllerService(service.getIdentifier(), service); + runner.enableControllerService(service); + } + + @Test + @DisplayName("Inserting records batches them into the configured batch size and reports success") + void testInsert() { + final int recordCount = 1000; + final int batchCount = 10; + + runner.setProperty(PutCQLRecord.BATCH_SIZE, "100"); + + for (int i = 0; i < recordCount; i++) { + mockReader.addRecord("Hello, world", "test_user"); + } + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_SUCCESS, 1); + + verify(service, times(batchCount)) + .insert(eq(new QualifiedTableName(null, "message")), anyList(), anyMap(), eq(DEFAULT_BATCH_TYPE), eq(WriteOverrides.NONE)); + } + + @Test + @DisplayName("Batch Statement Type defaults to UNLOGGED, so bulk writes do not pay for the distributed batchlog unless asked to") + void testDefaultBatchStatementType() { + assertEquals(DEFAULT_BATCH_TYPE.name(), PutCQLRecord.BATCH_STATEMENT_TYPE.getDefaultValue()); + } + + @DisplayName("Updating records with each update method and batch statement type produces the expected batched calls") + @ParameterizedTest + @CsvSource({ "DECREMENT,COUNTER", "INCREMENT,COUNTER", "SET,LOGGED" }) + void testUpdate(UpdateMethod updateMethod, String batchStatementType) { + runner.setProperty(PutCQLRecord.UPDATE_KEYS, "sender"); + runner.setProperty(PutCQLRecord.STATEMENT_TYPE, UPDATE_TYPE.getValue()); + runner.setProperty(PutCQLRecord.UPDATE_METHOD, updateMethod.name()); + runner.setProperty(PutCQLRecord.BATCH_STATEMENT_TYPE, batchStatementType); + + final int recordCount = 1050; + final int batchCount = 11; + + runner.setProperty(PutCQLRecord.BATCH_SIZE, "100"); + + for (int i = 0; i < recordCount; i++) { + mockReader.addRecord("Hello, world", "test_user"); + } + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_SUCCESS, 1); + + verify(service, times(batchCount)) + .update(eq(new QualifiedTableName(null, "message")), anyList(), anyMap(), eq(List.of("sender")), eq(updateMethod), + eq(CqlBatchType.valueOf(batchStatementType)), eq(WriteOverrides.NONE)); + } + + @Test + @DisplayName("An update method resolved from a FlowFile attribute is matched case-insensitively") + void testUpdateMethodFromAttributeIsCaseInsensitive() { + runner.setProperty(PutCQLRecord.UPDATE_KEYS, "sender"); + runner.setProperty(PutCQLRecord.STATEMENT_TYPE, UPDATE_TYPE.getValue()); + runner.setProperty(PutCQLRecord.UPDATE_METHOD, "USE_ATTR"); + + mockReader.addRecord("Hello, world", "test_user"); + + runner.enqueue("", Map.of(PutCQLRecord.UPDATE_METHOD_ATTRIBUTE, "set")); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_SUCCESS, 1); + + verify(service, times(1)) + .update(eq(new QualifiedTableName(null, "message")), anyList(), anyMap(), eq(List.of("sender")), eq(UpdateMethod.SET), eq(DEFAULT_BATCH_TYPE), eq(WriteOverrides.NONE)); + } + + @Test + @DisplayName("Setting Time To Live on the processor overrides the connection service's default for inserts") + void testTtlOverrideAppliesToInsert() { + runner.setProperty(PutCQLRecord.BATCH_SIZE, "100"); + runner.setProperty(PutCQLRecord.TTL, "1 hour"); + + mockReader.addRecord("Hello, world", "test_user"); + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_SUCCESS, 1); + + verify(service, times(1)) + .insert(eq(new QualifiedTableName(null, "message")), anyList(), anyMap(), eq(DEFAULT_BATCH_TYPE), eq(new WriteOverrides(Duration.ofHours(1), null))); + } + + @Test + @DisplayName("Setting Time To Live on the processor overrides the connection service's default for SET updates") + void testTtlOverrideAppliesToUpdate() { + runner.setProperty(PutCQLRecord.UPDATE_KEYS, "sender"); + runner.setProperty(PutCQLRecord.STATEMENT_TYPE, UPDATE_TYPE.getValue()); + runner.setProperty(PutCQLRecord.BATCH_SIZE, "100"); + runner.setProperty(PutCQLRecord.TTL, "30 min"); + + mockReader.addRecord("Hello, world", "test_user"); + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_SUCCESS, 1); + + verify(service, times(1)) + .update(eq(new QualifiedTableName(null, "message")), anyList(), anyMap(), eq(List.of("sender")), eq(UpdateMethod.SET), + eq(DEFAULT_BATCH_TYPE), eq(new WriteOverrides(Duration.ofMinutes(30), null))); + } + + @Test + @DisplayName("Setting Timestamp Field on the processor threads the field name through as a write override") + void testTimestampFieldOverrideAppliesToInsert() { + runner.setProperty(PutCQLRecord.BATCH_SIZE, "100"); + runner.setProperty(PutCQLRecord.TIMESTAMP_FIELD, "sender"); + + mockReader.addRecord("Hello, world", "test_user"); + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_SUCCESS, 1); + + verify(service, times(1)) + .insert(eq(new QualifiedTableName(null, "message")), anyList(), anyMap(), eq(DEFAULT_BATCH_TYPE), eq(new WriteOverrides(null, "sender"))); + } + + @Test + @DisplayName("Setting Timestamp Field to a name absent from the record schema routes to failure without writing anything") + void testTimestampFieldNotInSchemaRoutesToFailure() { + runner.setProperty(PutCQLRecord.BATCH_SIZE, "100"); + runner.setProperty(PutCQLRecord.TIMESTAMP_FIELD, "does_not_exist"); + + mockReader.addRecord("Hello, world", "test_user"); + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_FAILURE, 1); + + verify(service, times(0)).insert(any(QualifiedTableName.class), anyList(), anyMap(), any(CqlBatchType.class), any(WriteOverrides.class)); + } + + @Test + @DisplayName("Configuring INSERT with a COUNTER batch statement type fails validation up front") + void testInsertWithCounterBatchTypeIsInvalid() { + // Statement Type defaults to INSERT; Cassandra/ScyllaDB do not allow INSERT against counter tables, + // so INSERT + COUNTER batch type can never succeed and should fail validation up front. + runner.setProperty(PutCQLRecord.BATCH_STATEMENT_TYPE, "COUNTER"); + + runner.assertNotValid(); + } + + @Test + @DisplayName("An INSERT with a COUNTER batch statement type resolved from a FlowFile attribute routes to failure at runtime") + void testInsertWithCounterBatchTypeAttributeRoutesToFailure() { + // When Batch Statement Type is resolved from a FlowFile attribute, the invalid INSERT + COUNTER + // combination can't be caught at validation time, so it must be rejected at runtime instead. + runner.setProperty(PutCQLRecord.BATCH_STATEMENT_TYPE, "USE_ATTR"); + + mockReader.addRecord("Hello, world", "test_user"); + + runner.enqueue("", Map.of(PutCQLRecord.BATCH_STATEMENT_TYPE_ATTRIBUTE, "COUNTER")); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_FAILURE, 1); + } + + /** + * The three ways an insert can fail partway through a FlowFile's batches, each of which must route to + * failure and report how many records had already been written. + * + *

{@code batchesBeforeFailure} of zero is the "fails immediately" case, where nothing was written; + * one means the first batch landed before the second threw. The exception type is the axis that matters: + * {@link ProcessException} takes the generic error path and {@link IllegalArgumentException} the + * configuration-error path (which is how a bad primary key override surfaces), and both have to record + * progress the same way. + */ + static Stream insertFailures() { + return Stream.of( + arguments("fails on the first batch", new ProcessException("Test"), 0, "0"), + arguments("fails after one batch", new ProcessException("Test"), 1, "100"), + arguments("fails after one batch with a configuration error", + new IllegalArgumentException("/missing evaluated to no values."), 1, "100")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("insertFailures") + @DisplayName("An insert failure routes the FlowFile to failure, reporting the records written before it") + void testInsertFailureRoutesToFailureRecordingProgress(final String description, final RuntimeException failure, + final int batchesBeforeFailure, final String expectedRecordsWritten) { + runner.setProperty(PutCQLRecord.BATCH_SIZE, "100"); + + for (int i = 0; i < 250; i++) { + mockReader.addRecord("Hello, world", "test_user"); + } + + Stubber stubber = null; + for (int batch = 0; batch < batchesBeforeFailure; batch++) { + stubber = stubber == null ? doNothing() : stubber.doNothing(); + } + (stubber == null ? doThrow(failure) : stubber.doThrow(failure)) + .when(service) + .insert(any(QualifiedTableName.class), anyList(), anyMap(), any(CqlBatchType.class), any(WriteOverrides.class)); + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_FAILURE, 1); + + final MockFlowFile failedFlowFile = runner.getFlowFilesForRelationship(PutCQLRecord.REL_FAILURE).get(0); + assertEquals(expectedRecordsWritten, failedFlowFile.getAttribute(PutCQLRecord.RECORDS_WRITTEN_ATTRIBUTE)); + + // The batches that did land each left a provenance event behind; the one that threw did not, and the + // batches after it were never attempted. + assertEquals(batchesBeforeFailure, runner.getProvenanceEvents().size()); + } + + @Test + @DisplayName("A Cassandra query failure while inserting routes the FlowFile to retry rather than failure") + void testInsertQueryFailureRoutesToRetry() { + runner.setProperty(PutCQLRecord.BATCH_SIZE, "100"); + + mockReader.addRecord("Hello, world", "test_user"); + + doThrow(new QueryFailureException()) + .when(service) + .insert(any(QualifiedTableName.class), anyList(), anyMap(), any(CqlBatchType.class), any(WriteOverrides.class)); + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_RETRY, 1); + } + + @Test + @DisplayName("A Cassandra query failure while updating routes the FlowFile to retry rather than failure") + void testUpdateQueryFailureRoutesToRetry() { + runner.setProperty(PutCQLRecord.UPDATE_KEYS, "sender"); + runner.setProperty(PutCQLRecord.STATEMENT_TYPE, UPDATE_TYPE.getValue()); + runner.setProperty(PutCQLRecord.BATCH_SIZE, "100"); + + mockReader.addRecord("Hello, world", "test_user"); + + doThrow(new QueryFailureException()) + .when(service) + .update(any(QualifiedTableName.class), anyList(), anyMap(), eq(List.of("sender")), eq(UpdateMethod.SET), any(CqlBatchType.class), any(WriteOverrides.class)); + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_RETRY, 1); + } + + @Test + @DisplayName("Each batch emits its own provenance event, and the final one reports the true total records written") + void testProvenanceReportsRecordsWrittenPerBatchAndTotal() { + final int recordCount = 250; + + runner.setProperty(PutCQLRecord.BATCH_SIZE, "100"); + + for (int i = 0; i < recordCount; i++) { + mockReader.addRecord("Hello, world", "test_user"); + } + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_SUCCESS, 1); + + List events = runner.getProvenanceEvents(); + assertEquals(3, events.size()); + assertTrue(events.get(0).getDetails().contains("100 total")); + assertTrue(events.get(1).getDetails().contains("200 total")); + assertTrue(events.get(2).getDetails().contains("250 total")); + } + + @Test + @DisplayName("An invalid table name routes to failure with cql.records.written explicitly zero") + void testInvalidTableNameReportsZeroRecordsWritten() { + runner.setProperty(PutCQLRecord.TABLE, "not.a.valid.table.name"); + + mockReader.addRecord("Hello, world", "test_user"); + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_FAILURE, 1); + + MockFlowFile failedFlowFile = runner.getFlowFilesForRelationship(PutCQLRecord.REL_FAILURE).get(0); + assertEquals("0", failedFlowFile.getAttribute(PutCQLRecord.RECORDS_WRITTEN_ATTRIBUTE)); + } + + @Test + @DisplayName("A dynamic property named keyspace.table.field is accepted and described in terms of that field/table/keyspace") + void testDynamicPropertyDescriptorAcceptsQualifiedFieldName() { + PutCQLRecord processor = new PutCQLRecord(); + + PropertyDescriptor descriptor = processor.getSupportedDynamicPropertyDescriptor("my_keyspace.my_table.id_field"); + + assertTrue(descriptor.isDynamic()); + assertEquals("my_keyspace.my_table.id_field", descriptor.getName()); + assertEquals("This property sets a record path to be evaluated for field id_field on table my_keyspace.my_table", + descriptor.getDescription()); + } + + @ParameterizedTest + @DisplayName("A dynamic property name that isn't keyspace.table.field is rejected") + @ValueSource(strings = { + "id_field", // missing keyspace and table + "my_table.id_field", // missing keyspace + "my_keyspace.my_table.extra.id_field", // too many segments + "", // empty + "my_keyspace.my_table.", // trailing dot, empty field + ".my_table.id_field", // leading dot, empty keyspace + "1keyspace.my_table.id_field", // segment starting with a digit + "my-keyspace.my_table.id_field", // hyphen is not a valid CQL identifier character + "my_keyspace.my_table.id field" // whitespace inside a segment + }) + void testDynamicPropertyDescriptorRejectsNonQualifiedName(String propertyName) { + PutCQLRecord processor = new PutCQLRecord(); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> processor.getSupportedDynamicPropertyDescriptor(propertyName)); + + // Asserted on every case rather than once in a separate test: a user who mistypes one of several + // dynamic properties needs the message to say which one, and that is only useful if it holds for + // every rejection rather than for the one input a standalone test happened to pick. + assertTrue(exception.getMessage().contains(propertyName), + "Expected the exception message to name the invalid property, but was: " + exception.getMessage()); + } + + @Test + @SuppressWarnings("unchecked") + @DisplayName("A dynamic property RecordPath override for the target table is passed through as a primary key override on insert") + void testDynamicPropertyOverrideAppliesToInsert() { + runner.setProperty(PutCQLRecord.TABLE, "my_keyspace.message"); + runner.setProperty("my_keyspace.message.id", "/sender"); + + mockReader.addRecord("Hello, world", "test_user"); + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_SUCCESS, 1); + + final ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); + verify(service).insert(eq(new QualifiedTableName("my_keyspace", "message")), anyList(), captor.capture(), + eq(DEFAULT_BATCH_TYPE), eq(WriteOverrides.NONE)); + + final Map overrides = captor.getValue(); + final PrimaryKeyIdentifier identifier = new PrimaryKeyIdentifier("my_keyspace", "message", "id"); + assertEquals(1, overrides.size()); + assertTrue(overrides.containsKey(identifier)); + assertEquals("/sender", overrides.get(identifier).getPath()); + } + + @Test + @SuppressWarnings("unchecked") + @DisplayName("Multiple dynamic property RecordPath overrides for the same table are all passed through together") + void testMultipleDynamicPropertyOverridesApplyToInsert() { + runner.setProperty(PutCQLRecord.TABLE, "my_keyspace.message"); + runner.setProperty("my_keyspace.message.id", "/sender"); + runner.setProperty("my_keyspace.message.ts", "/message"); + + mockReader.addRecord("Hello, world", "test_user"); + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_SUCCESS, 1); + + final ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); + verify(service).insert(eq(new QualifiedTableName("my_keyspace", "message")), anyList(), captor.capture(), + eq(DEFAULT_BATCH_TYPE), eq(WriteOverrides.NONE)); + + final Map overrides = captor.getValue(); + assertEquals(2, overrides.size()); + assertEquals("/sender", overrides.get(new PrimaryKeyIdentifier("my_keyspace", "message", "id")).getPath()); + assertEquals("/message", overrides.get(new PrimaryKeyIdentifier("my_keyspace", "message", "ts")).getPath()); + } + + @Test + @SuppressWarnings("unchecked") + @DisplayName("A dynamic property RecordPath override for a different table is excluded from the overrides passed for the current write") + void testDynamicPropertyOverrideForOtherTableIsExcluded() { + runner.setProperty(PutCQLRecord.TABLE, "my_keyspace.message"); + runner.setProperty("my_keyspace.other_table.id", "/sender"); + + mockReader.addRecord("Hello, world", "test_user"); + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_SUCCESS, 1); + + final ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); + verify(service).insert(eq(new QualifiedTableName("my_keyspace", "message")), anyList(), captor.capture(), + eq(DEFAULT_BATCH_TYPE), eq(WriteOverrides.NONE)); + + assertTrue(captor.getValue().isEmpty()); + } + + @Test + @SuppressWarnings("unchecked") + @DisplayName("An unqualified Table name never matches any configured dynamic property override, since overrides always require a keyspace segment") + void testDynamicPropertyOverrideIgnoredWhenTableIsUnqualified() { + // TABLE is left at its unqualified default of "message" (set in setUp()), so the QualifiedTableName + // built at runtime has a null keyspace - which can never equal a dynamic property's required, + // non-null keyspace segment, so the override is silently never applied. + runner.setProperty("my_keyspace.message.id", "/sender"); + + mockReader.addRecord("Hello, world", "test_user"); + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_SUCCESS, 1); + + final ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); + verify(service).insert(eq(new QualifiedTableName(null, "message")), anyList(), captor.capture(), + eq(DEFAULT_BATCH_TYPE), eq(WriteOverrides.NONE)); + + assertTrue(captor.getValue().isEmpty()); + } + + @Test + @SuppressWarnings("unchecked") + @DisplayName("A dynamic property RecordPath override for the target table is passed through as a primary key override on update") + void testDynamicPropertyOverrideAppliesToUpdate() { + runner.setProperty(PutCQLRecord.TABLE, "my_keyspace.message"); + runner.setProperty(PutCQLRecord.UPDATE_KEYS, "sender"); + runner.setProperty(PutCQLRecord.STATEMENT_TYPE, UPDATE_TYPE.getValue()); + runner.setProperty("my_keyspace.message.id", "/message"); + + mockReader.addRecord("Hello, world", "test_user"); + + runner.enqueue(""); + runner.run(); + + runner.assertAllFlowFilesTransferred(PutCQLRecord.REL_SUCCESS, 1); + + final ArgumentCaptor> captor = ArgumentCaptor.forClass(Map.class); + verify(service).update(eq(new QualifiedTableName("my_keyspace", "message")), anyList(), captor.capture(), + eq(List.of("sender")), eq(UpdateMethod.SET), eq(DEFAULT_BATCH_TYPE), eq(WriteOverrides.NONE)); + + final Map overrides = captor.getValue(); + final PrimaryKeyIdentifier identifier = new PrimaryKeyIdentifier("my_keyspace", "message", "id"); + assertEquals(1, overrides.size()); + assertEquals("/message", overrides.get(identifier).getPath()); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/test/java/org/apache/nifi/processors/cql/mock/FailingRecordSetWriterFactory.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/test/java/org/apache/nifi/processors/cql/mock/FailingRecordSetWriterFactory.java new file mode 100644 index 000000000000..245e9ee90107 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/test/java/org/apache/nifi/processors/cql/mock/FailingRecordSetWriterFactory.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.processors.cql.mock; + +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.schema.access.SchemaNotFoundException; +import org.apache.nifi.serialization.RecordSetWriter; +import org.apache.nifi.serialization.RecordSetWriterFactory; +import org.apache.nifi.serialization.SimpleRecordSchema; +import org.apache.nifi.serialization.record.RecordSchema; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Collections; +import java.util.Map; + +/** + * A {@link RecordSetWriterFactory} whose {@link #createWriter} always fails. There is no existing NiFi mock for + * this failure mode: {@code MockRecordWriter} can only fail mid-write (via its {@code failAfterN} constructor + * argument), not at writer-creation time. Used to exercise ExecuteCQLQueryCallback's cleanup path when the + * output writer itself cannot be created. + */ +public class FailingRecordSetWriterFactory extends AbstractControllerService implements RecordSetWriterFactory { + @Override + public RecordSchema getSchema(Map variables, RecordSchema readSchema) { + return new SimpleRecordSchema(Collections.emptyList()); + } + + @Override + public RecordSetWriter createWriter(ComponentLog logger, RecordSchema schema, OutputStream out, Map variables) + throws SchemaNotFoundException, IOException { + throw new IOException("Unit test intentionally failing to create a RecordSetWriter"); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/test/java/org/apache/nifi/processors/cql/mock/MockCQLQueryExecutionService.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/test/java/org/apache/nifi/processors/cql/mock/MockCQLQueryExecutionService.java new file mode 100644 index 000000000000..9033373944a6 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-processors/src/test/java/org/apache/nifi/processors/cql/mock/MockCQLQueryExecutionService.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.processors.cql.mock; + +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.record.path.RecordPath; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.service.cql.api.constants.CqlBatchType; +import org.apache.nifi.service.cql.api.constants.UpdateMethod; +import org.apache.nifi.service.cql.api.exception.QueryFailureException; +import org.apache.nifi.service.cql.api.lookup.CqlStatementResult; +import org.apache.nifi.service.cql.api.metadata.PrimaryKey; +import org.apache.nifi.service.cql.api.metadata.PrimaryKeyIdentifier; +import org.apache.nifi.service.cql.api.metadata.QualifiedTableName; +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.service.cql.api.service.CQLQueryCallback; +import org.apache.nifi.service.cql.api.service.QueryOverrides; +import org.apache.nifi.service.cql.api.service.WriteOverrides; + +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * This test class exists to support @see org.apache.nifi.processors.cassandra.ExecuteCQLQueryRecordTest + */ +public class MockCQLQueryExecutionService extends AbstractControllerService implements CQLExecutionService { + /** + * Stands in for a DataStax page-fetch exception. The driver types cannot be referenced from this module, + * and they never reach a processor anyway - a real service translates them before they cross the + * {@link CQLExecutionService} boundary. + */ + private static class SimulatedPageFetchFailure extends RuntimeException { + } + + /** Sentinel for {@link #rowsBeforeFailure} meaning "deliver the whole result set successfully". */ + private static final int NEVER_FAIL = -1; + + private final Iterator records; + private QueryOverrides lastQueryOverrides; + private String lastCql; + private List lastParameters; + private int rowsBeforeFailure = NEVER_FAIL; + + public MockCQLQueryExecutionService(Iterator records) { + this.records = records; + } + + /** + * Makes {@link #query} deliver {@code rowsBeforeFailure} rows and then fail, the way a page fetch does + * partway through a large result set. Every row delivered in this mode reports {@code hasMore = true}, + * since by definition the result set was not exhausted. Pass {@code 0} to fail before any row arrives. + * + * @return this service, for chaining onto the constructor + */ + public MockCQLQueryExecutionService failAfter(final int rowsBeforeFailure) { + this.rowsBeforeFailure = rowsBeforeFailure; + return this; + } + + /** + * Unimplemented on purpose: nothing in {@code nifi-cql-processors} calls {@code execute} - it exists for + * callers reading values without a schema over them, such as a distributed map cache - so a mock return + * value here would only be a guess at behaviour no test exercises. Throwing keeps that honest, and makes + * the first processor to need it fail loudly rather than against a fiction. + */ + @Override + public CqlStatementResult execute(String cql, List parameters, QueryOverrides overrides) { + throw new UnsupportedOperationException("execute() is not stubbed; no processor test needs it yet"); + } + + /** + * Mirrors the structure of a real implementation's {@code query}, not just its happy path: the entire row + * loop sits inside the try, so an exception thrown by the callback - a record writer that fails + * mid-result-set, say - is caught here too, gets {@link CQLQueryCallback#clear()} called on it, and is + * rethrown as a {@link ProcessException}. Cleanup on that path is easy to get wrong, and a mock that let + * the callback's exception escape untouched would never exercise it. + */ + @Override + public void query(String cql, List parameters, CQLQueryCallback callback, QueryOverrides overrides) throws QueryFailureException { + this.lastQueryOverrides = overrides; + this.lastCql = cql; + this.lastParameters = parameters; + + final boolean failing = rowsBeforeFailure != NEVER_FAIL; + + try { + long rowNumber = 0; + + while (records.hasNext() && (!failing || rowNumber < rowsBeforeFailure)) { + final Record next = records.next(); + callback.receive(++rowNumber, next, failing || records.hasNext()); + } + + if (failing) { + throw new SimulatedPageFetchFailure(); + } + } catch (final SimulatedPageFetchFailure e) { + callback.clear(); + throw new QueryFailureException(); + } catch (final Exception ex) { + callback.clear(); + throw new ProcessException("Error querying CQL", ex); + } + } + + public QueryOverrides getLastQueryOverrides() { + return lastQueryOverrides; + } + + public String getLastCql() { + return lastCql; + } + + public List getLastParameters() { + return lastParameters; + } + + @Override + public void insert(QualifiedTableName table, Record record, Map primaryKeyOverrides, WriteOverrides overrides) { + throw new UnsupportedOperationException(); + } + + @Override + public void insert(QualifiedTableName table, List records, Map primaryKeyOverrides, CqlBatchType batchType, WriteOverrides overrides) { + throw new UnsupportedOperationException(); + } + + @Override + public String getTransitUrl(QualifiedTableName tableName) { + return ""; + } + + @Override + public void delete(QualifiedTableName cassandraTable, Record record, Map primaryKeyOverrides, List updateKeys) { + throw new UnsupportedOperationException(); + } + + @Override + public void update(QualifiedTableName cassandraTable, Record record, Map primaryKeyOverrides, + List updateKeys, UpdateMethod updateMethod, WriteOverrides overrides) { + throw new UnsupportedOperationException(); + } + + @Override + public void update(QualifiedTableName cassandraTable, List records, Map primaryKeyOverrides, List updateKeys, + UpdateMethod updateMethod, CqlBatchType batchType, WriteOverrides overrides) { + throw new UnsupportedOperationException(); + } + + @Override + public PrimaryKey getMetadata(QualifiedTableName table) { + return null; + } + + @Override + public String getDefaultKeyspace() { + return "nifi_test"; + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api-nar/pom.xml b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api-nar/pom.xml new file mode 100644 index 000000000000..2fd86c481312 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api-nar/pom.xml @@ -0,0 +1,41 @@ + + + + + nifi-cql-bundle + org.apache.nifi + 2.12.0-SNAPSHOT + + 4.0.0 + + nifi-cql-services-api-nar + nar + + + + org.apache.nifi + nifi-standard-shared-nar + 2.12.0-SNAPSHOT + nar + + + org.apache.nifi + nifi-cql-services-api + 2.12.0-SNAPSHOT + compile + + + \ No newline at end of file diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api-nar/src/main/resources/META-INF/LICENSE b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api-nar/src/main/resources/META-INF/LICENSE new file mode 100644 index 000000000000..1fb4e7c0f39f --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api-nar/src/main/resources/META-INF/LICENSE @@ -0,0 +1,266 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +APACHE NIFI SUBCOMPONENTS: + +The Apache NiFi project contains subcomponents with separate copyright +notices and license terms. Your use of the source code for the these +subcomponents is subject to the terms and conditions of the following +licenses. + +This product bundles 'asm' which is available under a 3-Clause BSD style license. +For details see http://asm.ow2.org/asmdex-license.html + + Copyright (c) 2012 France Télécom + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. Neither the name of the copyright holders nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF + THE POSSIBILITY OF SUCH DAMAGE. + +The binary distribution of this product bundles 'JNR x86asm' under an MIT +style license. + + Copyright (C) 2010 Wayne Meissner + Copyright (c) 2008-2009, Petr Kobalicek + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation + files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following + conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api-nar/src/main/resources/META-INF/NOTICE b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api-nar/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000000..2d9d2aa14743 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api-nar/src/main/resources/META-INF/NOTICE @@ -0,0 +1,227 @@ +nifi-cassandra-services-api-nar +Copyright 2016-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +****************** +Apache Software License v2 +****************** + + (ASLv2) The Netty Project + The following NOTICE information applies: + Copyright 2014 The Netty Project + ------------------------------------------------------------------------------- + This product contains the extensions to Java Collections Framework which has + been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + + This product contains a modified version of Robert Harder's Public Domain + Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + + This product contains a modified portion of 'Webbit', an event based + WebSocket and HTTP server, which can be obtained at: + + * LICENSE: + * license/LICENSE.webbit.txt (BSD License) + * HOMEPAGE: + * https://github.com/joewalnes/webbit + + This product contains a modified portion of 'SLF4J', a simple logging + facade for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + + This product contains a modified portion of 'Apache Harmony', an open source + Java SE, which can be obtained at: + + * LICENSE: + * license/LICENSE.harmony.txt (Apache License 2.0) + * HOMEPAGE: + * http://archive.apache.org/dist/harmony/ + + This product contains a modified portion of 'jbzip2', a Java bzip2 compression + and decompression library written by Matthew J. Francis. It can be obtained at: + + * LICENSE: + * license/LICENSE.jbzip2.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jbzip2/ + + This product contains a modified portion of 'libdivsufsort', a C API library to construct + the suffix array and the Burrows-Wheeler transformed string for any input string of + a constant-size alphabet written by Yuta Mori. It can be obtained at: + + * LICENSE: + * license/LICENSE.libdivsufsort.txt (MIT License) + * HOMEPAGE: + * https://github.com/y-256/libdivsufsort + + This product contains a modified portion of Nitsan Wakart's 'JCTools', Java Concurrency Tools for the JVM, + which can be obtained at: + + * LICENSE: + * license/LICENSE.jctools.txt (ASL2 License) + * HOMEPAGE: + * https://github.com/JCTools/JCTools + + This product optionally depends on 'JZlib', a re-implementation of zlib in + pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + + This product optionally depends on 'Compress-LZF', a Java library for encoding and + decoding data in LZF format, written by Tatu Saloranta. It can be obtained at: + + * LICENSE: + * license/LICENSE.compress-lzf.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/ning/compress + + This product optionally depends on 'lz4', a LZ4 Java compression + and decompression library written by Adrien Grand. It can be obtained at: + + * LICENSE: + * license/LICENSE.lz4.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jpountz/lz4-java + + This product optionally depends on 'lzma-java', a LZMA Java compression + and decompression library, which can be obtained at: + + * LICENSE: + * license/LICENSE.lzma-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/jponge/lzma-java + + This product contains a modified portion of 'jfastlz', a Java port of FastLZ compression + and decompression library written by William Kinney. It can be obtained at: + + * LICENSE: + * license/LICENSE.jfastlz.txt (MIT License) + * HOMEPAGE: + * https://code.google.com/p/jfastlz/ + + This product contains a modified portion of and optionally depends on 'Protocol Buffers', Google's data + interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/protobuf + + This product optionally depends on 'Bouncy Castle Crypto APIs' to generate + a temporary self-signed X.509 certificate when the JVM does not provide the + equivalent functionality. It can be obtained at: + + * LICENSE: + * license/LICENSE.bouncycastle.txt (MIT License) + * HOMEPAGE: + * http://www.bouncycastle.org/ + + This product optionally depends on 'Snappy', a compression library produced + by Google Inc, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy.txt (New BSD License) + * HOMEPAGE: + * https://github.com/google/snappy + + This product optionally depends on 'JBoss Marshalling', an alternative Java + serialization API, which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-marshalling.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://www.jboss.org/jbossmarshalling + + This product optionally depends on 'Caliper', Google's micro- + benchmarking framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.caliper.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/google/caliper + + This product optionally depends on 'Apache Log4J', a logging framework, which + can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + + This product optionally depends on 'Aalto XML', an ultra-high performance + non-blocking XML processor, which can be obtained at: + + * LICENSE: + * license/LICENSE.aalto-xml.txt (Apache License 2.0) + * HOMEPAGE: + * http://wiki.fasterxml.com/AaltoHome + + This product contains a modified version of 'HPACK', a Java implementation of + the HTTP/2 HPACK algorithm written by Twitter. It can be obtained at: + + * LICENSE: + * license/LICENSE.hpack.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/twitter/hpack + + This product contains a modified portion of 'Apache Commons Lang', a Java library + provides utilities for the java.lang API, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-lang.txt (Apache License 2.0) + * HOMEPAGE: + * https://commons.apache.org/proper/commons-lang/ + + This product contains a forked and modified version of Tomcat Native + + * LICENSE: + * ASL2 + * HOMEPAGE: + * http://tomcat.apache.org/native-doc/ + * https://svn.apache.org/repos/asf/tomcat/native/ + + (ASLv2) Guava + The following NOTICE information applies: + Guava + Copyright 2015 The Guava Authors + + (ASLv2) Dropwizard Metrics + The following NOTICE information applies: + Copyright (c) 2010-2013 Coda Hale, Yammer.com + + This product includes software developed by Coda Hale and Yammer, Inc. + + This product includes code derived from the JSR-166 project (ThreadLocalRandom, Striped64, + LongAdder), which was released with the following comments: + + Written by Doug Lea with assistance from members of JCP JSR-166 + Expert Group and released to the public domain, as explained at + http://creativecommons.org/publicdomain/zero/1.0/ + + ************************ + Eclipse Public License 1.0 + ************************ + + The following binary components are provided under the Eclipse Public License 1.0. See project link for details. + + (EPL 2.0)(GPL 2)(LGPL 2.1) JNR Posix ( jnr.posix ) https://github.com/jnr/jnr-posix/blob/master/LICENSE.txt + diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/pom.xml b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/pom.xml new file mode 100644 index 000000000000..023c11486340 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/pom.xml @@ -0,0 +1,91 @@ + + + + + nifi-cql-bundle + org.apache.nifi + 2.12.0-SNAPSHOT + + 4.0.0 + + nifi-cql-services-api + jar + + + + org.apache.nifi + nifi-api + provided + + + + org.apache.nifi + nifi-record + + + + org.apache.nifi + nifi-ssl-context-service-api + provided + + + org.apache.nifi + nifi-record-path + 2.12.0-SNAPSHOT + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + ban-database-client-dependencies + + enforce + + + + + + This module is the backend-agnostic contract that the processors and every other + non-database module compile against, so anything on its classpath leaks + transitively to all of them. It must not depend on any database client + library. Consistency-level names are declared locally in + CqlConsistencyLevel rather than derived from a driver enum for exactly + this reason. If you need a driver type here, the abstraction is wrong. + + true + + + org.apache.cassandra:java-driver-* + com.datastax.oss:* + + com.datastax.cassandra:* + + com.scylladb:* + + + + + + + + + + diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/ConnectionCompression.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/ConnectionCompression.java new file mode 100644 index 000000000000..79aba40608ad --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/ConnectionCompression.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cql.api.constants; + +import org.apache.nifi.components.DescribedValue; +import org.apache.nifi.service.cql.api.service.AbstractCQLExecutionService; + +/** + * Allowable values for {@link AbstractCQLExecutionService#COMPRESSION_TYPE}: driver-to-cluster transport compression. + */ +public enum ConnectionCompression implements DescribedValue { + NONE("none", "None", "None"), + LZ4("lz4", "LZ4", "LZ4"), + SNAPPY("snappy", "Snappy", "Snappy"); + + private String value; + private String displayName; + private String description; + + ConnectionCompression(String value, String displayName, String description) { + this.value = value; + this.displayName = displayName; + this.description = description; + } + + @Override + public String getValue() { + return value; + } + + @Override + public String getDisplayName() { + return displayName; + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/CqlBatchType.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/CqlBatchType.java new file mode 100644 index 000000000000..ab847d14db5b --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/CqlBatchType.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cql.api.constants; + +import org.apache.nifi.service.cql.api.service.CQLExecutionService; + +/** + * Batch statement type for a multi-record {@link CQLExecutionService} write. {@code COUNTER} is required for + * counter mutations; they cannot go in a {@code LOGGED} or {@code UNLOGGED} batch. + */ +public enum CqlBatchType { + LOGGED, + UNLOGGED, + COUNTER +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/CqlConsistencyLevel.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/CqlConsistencyLevel.java new file mode 100644 index 000000000000..09008aaadb9d --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/CqlConsistencyLevel.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cql.api.constants; + +import org.apache.nifi.components.DescribedValue; +import org.apache.nifi.service.cql.api.service.AbstractCQLExecutionService; + +/** + * Allowable values for {@link AbstractCQLExecutionService#CONSISTENCY_LEVEL}: how many replicas must acknowledge a + * read or write for it to succeed. Declared here, not taken from a driver enum, to keep this module + * driver-independent; an implementation passes the value straight through, so these names must stay in sync + * with what its driver parses ({@code CqlConsistencyLevelTest} / {@code ScyllaConsistencyLevelTest} in the + * backend modules check that). + */ +public enum CqlConsistencyLevel implements DescribedValue { + ANY("ANY", "ANY", + "A write must be written to at least one node, or to a hinted handoff. Provides the lowest " + + "latency and the weakest guarantee. Write-only; not valid for reads."), + ONE("ONE", "ONE", + "Must be satisfied by the closest replica."), + TWO("TWO", "TWO", + "Must be satisfied by the two closest replicas."), + THREE("THREE", "THREE", + "Must be satisfied by the three closest replicas."), + QUORUM("QUORUM", "QUORUM", + "Must be satisfied by a quorum of replicas across all datacenters."), + ALL("ALL", "ALL", + "Must be satisfied by every replica. Provides the strongest guarantee, and fails if any replica " + + "is unavailable."), + LOCAL_ONE("LOCAL_ONE", "LOCAL_ONE", + "Must be satisfied by the closest replica in the local datacenter."), + LOCAL_QUORUM("LOCAL_QUORUM", "LOCAL_QUORUM", + "Must be satisfied by a quorum of replicas in the local datacenter, avoiding cross-datacenter " + + "latency."), + EACH_QUORUM("EACH_QUORUM", "EACH_QUORUM", + "Must be satisfied by a quorum of replicas in every datacenter. Write-only; not valid for reads."), + SERIAL("SERIAL", "SERIAL", + "Serial consistency for lightweight transactions, across all datacenters. Applies to the " + + "conditional phase of an LWT rather than to an ordinary read or write."), + LOCAL_SERIAL("LOCAL_SERIAL", "LOCAL_SERIAL", + "Serial consistency for lightweight transactions, restricted to the local datacenter."); + + private String value; + private String displayName; + private String description; + + CqlConsistencyLevel(String value, String displayName, String description) { + this.value = value; + this.displayName = displayName; + this.description = description; + } + + @Override + public String getValue() { + return value; + } + + @Override + public String getDisplayName() { + return displayName; + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/PrimaryKeyFieldType.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/PrimaryKeyFieldType.java new file mode 100644 index 000000000000..96dc576f9c5b --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/PrimaryKeyFieldType.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cql.api.constants; + +import org.apache.nifi.service.cql.api.metadata.PrimaryKey; +import org.apache.nifi.service.cql.api.metadata.PrimaryKeyIdentifier; +import org.apache.nifi.service.cql.api.metadata.PrimaryKeyMetadata; + +/** + * Which part of a table's primary key a column belongs to, on a {@link PrimaryKeyMetadata} or + * {@link PrimaryKeyIdentifier}. The partition key places a row on a node; clustering columns order rows + * within a partition. See {@link PrimaryKey}. + */ +public enum PrimaryKeyFieldType { + /** A clustering column - orders rows within a partition. */ + CLUSTERING, + /** A partition key column - determines which partition a row belongs to. */ + PARTITION +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/UpdateMethod.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/UpdateMethod.java new file mode 100644 index 000000000000..d04bc62da64b --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/constants/UpdateMethod.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cql.api.constants; + +import org.apache.nifi.service.cql.api.service.CQLExecutionService; + +/** + * How {@link CQLExecutionService#update} applies a record's non-key fields. {@code INCREMENT}/{@code DECREMENT} + * apply only to counter columns and require a {@code COUNTER} or {@code UNLOGGED} batch. + */ +public enum UpdateMethod { + /** Subtracts a field's value from a counter column. */ + DECREMENT, + /** Adds a field's value to a counter column. */ + INCREMENT, + /** Overwrites a column's value. */ + SET +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/exception/QueryFailureException.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/exception/QueryFailureException.java new file mode 100644 index 000000000000..db85abd5d1d2 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/exception/QueryFailureException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cql.api.exception; + +/** + * Signals that a {@code CQLExecutionService} call failed at the driver/server level (a query failure, an + * unreachable cluster) - a failure retrying may fix. A configuration or data problem is surfaced instead as + * an unchecked exception such as {@link IllegalArgumentException}. + */ +public class QueryFailureException extends RuntimeException { +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/CqlCell.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/CqlCell.java new file mode 100644 index 000000000000..6ef1465adf11 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/CqlCell.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.api.lookup; + +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.service.cql.api.service.CQLQueryCallback; + +/** + * One cell of a result row: a column name and its value, with no schema imposed. Where + * {@link CQLQueryCallback} delivers rows as typed NiFi records, a cell hands back what the server sent - + * for a caller that owns its own encoding (opaque bytes, say) and would only have to undo any conversion. + */ +public interface CqlCell { + + /** + * The column name, exactly as the server sent it. Not necessarily a legal CQL identifier + * ({@code writetime(v)}, {@code [applied]}), nor unique within a row ({@code SELECT v, writetime(v), v}) + * - which is why {@link CqlRow} is a list rather than a map. + */ + String columnName(); + + /** + * Whether this cell holds a CQL null - distinct from the column being absent from the row, which is + * {@link CqlRow#findCell(String)}. + */ + boolean isNull(); + + /** + * The value as binary (a fresh copy the caller may keep and modify), or {@code null} if the cell is + * null. The accessor for opaque bytes, and the one an implementation must normalize across backends. + * + * @throws UnsupportedOperationException if the column's type is not binary + */ + byte[] getBytes(); + + /** + * The value as the backend decoded it, no conversion applied, or {@code null} if the cell is null. + * Always a JDK type; a CQL type with no JDK equivalent (a UDT, a tuple, {@code duration}) throws rather + * than let a driver class cross this boundary - use {@link CQLExecutionService#query} for those. + * + * @throws UnsupportedOperationException if the column's type has no JDK representation + */ + Object getObject(); +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/CqlRow.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/CqlRow.java new file mode 100644 index 000000000000..2e2bbdcdbcfe --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/CqlRow.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.api.lookup; + +import java.util.List; +import java.util.Optional; + +/** + * One row of a statement result, with no schema imposed: its cells in selection order. Every other accessor + * here derives from {@link #cells()}. A list rather than a name-keyed map because selection order carries + * information, and column names are neither unique ({@code SELECT v, writetime(v), v}) nor always legal + * identifiers. + */ +public interface CqlRow { + + /** + * The row's cells, in selection order. Never {@code null}; empty only if the statement selected no + * columns. + */ + List cells(); + + /** The column names, in selection order; may contain duplicates or non-identifier names. */ + default List columnNames() { + return cells().stream().map(CqlCell::columnName).toList(); + } + + /** + * The first cell with this column name, or empty if there is none. Use {@link #cells()} where a name + * repeats. + */ + default Optional findCell(final String columnName) { + return cells().stream().filter(cell -> cell.columnName().equals(columnName)).findFirst(); + } + + /** + * The binary value of the named column. Throws if the row has no such column (a coding error); returns + * {@code null} if the column holds a CQL null (data). + * + * @throws IllegalArgumentException if the row has no column of that name + */ + default byte[] getBytes(final String columnName) { + return requireCell(columnName).getBytes(); + } + + /** + * As {@link #getBytes(String)}, for a column whose value is wanted as the backend decoded it. + * + * @throws IllegalArgumentException if the row has no column of that name + */ + default Object getObject(final String columnName) { + return requireCell(columnName).getObject(); + } + + private CqlCell requireCell(final String columnName) { + return findCell(columnName).orElseThrow(() -> new IllegalArgumentException( + "No column named '" + columnName + "'; row has " + columnNames())); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/CqlStatementResult.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/CqlStatementResult.java new file mode 100644 index 000000000000..6a4483e3c313 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/CqlStatementResult.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.api.lookup; + +import org.apache.nifi.service.cql.api.service.CQLExecutionService; + +import java.util.List; + +/** + * The outcome of a statement executed through {@link CQLExecutionService#execute}. + */ +public interface CqlStatementResult { + + /** + * Whether a conditional statement ({@code IF NOT EXISTS}, {@code IF EXISTS}, {@code IF }) + * applied. Always {@code true} for an unconditional one, so a caller that never writes conditionally has + * no special case. The backends report this as an {@code [applied]} column - not a legal identifier - + * which is surfaced here as an outcome and excluded from {@link #rows()}. + */ + boolean wasApplied(); + + /** + * The rows the server returned; never {@code null}, often empty. + * + *

A rejected conditional write returns the row that is actually stored, letting a + * compare-and-set report the current value in one round trip. An applied conditional write and an + * unconditional write return no rows; a {@code SELECT} returns its selected rows. Implementations must + * normalize the empty-when-applied case, which the backends otherwise report differently. + */ + List rows(); +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/impl/StandardCqlCell.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/impl/StandardCqlCell.java new file mode 100644 index 000000000000..4c9193cfcc3e --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/impl/StandardCqlCell.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.api.lookup.impl; + +import org.apache.nifi.service.cql.api.lookup.CqlCell; + +import java.nio.ByteBuffer; +import java.util.Objects; + +/** + * A {@link CqlCell} holding an already-decoded value, shared by both session providers. + * + * @param columnName the column name as the server sent it + * @param value the decoded value, or {@code null} for a CQL null; must be a JDK type, or use + * {@link #unsupported} + * @param unsupportedTypeName the CQL type name when it has no JDK representation, else {@code null}; the + * failure is deferred to {@link #getObject()} so one unreadable column does not + * fail the whole row + */ +public record StandardCqlCell(String columnName, Object value, String unsupportedTypeName) implements CqlCell { + + public StandardCqlCell { + Objects.requireNonNull(columnName, "columnName is required"); + } + + /** + * A cell whose value is a JDK type the caller can read. + */ + public StandardCqlCell(final String columnName, final Object value) { + this(columnName, value, null); + } + + /** A cell whose CQL type has no JDK representation (a UDT, a tuple, {@code duration}); reading it throws. */ + public static StandardCqlCell unsupported(final String columnName, final String cqlTypeName) { + return new StandardCqlCell(columnName, null, Objects.requireNonNull(cqlTypeName, "cqlTypeName is required")); + } + + @Override + public boolean isNull() { + return value == null && unsupportedTypeName == null; + } + + @Override + public byte[] getBytes() { + requireSupported(); + + return switch (value) { + case null -> null; + case byte[] bytes -> bytes.clone(); + // Read via a duplicate so the shared buffer's position is left alone: the backend may hand the + // same buffer to more than one reader, and consuming it here would empty it for the next. + case ByteBuffer buffer -> { + final ByteBuffer readable = buffer.duplicate(); + final byte[] bytes = new byte[readable.remaining()]; + readable.get(bytes); + yield bytes; + } + default -> throw new UnsupportedOperationException(String.format( + "Column '%s' is not binary; its value is a %s", columnName, value.getClass().getName())); + }; + } + + @Override + public Object getObject() { + requireSupported(); + return value; + } + + private void requireSupported() { + if (unsupportedTypeName != null) { + throw new UnsupportedOperationException(String.format( + "Column '%s' has CQL type %s, which has no JDK representation. Use CQLExecutionService.query " + + "for this column; it maps such types to NiFi records.", columnName, unsupportedTypeName)); + } + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/impl/StandardCqlRow.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/impl/StandardCqlRow.java new file mode 100644 index 000000000000..50829f9cc6ce --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/impl/StandardCqlRow.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.api.lookup.impl; + +import org.apache.nifi.service.cql.api.lookup.CqlCell; +import org.apache.nifi.service.cql.api.lookup.CqlRow; + +import java.util.List; +import java.util.Objects; + +/** + * A {@link CqlRow} backed by a fixed cell list; the interface derives every other accessor from + * {@link #cells()}. + * + * @param cells the row's cells in selection order; copied, non-null + */ +public record StandardCqlRow(List cells) implements CqlRow { + + public StandardCqlRow { + cells = List.copyOf(Objects.requireNonNull(cells, "cells is required")); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/impl/StandardCqlStatementResult.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/impl/StandardCqlStatementResult.java new file mode 100644 index 000000000000..4708f471a1c7 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/lookup/impl/StandardCqlStatementResult.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.api.lookup.impl; + +import org.apache.nifi.service.cql.api.lookup.CqlRow; +import org.apache.nifi.service.cql.api.lookup.CqlStatementResult; + +import java.util.List; +import java.util.Objects; + +/** + * A {@link CqlStatementResult} over a fixed outcome and row list. + * + * @param wasApplied whether a conditional statement applied; {@code true} for an unconditional one + * @param rows the rows the server returned, in order; copied, non-null + */ +public record StandardCqlStatementResult(boolean wasApplied, List rows) implements CqlStatementResult { + + /** An outcome with no rows, for a statement that returned none. */ + public static final CqlStatementResult APPLIED_WITH_NO_ROWS = new StandardCqlStatementResult(true, List.of()); + + public StandardCqlStatementResult { + rows = List.copyOf(Objects.requireNonNull(rows, "rows is required")); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/metadata/PrimaryKey.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/metadata/PrimaryKey.java new file mode 100644 index 000000000000..57d39f0e3403 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/metadata/PrimaryKey.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cql.api.metadata; + +import java.util.List; + +/** + * A table's primary key, from {@code CQLExecutionService#getMetadata}, exactly as the cluster's schema + * metadata reports it. Matching column names against other structures (such as + * {@link PrimaryKeyIdentifier}) is the caller's job. + * + * @param partitionKey the partition key columns, in declared order (which governs a row's token) + * @param clusteringKeys the clustering columns, in declared order (which governs row order within a + * partition); empty if the table has none + */ +public record PrimaryKey(List partitionKey, List clusteringKeys) { + +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/metadata/PrimaryKeyIdentifier.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/metadata/PrimaryKeyIdentifier.java new file mode 100644 index 000000000000..f5e3b603bd1c --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/metadata/PrimaryKeyIdentifier.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cql.api.metadata; + +/** + * Names one column of one keyspace-qualified table: the key type of the {@code primaryKeyOverrides} map on + * {@code CQLExecutionService#insert}/{@code update}/{@code delete}, each entry mapping a column to a + * {@code RecordPath} that resolves its value from a record instead of a same-named field. Caller-supplied + * (e.g. a {@code PutCQLRecord} dynamic property), so unlike {@link PrimaryKeyMetadata#name()} the + * {@code fieldName} may not be driver-normalized. + * + * @param keyspace the target table's keyspace + * @param tableName the target table, without keyspace + * @param fieldName the target column, as the caller supplied it + */ +public record PrimaryKeyIdentifier(String keyspace, String tableName, String fieldName) { +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/metadata/PrimaryKeyMetadata.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/metadata/PrimaryKeyMetadata.java new file mode 100644 index 000000000000..99fcfad828c6 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/metadata/PrimaryKeyMetadata.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cql.api.metadata; + +import org.apache.nifi.service.cql.api.constants.PrimaryKeyFieldType; + +/** + * One column of a table's primary key, within a {@link PrimaryKey} from + * {@code CQLExecutionService#getMetadata}. + * + * @param name the column's real, driver-normalized (case-correct, unquoted) name + * @param position the column's 0-based order within its own group (partition columns, or clustering + * columns), not a single ordinal across both + * @param fieldType whether this is a partition key column or a clustering column + */ +public record PrimaryKeyMetadata(String name, int position, PrimaryKeyFieldType fieldType) { +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/metadata/QualifiedTableName.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/metadata/QualifiedTableName.java new file mode 100644 index 000000000000..27aee8ae4865 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/metadata/QualifiedTableName.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cql.api.metadata; + +import org.apache.nifi.util.StringUtils; + +/** + * A table name, optionally keyspace-qualified, for every table-taking {@code CQLExecutionService} method. An + * unqualified instance ({@code null}/empty {@code keyspace}) resolves against the connection's default + * keyspace - the driver session's for statement building, or the service's configured one for schema + * metadata lookups. + * + * @param keyspace the table's keyspace, or {@code null}/empty if unqualified + * @param table the table name, without keyspace + */ +public record QualifiedTableName(String keyspace, String table) { + /** + * @return whether {@code keyspace} is set, so no default needs resolving + */ + public boolean isQualified() { + return StringUtils.isNotEmpty(keyspace); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/AbstractCQLExecutionService.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/AbstractCQLExecutionService.java new file mode 100644 index 000000000000..41f660567074 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/AbstractCQLExecutionService.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.api.service; + +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.resource.ResourceCardinality; +import org.apache.nifi.components.resource.ResourceType; +import org.apache.nifi.controller.AbstractControllerService; +import org.apache.nifi.expression.ExpressionLanguageScope; +import org.apache.nifi.processor.util.StandardValidators; +import org.apache.nifi.service.cql.api.constants.ConnectionCompression; +import org.apache.nifi.service.cql.api.constants.CqlConsistencyLevel; +import org.apache.nifi.ssl.SSLContextProvider; + +/** + * Base class for {@link CQLExecutionService} implementations. Holds the shared connection property + * descriptors both the Cassandra and ScyllaDB services expose identically, so a subclass only declares the + * order it lists them in and the driver-specific wiring behind them. + */ +public abstract class AbstractCQLExecutionService extends AbstractControllerService { + + public static final PropertyDescriptor CONTACT_POINTS = new PropertyDescriptor.Builder() + .name("Cassandra Contact Points") + .description(""" + Contact points are addresses of Cassandra nodes, as a comma-separated list of \ + hostname:port entries - for example node1:9042,node2:9042. An IPv6 address must be \ + bracketed to carry a port, as [::1]:9042. If an entry names no port, the default \ + client port of 9042 is used.""") + .required(true) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .addValidator(ContactPoints.VALIDATOR) + .build(); + + public static final PropertyDescriptor DATACENTER = new PropertyDescriptor.Builder() + .name("Cassandra Datacenter") + .description("The datacenter setting to use with your node/cluster.") + .required(true) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor KEYSPACE = new PropertyDescriptor.Builder() + .name("Default Keyspace") + .description(""" + The Cassandra Keyspace to connect to. If no keyspace is specified, the query will need to \ + include the keyspace name before any table reference, in case of 'query' native processors or \ + if the processor supports the 'Table' property, the keyspace name has to be provided with the \ + table name in the form of .

""") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor PROP_SSL_CONTEXT_SERVICE = new PropertyDescriptor.Builder() + .name("SSL Context Service") + .description(""" + The SSL Context Service used to provide client certificate information for TLS/SSL \ + connections.""") + .required(false) + .identifiesControllerService(SSLContextProvider.class) + .build(); + + public static final PropertyDescriptor USERNAME = new PropertyDescriptor.Builder() + .name("Username") + .description("Username to access the Cassandra cluster") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor PASSWORD = new PropertyDescriptor.Builder() + .name("Password") + .description("Password to access the Cassandra cluster") + .required(false) + .sensitive(true) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .build(); + + public static final PropertyDescriptor CONSISTENCY_LEVEL = new PropertyDescriptor.Builder() + .name("Consistency Level") + .description("The strategy for how many replicas must respond before results are returned.") + .required(true) + .allowableValues(CqlConsistencyLevel.class) + .defaultValue(CqlConsistencyLevel.ONE.getValue()) + .build(); + + public static final PropertyDescriptor FETCH_SIZE = new PropertyDescriptor.Builder() + .name("Fetch size") + .description(""" + The number of result rows to be fetched from the result set at a time - the page size the \ + driver requests per round trip, not a cap on total rows returned. Zero is the default and means \ + the driver's own default page size (5000) is used.""") + .defaultValue("0") + .required(true) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .addValidator(StandardValidators.INTEGER_VALIDATOR) + .build(); + + public static final PropertyDescriptor COMPRESSION_TYPE = new PropertyDescriptor.Builder() + .name("Compression Type") + .description("Enable compression at transport-level requests and responses") + .required(false) + .allowableValues(ConnectionCompression.class) + .addValidator(StandardValidators.NON_EMPTY_VALIDATOR) + .defaultValue(ConnectionCompression.NONE.getValue()) + .build(); + + public static final PropertyDescriptor READ_TIMEOUT = new PropertyDescriptor.Builder() + .name("Read Timeout") + .description("Read timeout. 0 means no timeout. If no value is set, the underlying default will be used.") + .required(false) + .defaultValue("30 sec") + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .build(); + + public static final PropertyDescriptor CONNECT_TIMEOUT = new PropertyDescriptor.Builder() + .name("Connect Timeout") + .description("Connection timeout. 0 means no timeout. If no value is set, the underlying default will be used.") + .required(false) + .defaultValue("30 sec") + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .build(); + + public static final PropertyDescriptor DEFAULT_TTL = new PropertyDescriptor.Builder() + .name("Default Time To Live") + .description(""" + The default Time To Live (TTL) to apply to INSERT statements and SET-method UPDATE statements when a \ + processor does not override it. Counter UPDATE statements never have a TTL applied, since Cassandra/ScyllaDB do not \ + support one on counter columns. If not set, no TTL is applied and the table's own default_time_to_live (if any) governs.""") + .required(false) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .addValidator(StandardValidators.TIME_PERIOD_VALIDATOR) + .build(); + + /** + * File settings override the other connection properties; anything the file omits falls back to them. + * This is the {@code DriverConfigLoader.compose(file, properties)} precedence of the DataStax Java + * Driver, which ScyllaDB's shard-aware fork shares along with the HOCON format. + */ + public static final PropertyDescriptor DRIVER_CONFIGURATION_FILE = new PropertyDescriptor.Builder() + .name("Driver Configuration File") + .description(""" + Path to an optional Java Driver configuration file using the driver's standard \ + typesafe-config (HOCON) format, for example an application.conf providing advanced \ + settings such as load balancing policy, retry policy, or speculative execution that \ + are not otherwise exposed as properties on this service. Settings in this file take \ + precedence over the other connection properties configured here; any setting not \ + present in the file falls back to those properties or the driver's built-in defaults. \ + ScyllaDB's Java Driver is a shard-aware fork of the DataStax Java Driver that reads the \ + same configuration format, so this file works unchanged when connecting to ScyllaDB.""") + .required(false) + .identifiesExternalResource(ResourceCardinality.SINGLE, ResourceType.FILE) + .expressionLanguageSupported(ExpressionLanguageScope.ENVIRONMENT) + .build(); +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/CQLExecutionService.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/CQLExecutionService.java new file mode 100644 index 000000000000..416484eea3de --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/CQLExecutionService.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.api.service; + +import org.apache.nifi.controller.ControllerService; +import org.apache.nifi.record.path.RecordPath; +import org.apache.nifi.service.cql.api.constants.CqlBatchType; +import org.apache.nifi.service.cql.api.constants.UpdateMethod; +import org.apache.nifi.service.cql.api.exception.QueryFailureException; +import org.apache.nifi.service.cql.api.lookup.CqlStatementResult; +import org.apache.nifi.service.cql.api.metadata.PrimaryKey; +import org.apache.nifi.service.cql.api.metadata.PrimaryKeyIdentifier; +import org.apache.nifi.service.cql.api.metadata.QualifiedTableName; + +import java.util.List; +import java.util.Map; + +/** + * Backend-agnostic connection service for Apache Cassandra/ScyllaDB, so {@code PutCQLRecord}, + * {@code ExecuteCQLQueryRecord} and {@code CQLDistributedMapCache} need not know which driver + * implementation is configured. The shared connection property descriptors live on + * {@link AbstractCQLExecutionService}; the methods here are the query/write contract each implementation + * must provide. + */ +public interface CQLExecutionService extends ControllerService { + + /** + * Executes a statement (typically a {@code SELECT}) and streams rows to {@code callback} one at a time, + * so arbitrarily large result sets are supported. Supply bind-marker values through {@code parameters} + * rather than interpolating them into {@code cql}, so untrusted input is sent as data, never parsed as + * CQL. + * + * @param cql statement text with {@code ?} bind markers + * @param parameters positional bind values, in order, or {@code null}/empty if none; each is converted + * to the Java type its bind marker's CQL type requires, so text still binds to a non-text column + * @param callback invoked once per row; the last row has {@code hasMore == false} + * @param overrides per-call fetch size/timeout; {@link QueryOverrides#NONE} for none + * @throws QueryFailureException on a driver/server-level failure (malformed input may instead be unchecked) + */ + void query(String cql, List parameters, CQLQueryCallback callback, QueryOverrides overrides) throws QueryFailureException; + + /** + * Executes a statement (not only {@code SELECT}) and returns its outcome. The counterpart to + * {@link #query} for two things a per-row callback cannot express: the applied/rejected outcome of a + * conditional write, and reading values with no schema imposed. Materializes its rows, so it is for + * bounded results (conditional writes, single-row lookups, existence probes); use {@link #query} for + * large result sets. + * + * @param cql statement text with {@code ?} bind markers + * @param parameters positional bind values, in order, converted as in {@link #query}, or {@code null}/empty if none + * @param overrides per-call fetch size/timeout; {@link QueryOverrides#NONE} for none + * @return the outcome; never {@code null} + * @throws QueryFailureException on a driver/server-level failure + */ + CqlStatementResult execute(String cql, List parameters, QueryOverrides overrides) throws QueryFailureException; + + /** + * Writes one record to {@code table} via {@code INSERT}. Each field maps to the column of the same name, + * primary key columns included, unless {@code primaryKeyOverrides} redirects one. + * + * @param table target table, keyspace-qualified or unqualified against the connection's default keyspace + * @param record the record to write + * @param primaryKeyOverrides maps a {@link PrimaryKeyIdentifier} to a {@link RecordPath} that resolves + * that column's value from the record (must select exactly one value) instead of a same-named field + * @param overrides per-call TTL/write-timestamp; {@link WriteOverrides#NONE} for none + * @throws QueryFailureException on a driver/server-level failure + */ + void insert(QualifiedTableName table, org.apache.nifi.serialization.record.Record record, Map primaryKeyOverrides, + WriteOverrides overrides) throws QueryFailureException; + + /** + * Batch form of {@link #insert(QualifiedTableName, org.apache.nifi.serialization.record.Record, Map, WriteOverrides)}: + * one {@code BatchStatement} of {@code batchType} over all {@code records}, which must share a schema + * (one prepared statement is built from the first and reused). {@code null}/empty {@code records} is a no-op. + * + * @throws QueryFailureException on a driver/server-level failure + */ + void insert(QualifiedTableName table, List records, Map primaryKeyOverrides, + CqlBatchType batchType, WriteOverrides overrides) throws QueryFailureException; + + /** + * @param tableName the table a write targeted + * @return a provenance identifier for that write; not a dereferenceable URL + */ + String getTransitUrl(QualifiedTableName tableName); + + /** + * Deletes the row identified by {@code updateKeys} from {@code cassandraTable}. + * + * @param cassandraTable target table, keyspace-qualified or unqualified against the default keyspace + * @param record supplies each {@code updateKeys} column's value by matching field name + * @param primaryKeyOverrides as in {@link #insert} + * @param updateKeys column names identifying the row; non-empty, each a field in {@code record} + * @throws QueryFailureException on a driver/server-level failure + */ + void delete(QualifiedTableName cassandraTable, org.apache.nifi.serialization.record.Record record, Map primaryKeyOverrides, + List updateKeys) throws QueryFailureException; + + /** + * Writes one record to {@code cassandraTable} via {@code UPDATE}: {@code updateKeys} form the + * {@code WHERE} clause, every other field is applied per {@code updateMethod}. + * + * @param cassandraTable target table, keyspace-qualified or unqualified against the default keyspace + * @param record the record to write + * @param primaryKeyOverrides as in {@link #insert} + * @param updateKeys column names identifying the row; non-empty, each a field in {@code record} + * @param updateMethod {@code SET}, or {@code INCREMENT}/{@code DECREMENT} for a counter column + * @param overrides per-call TTL/write-timestamp; ignored for {@code INCREMENT}/{@code DECREMENT}, which + * take neither a TTL nor a custom write timestamp + * @throws QueryFailureException on a driver/server-level failure + */ + void update(QualifiedTableName cassandraTable, org.apache.nifi.serialization.record.Record record, Map primaryKeyOverrides, + List updateKeys, UpdateMethod updateMethod, WriteOverrides overrides) throws QueryFailureException; + + /** + * Batch form of {@link #update(QualifiedTableName, org.apache.nifi.serialization.record.Record, Map, List, UpdateMethod, WriteOverrides)}: + * one {@code BatchStatement} of {@code batchType} over all {@code records}, which must share a schema. + * {@code INCREMENT}/{@code DECREMENT} require a {@code COUNTER} or {@code UNLOGGED} batch. + * {@code null}/empty {@code records} is a no-op. + * + * @throws QueryFailureException on a driver/server-level failure + */ + void update(QualifiedTableName cassandraTable, List records, Map primaryKeyOverrides, + List updateKeys, UpdateMethod updateMethod, + CqlBatchType batchType, WriteOverrides overrides) throws QueryFailureException; + + /** + * @param table the table to describe, keyspace-qualified or unqualified against the default keyspace + * @return {@code table}'s primary key (partition and clustering columns, in declared order), from the + * cluster's schema metadata + */ + PrimaryKey getMetadata(QualifiedTableName table); + + /** + * @return the default keyspace configured on this service, or {@code null} if none + */ + String getDefaultKeyspace(); +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/CQLQueryCallback.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/CQLQueryCallback.java new file mode 100644 index 000000000000..25d2fe66dcb9 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/CQLQueryCallback.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cql.api.service; + +import org.apache.nifi.serialization.record.Record; + +/** + * Receives {@link CQLExecutionService#query} rows one at a time, so a caller can act on rows before the + * whole result set is fetched. + */ +public interface CQLQueryCallback { + /** + * @param rowNumber the 1-based position of {@code result} in the result set + * @param result one row as a {@link Record}, its schema derived from the result set's column metadata + * @param hasMore whether another row follows; {@code false} on the last row, the signal to finalize + */ + void receive(long rowNumber, Record result, boolean hasMore); + + /** Called on a fatal error, so the callback can discard whatever it has accumulated. */ + void clear(); + + boolean hasSentOriginal(); +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/ContactPoints.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/ContactPoints.java new file mode 100644 index 000000000000..b5200cb2279c --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/ContactPoints.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.api.service; + +import org.apache.nifi.components.ValidationResult; +import org.apache.nifi.components.Validator; + +import java.util.ArrayList; +import java.util.List; + +/** + * Parses the Cassandra Contact Points property into host/port pairs, by the same rules at + * validation time ({@link #VALIDATOR}) and connection time ({@link #parse}), so the two cannot drift apart. + * No name resolution happens here, so validation never blocks on DNS. + * + *

Accepted comma-separated entry forms: + *

    + *
  • {@code host:port} or bare {@code host} (gets {@link #DEFAULT_PORT}), where host is a hostname or + * IPv4 literal
    • + *
  • a bracketed IPv6 literal, optional port: {@code [::1]:9042}, {@code [::1]}
    • + *
  • an unbracketed IPv6 literal (more than one colon), taken whole as the host, since a trailing port + * cannot be told from the address's own final group
    • + *
+ */ +public final class ContactPoints { + + /** The default CQL client port, used for any entry that names no port of its own. */ + public static final int DEFAULT_PORT = 9042; + + /** + * Validates a Contact Points value by parsing it. A value containing Expression Language passes through + * unvalidated; {@link #parse} checks it at connection time instead, with the same error messages. + */ + public static final Validator VALIDATOR = (subject, input, context) -> { + if (context.isExpressionLanguageSupported(subject) && context.isExpressionLanguagePresent(input)) { + return new ValidationResult.Builder() + .subject(subject).input(input).explanation("Expression Language present").valid(true).build(); + } + + try { + parse(input); + return new ValidationResult.Builder().subject(subject).input(input).valid(true).build(); + } catch (final IllegalArgumentException e) { + return new ValidationResult.Builder() + .subject(subject).input(input).explanation(e.getMessage()).valid(false).build(); + } + }; + + /** One parsed contact point. The host is never blank; the port is always in {@code 1-65535}. */ + public record HostPort(String host, int port) { + } + + private ContactPoints() { + } + + /** + * @param contactPointList comma-separated entries in the forms listed on this class + * @return one {@link HostPort} per entry, in order + * @throws IllegalArgumentException if blank, or any entry is malformed (the message names it) + */ + public static List parse(final String contactPointList) { + if (contactPointList == null || contactPointList.isBlank()) { + throw new IllegalArgumentException("At least one contact point is required"); + } + + final List contactPoints = new ArrayList<>(); + for (final String entry : contactPointList.split(",")) { + contactPoints.add(toContactPoint(entry.trim())); + } + + return contactPoints; + } + + private static HostPort toContactPoint(final String entry) { + final String host; + final String portText; + + if (entry.startsWith("[")) { + final int closingBracket = entry.indexOf(']'); + if (closingBracket < 0) { + throw new IllegalArgumentException(String.format( + "Contact point '%s' has an unterminated '[': a bracketed IPv6 literal looks like [::1] or [::1]:9042", entry)); + } + + host = entry.substring(1, closingBracket); + final String remainder = entry.substring(closingBracket + 1); + if (remainder.isEmpty()) { + portText = null; + } else if (remainder.startsWith(":")) { + portText = remainder.substring(1); + } else { + throw new IllegalArgumentException(String.format( + "Contact point '%s' has trailing text after ']': only an optional :port may follow", entry)); + } + } else { + final int firstColon = entry.indexOf(':'); + if (firstColon >= 0 && firstColon == entry.lastIndexOf(':')) { + host = entry.substring(0, firstColon); + portText = entry.substring(firstColon + 1); + } else { + // No colon (a bare host), or several (an unbracketed IPv6 literal): either way, no port. + host = entry; + portText = null; + } + } + + if (host.isBlank()) { + throw new IllegalArgumentException(String.format("Contact point '%s' has no host", entry)); + } + + return new HostPort(host.trim(), toPort(entry, portText)); + } + + private static int toPort(final String entry, final String portText) { + if (portText == null) { + return DEFAULT_PORT; + } + + final int port; + try { + port = Integer.parseInt(portText.trim()); + } catch (final NumberFormatException e) { + throw new IllegalArgumentException(String.format( + "Contact point '%s' has an invalid port '%s'", entry, portText.trim()), e); + } + + if (port < 1 || port > 65535) { + throw new IllegalArgumentException(String.format( + "Contact point '%s' has out-of-range port %d: expected 1-65535", entry, port)); + } + + return port; + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/QueryOverrides.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/QueryOverrides.java new file mode 100644 index 000000000000..18549e5c3db3 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/QueryOverrides.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.api.service; + +import java.time.Duration; + +/** + * Per-call overrides for one {@link CQLExecutionService#query}; a {@code null} field means "use the value + * configured on the service". + */ +public record QueryOverrides(Integer fetchSize, Duration timeout) { + public static final QueryOverrides NONE = new QueryOverrides(null, null); +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/WriteOverrides.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/WriteOverrides.java new file mode 100644 index 000000000000..76cc9d9e54a3 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/main/java/org/apache/nifi/service/cql/api/service/WriteOverrides.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.api.service; + +import java.time.Duration; + +/** + * Per-call overrides for one {@link CQLExecutionService#insert} or {@link CQLExecutionService#update}; a + * {@code null} field defers to the service's configuration. + * + * @param ttl the Time To Live to apply, or {@code null} for the service's default + * @param timestampField a record field supplying that record's CQL write timestamp, or {@code null} for + * the current time; ignored for Increment/Decrement + */ +public record WriteOverrides(Duration ttl, String timestampField) { + public static final WriteOverrides NONE = new WriteOverrides(null, null); +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/test/java/org/apache/nifi/service/cql/api/service/ContactPointsTest.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/test/java/org/apache/nifi/service/cql/api/service/ContactPointsTest.java new file mode 100644 index 000000000000..8142ad9de7ff --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-services-api/src/test/java/org/apache/nifi/service/cql/api/service/ContactPointsTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cql.api.service; + +import org.apache.nifi.service.cql.api.service.ContactPoints.HostPort; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class ContactPointsTest { + + @ParameterizedTest + @CsvSource({ + // hostname and IPv4, with and without a port + "node1:9042, node1, 9042", + "node1, node1, 9042", + "10.0.0.5:19042, 10.0.0.5, 19042", + "10.0.0.5, 10.0.0.5, 9042", + // bracketed IPv6, with and without a port + "[::1]:19042, ::1, 19042", + "[::1], ::1, 9042", + "[2001:db8::5]:9042, 2001:db8::5, 9042", + // unbracketed IPv6: the whole entry is the host, so no port can be named + "::1, ::1, 9042", + "2001:db8::5, 2001:db8::5, 9042", + // surrounding whitespace is insignificant + "' node1 : 9042 ', node1, 9042", + }) + void parsesOneEntry(final String entry, final String expectedHost, final int expectedPort) { + assertEquals(List.of(new HostPort(expectedHost, expectedPort)), ContactPoints.parse(entry)); + } + + @Test + void parsesSeveralEntriesInOrder() { + assertEquals( + List.of(new HostPort("node1", 9042), new HostPort("node2", 19042), new HostPort("::1", 9042)), + ContactPoints.parse("node1, node2:19042, [::1]")); + } + + /** + * Every rejection names the offending entry, since a multi-entry value would otherwise leave the user + * guessing which part was wrong. + */ + @ParameterizedTest + @ValueSource(strings = { + "node1:abc", // non-numeric port + "node1:", // empty port + "node1:0", // port below range + "node1:65536", // port above range + ":9042", // no host + "[::1", // unterminated bracket + "[::1]x", // trailing text after the bracket + }) + void rejectsMalformedEntryNamingIt(final String entry) { + final IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, () -> ContactPoints.parse("good:9042," + entry)); + assertTrue(e.getMessage().contains(entry.trim()), + () -> "expected message to name the entry '" + entry + "' but was: " + e.getMessage()); + } + + @ParameterizedTest + @ValueSource(strings = {"", " "}) + void rejectsBlankValue(final String value) { + assertThrows(IllegalArgumentException.class, () -> ContactPoints.parse(value)); + } + + @Test + void rejectsNullValue() { + assertThrows(IllegalArgumentException.class, () -> ContactPoints.parse(null)); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/pom.xml b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/pom.xml new file mode 100644 index 000000000000..16ebb04fd72b --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/pom.xml @@ -0,0 +1,103 @@ + + + + + nifi-cql-bundle + org.apache.nifi + 2.12.0-SNAPSHOT + + 4.0.0 + + nifi-cql-session-provider-shared + jar + + + 4.19.3 + + + + + org.apache.nifi + nifi-api + + + org.apache.nifi + nifi-utils + + + org.apache.nifi + nifi-cql-services-api + 2.12.0-SNAPSHOT + provided + + + + + org.apache.cassandra + java-driver-core + ${driver.version} + provided + + + + + org.apache.cassandra + java-driver-query-builder + ${driver.version} + + + org.apache.cassandra + java-driver-core + + + org.apache.cassandra + java-driver-guava-shaded + + + + + + org.apache.nifi + nifi-ssl-context-service-api + + + org.apache.nifi + nifi-framework-api + + + + org.apache.avro + avro + + + org.apache.nifi + nifi-record + compile + + + org.apache.nifi + nifi-avro-record-utils + 2.12.0-SNAPSHOT + compile + + + + org.apache.nifi + nifi-mock + test + + + diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/CassandraCQLExecutionService.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/CassandraCQLExecutionService.java new file mode 100644 index 000000000000..b626c9d341b4 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/CassandraCQLExecutionService.java @@ -0,0 +1,1272 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.nifi.service.cassandra; + +import com.datastax.oss.driver.api.core.AllNodesFailedException; +import com.datastax.oss.driver.api.core.CqlIdentifier; +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.CqlSessionBuilder; +import com.datastax.oss.driver.api.core.DriverTimeoutException; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.config.DriverConfigLoader; +import com.datastax.oss.driver.api.core.cql.BatchStatement; +import com.datastax.oss.driver.api.core.cql.BatchStatementBuilder; +import com.datastax.oss.driver.api.core.cql.BatchType; +import com.datastax.oss.driver.api.core.cql.BoundStatement; +import com.datastax.oss.driver.api.core.cql.ColumnDefinitions; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import com.datastax.oss.driver.api.core.cql.SimpleStatementBuilder; +import com.datastax.oss.driver.api.core.data.UdtValue; +import com.datastax.oss.driver.api.core.metadata.schema.ColumnMetadata; +import com.datastax.oss.driver.api.core.metadata.schema.KeyspaceMetadata; +import com.datastax.oss.driver.api.core.metadata.schema.TableMetadata; +import com.datastax.oss.driver.api.core.servererrors.QueryExecutionException; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.DataTypes; +import com.datastax.oss.driver.api.core.type.ListType; +import com.datastax.oss.driver.api.core.type.MapType; +import com.datastax.oss.driver.api.core.type.SetType; +import com.datastax.oss.driver.api.core.type.TupleType; +import com.datastax.oss.driver.api.core.type.UserDefinedType; +import com.datastax.oss.driver.api.core.type.codec.registry.MutableCodecRegistry; +import com.datastax.oss.driver.api.querybuilder.QueryBuilder; +import com.datastax.oss.driver.api.querybuilder.delete.DeleteSelection; +import com.datastax.oss.driver.api.querybuilder.insert.Insert; +import com.datastax.oss.driver.api.querybuilder.insert.InsertInto; +import com.datastax.oss.driver.api.querybuilder.insert.RegularInsert; +import com.datastax.oss.driver.api.querybuilder.relation.Relation; +import com.datastax.oss.driver.api.querybuilder.update.Update; +import com.datastax.oss.driver.api.querybuilder.update.UpdateStart; +import com.datastax.oss.driver.api.querybuilder.update.UpdateWithAssignments; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.apache.commons.lang3.StringUtils; +import org.apache.nifi.annotation.documentation.CapabilityDescription; +import org.apache.nifi.annotation.documentation.SeeAlso; +import org.apache.nifi.annotation.documentation.Tags; +import org.apache.nifi.annotation.lifecycle.OnDisabled; +import org.apache.nifi.annotation.lifecycle.OnEnabled; +import org.apache.nifi.avro.AvroTypeUtil; +import org.apache.nifi.components.ConfigVerificationResult; +import org.apache.nifi.components.PropertyDescriptor; +import org.apache.nifi.components.PropertyValue; +import org.apache.nifi.controller.ConfigurationContext; +import org.apache.nifi.controller.VerifiableControllerService; +import org.apache.nifi.logging.ComponentLog; +import org.apache.nifi.processor.exception.ProcessException; +import org.apache.nifi.record.path.FieldValue; +import org.apache.nifi.record.path.RecordPath; +import org.apache.nifi.record.path.RecordPathResult; +import org.apache.nifi.serialization.record.MapRecord; +import org.apache.nifi.serialization.record.Record; +import org.apache.nifi.serialization.record.RecordSchema; +import org.apache.nifi.serialization.record.util.DataTypeUtils; +import org.apache.nifi.service.cassandra.mapping.CassandraUdtSchemaMapper; +import org.apache.nifi.service.cassandra.mapping.CharacterCodec; +import org.apache.nifi.service.cassandra.mapping.FlexibleBigIntCodec; +import org.apache.nifi.service.cassandra.mapping.FlexibleBooleanCodec; +import org.apache.nifi.service.cassandra.mapping.FlexibleCounterCodec; +import org.apache.nifi.service.cassandra.mapping.FlexibleDateCodec; +import org.apache.nifi.service.cassandra.mapping.FlexibleDoubleCodec; +import org.apache.nifi.service.cassandra.mapping.FlexibleFloatCodec; +import org.apache.nifi.service.cassandra.mapping.FlexibleIntCodec; +import org.apache.nifi.service.cassandra.mapping.FlexibleSmallIntCodec; +import org.apache.nifi.service.cassandra.mapping.FlexibleTimeCodec; +import org.apache.nifi.service.cassandra.mapping.FlexibleTinyIntCodec; +import org.apache.nifi.service.cassandra.mapping.JavaSQLDateCodec; +import org.apache.nifi.service.cassandra.mapping.JavaSQLTimeCodec; +import org.apache.nifi.service.cassandra.mapping.JavaSQLTimestampCodec; +import org.apache.nifi.service.cql.api.constants.CqlBatchType; +import org.apache.nifi.service.cql.api.constants.PrimaryKeyFieldType; +import org.apache.nifi.service.cql.api.constants.UpdateMethod; +import org.apache.nifi.service.cql.api.exception.QueryFailureException; +import org.apache.nifi.service.cql.api.lookup.CqlCell; +import org.apache.nifi.service.cql.api.lookup.CqlRow; +import org.apache.nifi.service.cql.api.lookup.CqlStatementResult; +import org.apache.nifi.service.cql.api.lookup.impl.StandardCqlCell; +import org.apache.nifi.service.cql.api.lookup.impl.StandardCqlRow; +import org.apache.nifi.service.cql.api.lookup.impl.StandardCqlStatementResult; +import org.apache.nifi.service.cql.api.metadata.PrimaryKey; +import org.apache.nifi.service.cql.api.metadata.PrimaryKeyIdentifier; +import org.apache.nifi.service.cql.api.metadata.PrimaryKeyMetadata; +import org.apache.nifi.service.cql.api.metadata.QualifiedTableName; +import org.apache.nifi.service.cql.api.service.AbstractCQLExecutionService; +import org.apache.nifi.service.cql.api.service.CQLExecutionService; +import org.apache.nifi.service.cql.api.service.CQLQueryCallback; +import org.apache.nifi.service.cql.api.service.ContactPoints; +import org.apache.nifi.service.cql.api.service.QueryOverrides; +import org.apache.nifi.service.cql.api.service.WriteOverrides; +import org.apache.nifi.ssl.SSLContextProvider; + +import java.io.File; +import java.net.InetSocketAddress; +import java.time.Duration; +import java.time.Instant; +import java.time.format.DateTimeParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import javax.net.ssl.SSLContext; + +@Tags({"cassandra", "cql", "database", "connection", "session", "pooling"}) +@CapabilityDescription("Provides a CQL connection session for CQL processors and controller services to work with Apache Cassandra. " + + "Use this service for Apache Cassandra clusters; use ScyllaDBCQLExecutionService for ScyllaDB.") +@SeeAlso(classNames = { + "org.apache.nifi.processors.cql.PutCQLRecord", + "org.apache.nifi.processors.cql.ExecuteCQLQueryRecord", + "org.apache.nifi.service.cql.cache.CQLDistributedMapCache", + "org.apache.nifi.service.scylladb.ScyllaDBCQLExecutionService" +}) +public class CassandraCQLExecutionService extends AbstractCQLExecutionService implements CQLExecutionService, VerifiableControllerService { + + /** + * The column a conditional statement's outcome arrives in. Named with brackets by the protocol precisely + * because no CQL identifier may contain them, so it can never collide with a real column. + */ + private static final String APPLIED_COLUMN = "[applied]"; + + /** + * Bind marker name for an explicit per-record write timestamp (see {@link WriteOverrides#timestampField()}). + * Namespaced to make an accidental collision with a real column name effectively impossible. + */ + private static final String WRITE_TIMESTAMP_BIND_MARKER = "nifi_write_timestamp"; + + private static final boolean VALUE_LOGGING_ENABLED = Boolean.getBoolean("ENABLE_CQL_VALUE_LOGGING"); + + private CqlSession cassandraSession; + + private int pageSize; + + private Duration defaultTtl; + + public static final List PROPERTY_DESCRIPTORS = List.of( + CONTACT_POINTS, + DATACENTER, + KEYSPACE, + USERNAME, + PASSWORD, + PROP_SSL_CONTEXT_SERVICE, + FETCH_SIZE, + READ_TIMEOUT, + CONNECT_TIMEOUT, + CONSISTENCY_LEVEL, + COMPRESSION_TYPE, + DEFAULT_TTL, + DRIVER_CONFIGURATION_FILE + ); + + @Override + public List getSupportedPropertyDescriptors() { + return PROPERTY_DESCRIPTORS; + } + + @OnEnabled + public void onEnabled(final ConfigurationContext context) { + tableMetadataCache = new ConcurrentHashMap<>(); + connectToCassandra(context); + } + + @OnDisabled + public void onDisabled() { + if (cassandraSession != null) { + cassandraSession.close(); + cassandraSession = null; + } + } + + private Map tableMetadataCache; + private String defaultKeyspace; + + /** + * Returns the given table name with the connection's default keyspace filled in when it is unqualified, so + * metadata lookups have the explicit keyspace they require. + */ + private QualifiedTableName resolveKeyspace(QualifiedTableName table) { + return table.isQualified() ? table : new QualifiedTableName(defaultKeyspace, table.table()); + } + + private void connectToCassandra(ConfigurationContext context) { + if (cassandraSession == null) { + this.pageSize = context.getProperty(FETCH_SIZE).evaluateAttributeExpressions().asInteger(); + + final PropertyValue defaultTtlProperty = context.getProperty(DEFAULT_TTL).evaluateAttributeExpressions(); + this.defaultTtl = defaultTtlProperty.isSet() ? defaultTtlProperty.asDuration() : null; + + final CqlSession cqlSession = buildSession(context); + + try { + MutableCodecRegistry codecRegistry = + (MutableCodecRegistry) cqlSession.getContext().getCodecRegistry(); + + codecRegistry.register(new JavaSQLTimestampCodec()); + codecRegistry.register(new JavaSQLDateCodec()); + codecRegistry.register(new JavaSQLTimeCodec()); + codecRegistry.register(new CharacterCodec()); + codecRegistry.register(new FlexibleCounterCodec()); + codecRegistry.register(new FlexibleBooleanCodec()); + codecRegistry.register(new FlexibleTinyIntCodec()); + codecRegistry.register(new FlexibleSmallIntCodec()); + codecRegistry.register(new FlexibleIntCodec()); + codecRegistry.register(new FlexibleBigIntCodec()); + codecRegistry.register(new FlexibleFloatCodec()); + codecRegistry.register(new FlexibleDoubleCodec()); + codecRegistry.register(new FlexibleDateCodec()); + codecRegistry.register(new FlexibleTimeCodec()); + } catch (Exception ex) { + cqlSession.close(); + throw new ProcessException("There was an error registering conversion codecs.", ex); + } + + cassandraSession = cqlSession; + } + } + + private CqlSession buildSession(final ConfigurationContext context) { + return buildSession(context, true); + } + + /** + * @param bindKeyspace when {@code false}, the session is opened without binding the configured keyspace. + * {@link #verify(ConfigurationContext, ComponentLog, Map)} passes {@code false} so a keyspace that does + * not exist is reported by the dedicated "Verify Keyspace" step rather than failing session creation + * outright - the driver rejects an unknown keyspace during {@code CqlSession.build()}. + */ + private CqlSession buildSession(final ConfigurationContext context, final boolean bindKeyspace) { + final String consistencyLevel = context.getProperty(CONSISTENCY_LEVEL).getValue(); + final String compression = context.getProperty(COMPRESSION_TYPE).getValue(); + final String contactPointList = context.getProperty(CONTACT_POINTS).evaluateAttributeExpressions().getValue(); + + List contactPoints = getContactPoints(contactPointList); + + // Set up the client for secure (SSL/TLS communications) if configured to do so + final SSLContextProvider sslContextProvider = + context.getProperty(PROP_SSL_CONTEXT_SERVICE).asControllerService(SSLContextProvider.class); + final SSLContext sslContext; + + if (sslContextProvider == null) { + sslContext = null; + } else { + sslContext = sslContextProvider.createContext(); + } + + final String username; + final String password; + PropertyValue usernameProperty = context.getProperty(USERNAME).evaluateAttributeExpressions(); + PropertyValue passwordProperty = context.getProperty(PASSWORD).evaluateAttributeExpressions(); + + if (usernameProperty != null && passwordProperty != null) { + username = usernameProperty.getValue(); + password = passwordProperty.getValue(); + } else { + username = null; + password = null; + } + + final Duration readTimeout = context.getProperty(READ_TIMEOUT).evaluateAttributeExpressions().asDuration(); + final Duration connectTimeout = context.getProperty(CONNECT_TIMEOUT).evaluateAttributeExpressions().asDuration(); + + final String datacenter = context.getProperty(DATACENTER).evaluateAttributeExpressions().getValue(); + + final String sessionKeyspace = context.getProperty(KEYSPACE).evaluateAttributeExpressions().getValue(); + this.defaultKeyspace = sessionKeyspace; + + final DriverConfigLoader propertyBasedLoader = + DriverConfigLoader.programmaticBuilder() + .withDuration(DefaultDriverOption.CONNECTION_CONNECT_TIMEOUT, connectTimeout) + .withDuration(DefaultDriverOption.REQUEST_TIMEOUT, readTimeout) + .withString(DefaultDriverOption.REQUEST_CONSISTENCY, consistencyLevel) + .withString(DefaultDriverOption.PROTOCOL_COMPRESSION, compression) + .build(); + + final DriverConfigLoader loader = buildConfigLoader(context, propertyBasedLoader); + + CqlSessionBuilder builder = CqlSession.builder() + .addContactPoints(contactPoints); + + if (StringUtils.isNotBlank(username) && StringUtils.isNotBlank(password)) { + builder = builder.withAuthCredentials(username, password); + } + + builder = builder + .withSslContext(sslContext) + .withLocalDatacenter(datacenter) + .withConfigLoader(loader); + + if (bindKeyspace) { + builder = builder.withKeyspace(sessionKeyspace); + } + + return builder.build(); + } + + /** + * Builds the {@link DriverConfigLoader} used to open the session. When a Driver Configuration File is + * configured, it is composed as the primary source ahead of {@code propertyBasedLoader}, so any option + * defined in the file overrides the corresponding connection property, while options absent from the + * file still fall back to the property-derived configuration. The file uses the driver's standard + * typesafe-config format, which the ScyllaDB Java Driver reads identically, so this method requires no + * override for ScyllaDB compatibility. + */ + protected DriverConfigLoader buildConfigLoader(final ConfigurationContext context, final DriverConfigLoader propertyBasedLoader) { + final PropertyValue configFileProperty = context.getProperty(DRIVER_CONFIGURATION_FILE); + + if (configFileProperty == null || !configFileProperty.isSet()) { + return propertyBasedLoader; + } + + final String configFilePath = configFileProperty.evaluateAttributeExpressions().getValue(); + final DriverConfigLoader fileBasedLoader = DriverConfigLoader.fromFile(new File(configFilePath)); + + return DriverConfigLoader.compose(fileBasedLoader, propertyBasedLoader); + } + + @Override + public List verify(final ConfigurationContext context, final ComponentLog verificationLogger, final Map variables) { + final List results = new ArrayList<>(); + + final String datacenter = context.getProperty(DATACENTER).evaluateAttributeExpressions().getValue(); + + CqlSession session = null; + try { + // Opened without binding the keyspace: the driver rejects an unknown keyspace during build(), + // which would collapse a missing-keyspace failure into this step instead of the "Verify Keyspace" + // one below. Keyspace existence is checked explicitly against the session metadata further down. + session = buildSession(context, false); + results.add(new ConfigVerificationResult.Builder() + .verificationStepName("Establish Connection") + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .explanation("Successfully connected to Cassandra using datacenter [" + datacenter + "]") + .build()); + } catch (final Exception e) { + verificationLogger.warn("Failed to establish Cassandra connection using datacenter [{}]: {}", datacenter, e.getMessage()); + results.add(new ConfigVerificationResult.Builder() + .verificationStepName("Establish Connection") + .outcome(ConfigVerificationResult.Outcome.FAILED) + .explanation("Failed to connect using datacenter [" + datacenter + "]: " + e.getMessage()) + .build()); + } + + if (session != null) { + try { + // CqlSessionBuilder.build() succeeds even when no node belongs to the configured local + // datacenter: the driver only filters nodes by datacenter when computing a query plan for + // an actual request. Executing a trivial statement here forces that node selection so a + // bad datacenter name is caught during verification instead of on first real query. + try { + session.execute("SELECT release_version FROM system.local"); + results.add(new ConfigVerificationResult.Builder() + .verificationStepName("Verify Datacenter") + .outcome(ConfigVerificationResult.Outcome.SUCCESSFUL) + .explanation("Found an available node in datacenter [" + datacenter + "]") + .build()); + } catch (final Exception e) { + verificationLogger.warn("No available node found in datacenter [{}]: {}", datacenter, e.getMessage()); + results.add(new ConfigVerificationResult.Builder() + .verificationStepName("Verify Datacenter") + .outcome(ConfigVerificationResult.Outcome.FAILED) + .explanation("No available node found in datacenter [" + datacenter + "]: " + e.getMessage()) + .build()); + } + + final String keyspaceName = context.getProperty(KEYSPACE).isSet() + ? context.getProperty(KEYSPACE).evaluateAttributeExpressions().getValue() : null; + + if (StringUtils.isNotBlank(keyspaceName)) { + final boolean keyspaceFound = session.getMetadata().getKeyspace(keyspaceName).isPresent(); + results.add(new ConfigVerificationResult.Builder() + .verificationStepName("Verify Keyspace") + .outcome(keyspaceFound ? ConfigVerificationResult.Outcome.SUCCESSFUL : ConfigVerificationResult.Outcome.FAILED) + .explanation(keyspaceFound + ? "Found keyspace [" + keyspaceName + "]" + : "Keyspace [" + keyspaceName + "] was not found in the cluster") + .build()); + } + } finally { + session.close(); + } + } + + return results; + } + + private List getContactPoints(String contactPointList) { + return ContactPoints.parse(contactPointList).stream() + .map(contactPoint -> new InetSocketAddress(contactPoint.host(), contactPoint.port())) + .collect(Collectors.toList()); + } + + private void cacheMetadata(QualifiedTableName table) { + final QualifiedTableName resolved = resolveKeyspace(table); + getLogger().debug("Fetching metadata for {}.{}", resolved.keyspace(), resolved.table()); + + if (!tableMetadataCache.containsKey(resolved)) { + getLogger().debug("Metadata for {}.{} not found, fetching from cluster.", resolved.keyspace(), resolved.table()); + Optional keyspaceOpt = cassandraSession.getMetadata() + .getKeyspace(resolved.keyspace()); + if (keyspaceOpt.isEmpty()) { + throw new RuntimeException("Empty keyspace metadata"); + } + + KeyspaceMetadata keyspaceMetadata = keyspaceOpt.get(); + Optional tableOpt = keyspaceMetadata.getTable(resolved.table()); + if (tableOpt.isEmpty()) { + throw new RuntimeException("Empty table metadata"); + } + + TableMetadata metadata = tableOpt.get(); + PrimaryKey tableInfo = convertTableMetadata(metadata); + + tableMetadataCache.put(resolved, tableInfo); + getLogger().debug("Metadata fetched and cached for {}.{}", resolved.keyspace(), resolved.table()); + } + } + + private PrimaryKey convertTableMetadata(TableMetadata metadata) { + List partitions = new ArrayList<>(); + List clustering = new ArrayList<>(); + + int loc = 0; + for (ColumnMetadata m : metadata.getPartitionKey()) { + partitions.add(new PrimaryKeyMetadata(m.getName().asInternal(), loc++, PrimaryKeyFieldType.PARTITION)); + } + + loc = 0; + for (ColumnMetadata m : metadata.getClusteringColumns().keySet()) { + clustering.add(new PrimaryKeyMetadata(m.getName().asInternal(), loc++, PrimaryKeyFieldType.CLUSTERING)); + } + + return new PrimaryKey(partitions, clustering); + } + + private static String valuesFragment(final Object[] values) { + return VALUE_LOGGING_ENABLED ? " with values " + Arrays.toString(values) : ""; + } + + private static String valuesFragment(final List values) { + return VALUE_LOGGING_ENABLED && values != null && !values.isEmpty() ? " with values " + values : ""; + } + + @Override + public void query(String cql, List parameters, CQLQueryCallback callback, QueryOverrides overrides) throws QueryFailureException { + if (getLogger().isDebugEnabled()) { + getLogger().debug("Executing CQL query: {}{}", cql, valuesFragment(parameters)); + } + + BoundStatement boundStatement = bindStatement(cql, parameters, overrides); + + AtomicReference schemaReference = new AtomicReference<>(); + + // The whole lifecycle of the query - the initial execute() as well as every page fetch triggered by + // hasNext()/next() as the ResultSet is iterated - can surface a QueryExecutionException, so all of it + // is covered by one try, rather than only the page-fetch calls. + try { + ResultSet results = cassandraSession.execute(boundStatement); + Iterator resultsIterator = results.iterator(); + long rowNumber = 0; + + List columnNames = new ArrayList<>(); + + while (resultsIterator.hasNext()) { + Row row = resultsIterator.next(); + + if (schemaReference.get() == null) { + Schema generatedAvroSchema = createSchema(results); + RecordSchema converted = AvroTypeUtil.createSchema(generatedAvroSchema); + schemaReference.set(converted); + } + + if (columnNames.isEmpty()) { + row.getColumnDefinitions().forEach(def -> { + columnNames.add(def.getName().toString()); + }); + } + + Map resultMap = new HashMap<>(); + + for (int x = 0; x < columnNames.size(); x++) { + resultMap.put(columnNames.get(x), row.getObject(x)); + } + + MapRecord record = new MapRecord(schemaReference.get(), resultMap); + + callback.receive(++rowNumber, record, resultsIterator.hasNext()); + } + // Each of these is worth another attempt: a coordinator/replica-level execution failure, no reachable + // node, or a client-side request timeout. Deliberately not their DriverException base type - the + // validation errors under it (SyntaxError, InvalidQueryException, UnauthorizedException) can never + // succeed on a retry, so they belong on the failure path below. + } catch (QueryExecutionException | AllNodesFailedException | DriverTimeoutException qee) { + getLogger().error("Error executing query", qee); + callback.clear(); + throw new QueryFailureException(); + } catch (Exception ex) { + callback.clear(); + throw new ProcessException("Error querying CQL", ex); + } + } + + @Override + public CqlStatementResult execute(final String cql, final List parameters, final QueryOverrides overrides) + throws QueryFailureException { + if (getLogger().isDebugEnabled()) { + getLogger().debug("Executing CQL statement: {}{}", cql, valuesFragment(parameters)); + } + + final BoundStatement boundStatement = bindStatement(cql, parameters, overrides); + + try { + final ResultSet results = cassandraSession.execute(boundStatement); + final boolean wasApplied = results.wasApplied(); + + // The outcome column is present exactly when the statement was conditional - wasApplied() alone + // cannot tell us, since it also returns true for every unconditional statement. + final boolean conditional = results.getColumnDefinitions().contains(APPLIED_COLUMN); + + // A conditional write that applied has no prior value to report, so it reports no rows. The + // backends disagree about what they send in that case: Cassandra returns the outcome column + // alone, ScyllaDB returns the other columns alongside it. Normalizing here is the whole point of + // having one API over both - otherwise callers would find rows().isEmpty() backend-dependent. + if (conditional && wasApplied) { + return new StandardCqlStatementResult(true, List.of()); + } + + final List rows = new ArrayList<>(); + for (final Row row : results) { + rows.add(new StandardCqlRow(toCells(row))); + } + + return new StandardCqlStatementResult(wasApplied, rows); + } catch (QueryExecutionException | AllNodesFailedException | DriverTimeoutException qee) { + getLogger().error("Error executing statement", qee); + throw new QueryFailureException(); + } catch (Exception ex) { + throw new ProcessException("Error executing CQL", ex); + } + } + + /** + * Converts one driver row into cells, dropping the {@code [applied]} conditional-outcome column and marking + * any column whose CQL type has no JDK representation as unsupported. + */ + private List toCells(final Row row) { + final ColumnDefinitions definitions = row.getColumnDefinitions(); + final List cells = new ArrayList<>(definitions.size()); + + for (int index = 0; index < definitions.size(); index++) { + final String columnName = definitions.get(index).getName().toString(); + if (APPLIED_COLUMN.equals(columnName)) { + continue; + } + + final DataType columnType = definitions.get(index).getType(); + cells.add(hasJdkRepresentation(columnType) + ? new StandardCqlCell(columnName, row.getObject(index)) + : StandardCqlCell.unsupported(columnName, columnType.asCql(true, false))); + } + + return cells; + } + + /** + * Reports whether a CQL type decodes to a JDK type rather than a driver class, recursing through collection + * types so a {@code list} is rejected for its element type. + */ + private static boolean hasJdkRepresentation(final DataType type) { + return switch (type) { + case ListType list -> hasJdkRepresentation(list.getElementType()); + case SetType set -> hasJdkRepresentation(set.getElementType()); + case MapType map -> hasJdkRepresentation(map.getKeyType()) && hasJdkRepresentation(map.getValueType()); + default -> !(type instanceof UserDefinedType || type instanceof TupleType || DataTypes.DURATION.equals(type)); + }; + } + + /** + * Builds and binds a statement, applying any per-call fetch size and timeout overrides. Shared by + * {@link #query} and {@link #execute}, which differ only in what they do with the result. + */ + private BoundStatement bindStatement(final String cql, final List parameters, final QueryOverrides overrides) { + SimpleStatementBuilder statementBuilder = SimpleStatement.builder(cql) + .setPageSize(resolveFetchSize(overrides, pageSize)); + + final Duration timeoutOverride = resolveTimeoutOverride(overrides); + if (timeoutOverride != null) { + statementBuilder = statementBuilder.setTimeout(timeoutOverride); + } + + final SimpleStatement statement = statementBuilder.build(); + final PreparedStatement preparedStatement = cassandraSession.prepare(statement); + + return parameters != null && !parameters.isEmpty() + ? preparedStatement.bind(toBindValues(preparedStatement, parameters)) + : preparedStatement.bind(); + } + + static int resolveFetchSize(final QueryOverrides overrides, final int defaultFetchSize) { + return overrides != null && overrides.fetchSize() != null ? overrides.fetchSize() : defaultFetchSize; + } + + static Duration resolveTimeoutOverride(final QueryOverrides overrides) { + return overrides != null ? overrides.timeout() : null; + } + + static BatchType toDriverBatchType(final CqlBatchType batchType) { + return switch (batchType) { + case LOGGED -> BatchType.LOGGED; + case UNLOGGED -> BatchType.UNLOGGED; + case COUNTER -> BatchType.COUNTER; + }; + } + + static Integer resolveTtlSeconds(final WriteOverrides overrides, final Duration defaultTtl) { + final Duration ttl = overrides != null && overrides.ttl() != null ? overrides.ttl() : defaultTtl; + return ttl != null ? Math.toIntExact(ttl.toSeconds()) : null; + } + + static boolean hasTimestampOverride(final WriteOverrides overrides) { + return overrides != null && overrides.timestampField() != null; + } + + /** + * Converts a record field's value into epoch microseconds, the unit Cassandra/ScyllaDB expect for an + * explicit write timestamp (distinct from the millisecond precision of the CQL {@code timestamp} type). + */ + static long toEpochMicros(final Object value) { + final Instant instant; + if (value instanceof Instant instantValue) { + instant = instantValue; + } else if (value instanceof java.util.Date dateValue) { + instant = dateValue.toInstant(); + } else if (value instanceof Number numberValue) { + instant = Instant.ofEpochMilli(numberValue.longValue()); + } else { + throw new IllegalArgumentException("Cannot derive a write timestamp from a value of type " + + (value == null ? "null" : value.getClass().getName())); + } + + return instant.getEpochSecond() * 1_000_000L + instant.getNano() / 1_000L; + } + + protected GeneratedResult generateInsert(QualifiedTableName table, RecordSchema schema, + Map primaryKeyOverrides, + Integer ttlSeconds, boolean includeTimestampMarker) { + InsertInto insertQuery = table.isQualified() + ? QueryBuilder.insertInto(table.keyspace(), table.table()) + : QueryBuilder.insertInto(table.table()); + List keys = new ArrayList<>(); + + RegularInsert regularInsert = null; + for (String fieldName : columnNamesIncludingOverrideOnly(table, schema, primaryKeyOverrides)) { + if (regularInsert == null) { + regularInsert = insertQuery.value(fieldName, QueryBuilder.bindMarker(fieldName)); + } else { + regularInsert = regularInsert.value(fieldName, QueryBuilder.bindMarker(fieldName)); + } + + keys.add(fieldName); + } + + if (regularInsert == null) { + throw new ProcessException("Could not build an insert statement from the supplied record"); + } + + Insert insert = regularInsert; + if (ttlSeconds != null) { + insert = insert.usingTtl(ttlSeconds); + } + if (includeTimestampMarker) { + // A separate named bind marker (set by name on the bound statement, not part of "keys") rather than + // a literal, since - unlike TTL - each record in a batch can carry a different write timestamp. + insert = insert.usingTimestamp(QueryBuilder.bindMarker(WRITE_TIMESTAMP_BIND_MARKER)); + } + + return new GeneratedResult(insert.build(), keys); + } + + /** + * Returns every schema field name in schema order, followed by any primary key override target column for + * this table that has no same-named schema field. + */ + private List columnNamesIncludingOverrideOnly(QualifiedTableName table, RecordSchema schema, + Map primaryKeyOverrides) { + final List columnNames = new ArrayList<>(schema.getFieldNames()); + for (String overrideFieldName : overrideFieldNamesForTable(table, primaryKeyOverrides)) { + if (!columnNames.contains(overrideFieldName)) { + columnNames.add(overrideFieldName); + } + } + return columnNames; + } + + /** + * The field names of every primary key override configured for this specific table, resolving an + * unqualified {@code table} against {@link #defaultKeyspace} the same way {@link #getRecordPathOverride} + * does. + */ + private Set overrideFieldNamesForTable(QualifiedTableName table, Map primaryKeyOverrides) { + final QualifiedTableName resolved = resolveKeyspace(table); + return primaryKeyOverrides.keySet().stream() + .filter(identifier -> identifier.keyspace().equals(resolved.keyspace()) && identifier.tableName().equals(resolved.table())) + .map(PrimaryKeyIdentifier::fieldName) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } + + @Override + public void insert(QualifiedTableName table, Record record, Map primaryKeyOverrides, + WriteOverrides overrides) throws QueryFailureException { + final boolean useTimestampOverride = hasTimestampOverride(overrides); + GeneratedResult result = generateInsert(table, record.getSchema(), primaryKeyOverrides, resolveTtlSeconds(overrides, defaultTtl), useTimestampOverride); + PreparedStatement preparedStatement = cassandraSession.prepare(result.statement); + + Object[] values = getBindValues(table, primaryKeyOverrides, record, preparedStatement, result.keysUsed); + if (useTimestampOverride) { + setTimestampBindValue(values, preparedStatement, record, overrides); + } + + if (getLogger().isDebugEnabled()) { + getLogger().debug("Executing CQL insert: {}{}", result.statement.getQuery(), valuesFragment(values)); + } + + BoundStatement boundStatement = preparedStatement.bind(values); + + try { + cassandraSession.execute(boundStatement); + } catch (QueryExecutionException | AllNodesFailedException | DriverTimeoutException qee) { + getLogger().error("Error executing insert", qee); + throw new QueryFailureException(); + } + } + + @Override + public void insert(QualifiedTableName table, List records, Map primaryKeyOverrides, + CqlBatchType batchType, WriteOverrides overrides) throws QueryFailureException { + if (records == null || records.isEmpty()) { + return; + } + + final boolean useTimestampOverride = hasTimestampOverride(overrides); + BatchStatementBuilder builder = BatchStatement.builder(toDriverBatchType(batchType)); + GeneratedResult result = generateInsert(table, records.get(0).getSchema(), primaryKeyOverrides, resolveTtlSeconds(overrides, defaultTtl), useTimestampOverride); + PreparedStatement preparedStatement = cassandraSession.prepare(result.statement); + + if (getLogger().isDebugEnabled()) { + getLogger().debug("Executing CQL {} batch insert of {} record(s): {}", batchType, records.size(), result.statement.getQuery()); + } + + int recordNumber = 0; + for (Record record : records) { + recordNumber++; + Object[] values = getBindValues(table, primaryKeyOverrides, record, preparedStatement, result.keysUsed); + if (useTimestampOverride) { + setTimestampBindValue(values, preparedStatement, record, overrides); + } + + if (VALUE_LOGGING_ENABLED && getLogger().isDebugEnabled()) { + getLogger().debug("Batch insert record {}{}", recordNumber, valuesFragment(values)); + } + + builder.addStatement(preparedStatement.bind(values)); + } + + try { + cassandraSession.execute(builder.build()); + } catch (QueryExecutionException | AllNodesFailedException | DriverTimeoutException qee) { + getLogger().error("Error executing batch insert", qee); + throw new QueryFailureException(); + } + } + + @Override + public String getTransitUrl(QualifiedTableName tableName) { + final String qualifiedName = tableName.isQualified() + ? tableName.keyspace() + "." + tableName.table() + : tableName.table(); + return "cassandra://" + cassandraSession.getMetadata().getClusterName() + "." + qualifiedName; + } + + /** + * Creates an Avro schema from the given result set. The metadata (column definitions, data types, etc.) is used + * to determine a schema for Avro. + * + * @param rs The result set from which an Avro schema will be created + * @return An Avro schema corresponding to the given result set's metadata + */ + public static Schema createSchema(final ResultSet rs) { + final ColumnDefinitions columnDefinitions = rs.getColumnDefinitions(); + final int nrOfColumns = (columnDefinitions == null ? 0 : columnDefinitions.size()); + String tableName = "NiFi_Cassandra_Query_Record"; + if (nrOfColumns > 0) { + String tableNameFromMeta = columnDefinitions.get(0).getTable().toString(); + if (!StringUtils.isBlank(tableNameFromMeta)) { + tableName = tableNameFromMeta; + } + } + + final SchemaBuilder.FieldAssembler builder = SchemaBuilder.record(tableName).namespace("any.data").fields(); + if (columnDefinitions != null) { + final Map udtSchemaCache = new HashMap<>(); + for (int i = 0; i < nrOfColumns; i++) { + DataType dataType = columnDefinitions.get(i).getType(); + if (dataType == null) { + throw new IllegalArgumentException("No data type for column[" + i + "] with name " + + columnDefinitions.get(i).getName()); + } + + builder.name(columnDefinitions.get(i).getName().toString()) + .type(CassandraUdtSchemaMapper.toAvroSchema(dataType, udtSchemaCache)) + .noDefault(); + } + } + return builder.endRecord(); + } + + protected GeneratedResult generateDelete(QualifiedTableName cassandraTable, Record record, + Map primaryKeyOverrides, List deleteKeyNames) { + RecordSchema schema = record.getSchema(); + + if (deleteKeyNames == null || deleteKeyNames.isEmpty()) { + throw new IllegalArgumentException("No delete keys were specified"); + } + + // Verify every delete key is either a real record field or covered by a primary key override for this + // table, matching generateUpdate: a key resolved purely by RecordPath has no same-named record field, + // but getBindValues can still resolve its value from the override alone. + final Set overrideFieldNames = overrideFieldNamesForTable(cassandraTable, primaryKeyOverrides); + for (String deleteKey : deleteKeyNames) { + if (!schema.getFieldNames().contains(deleteKey) && !overrideFieldNames.contains(deleteKey)) { + throw new IllegalArgumentException("Delete key '" + deleteKey + "' is not present in the record schema"); + } + } + + DeleteSelection deleteSelection = cassandraTable.isQualified() + ? QueryBuilder.deleteFrom(cassandraTable.keyspace(), cassandraTable.table()) + : QueryBuilder.deleteFrom(cassandraTable.table()); + + List whereCriteria = new ArrayList<>(); + List keysUsedInOrder = new ArrayList<>(); + + for (String fieldName : deleteKeyNames) { + whereCriteria.add(Relation.column(fieldName).isEqualTo(QueryBuilder.bindMarker(fieldName))); + keysUsedInOrder.add(fieldName); + } + + return new GeneratedResult(deleteSelection.where(whereCriteria).build(), keysUsedInOrder); + } + + protected GeneratedResult generateUpdate(QualifiedTableName cassandraTable, Record record, Map primaryKeyOverrides, + List updateKeyNames, UpdateMethod updateMethod, Integer ttlSeconds, boolean includeTimestampMarker) { + RecordSchema schema = record.getSchema(); + + List keysUsedInOrder = new ArrayList<>(); + + if (updateKeyNames == null || updateKeyNames.isEmpty()) { + throw new IllegalArgumentException("No Update Keys were specified"); + } + + // Verify every update key is either a real record field or covered by a primary key override for + // this table - an update key with neither, such as a date column derived from a timestamp field via + // a format() RecordPath override, would otherwise be rejected here even though getBindValues can + // resolve its value perfectly well from the override alone. + final Set overrideFieldNames = overrideFieldNamesForTable(cassandraTable, primaryKeyOverrides); + for (String updateKey : updateKeyNames) { + if (!schema.getFieldNames().contains(updateKey) && !overrideFieldNames.contains(updateKey)) { + throw new IllegalArgumentException("Update key '" + updateKey + "' is not present in the record schema"); + } + } + + UpdateStart updateQueryStart = cassandraTable.isQualified() + ? QueryBuilder.update(cassandraTable.keyspace(), cassandraTable.table()) + : QueryBuilder.update(cassandraTable.table()); + + // Cassandra/ScyllaDB do not support a TTL or a custom write timestamp on counter columns, so both are + // only applied for SET updates. + if (ttlSeconds != null && updateMethod == UpdateMethod.SET) { + updateQueryStart = updateQueryStart.usingTtl(ttlSeconds); + } + if (includeTimestampMarker && updateMethod == UpdateMethod.SET) { + updateQueryStart = updateQueryStart.usingTimestamp(QueryBuilder.bindMarker(WRITE_TIMESTAMP_BIND_MARKER)); + } + + UpdateWithAssignments updateAssignments = null; + + List otherKeys = schema.getFieldNames().stream() + .filter(fieldName -> !updateKeyNames.contains(fieldName)) + .toList(); + + for (String fieldName : otherKeys) { + if (updateMethod == UpdateMethod.SET) { + updateAssignments = updateAssignments == null ? updateQueryStart.setColumn(fieldName, QueryBuilder.bindMarker(fieldName)) + : updateAssignments.setColumn(fieldName, QueryBuilder.bindMarker(fieldName)); + } else if (updateMethod == UpdateMethod.INCREMENT) { + updateAssignments = updateAssignments == null ? updateQueryStart.increment(fieldName, QueryBuilder.bindMarker(fieldName)) + : updateAssignments.increment(fieldName, QueryBuilder.bindMarker(fieldName)); + } else if (updateMethod == UpdateMethod.DECREMENT) { + updateAssignments = updateAssignments == null ? updateQueryStart.decrement(fieldName, QueryBuilder.bindMarker(fieldName)) + : updateAssignments.decrement(fieldName, QueryBuilder.bindMarker(fieldName)); + } else { + throw new IllegalArgumentException("Update Method '" + updateMethod + "' is not valid."); + } + + keysUsedInOrder.add(fieldName); + } + + if (updateAssignments == null) { + throw new ProcessException("No update assignment found"); + } + + Update update = null; + + for (String fieldName : updateKeyNames) { + update = update == null ? updateAssignments.whereColumn(fieldName).isEqualTo(QueryBuilder.bindMarker(fieldName)) + : update.whereColumn(fieldName).isEqualTo(QueryBuilder.bindMarker(fieldName)); + keysUsedInOrder.add(fieldName); + } + + return new GeneratedResult(update.build(), keysUsedInOrder); + } + + @Override + public void delete(QualifiedTableName cassandraTable, Record record, Map primaryKeyOverrides, + List updateKeys) throws QueryFailureException { + GeneratedResult result = generateDelete(cassandraTable, record, primaryKeyOverrides, updateKeys); + PreparedStatement preparedStatement = cassandraSession.prepare(result.statement()); + + Object[] values = getBindValues(cassandraTable, primaryKeyOverrides, record, preparedStatement, result.keysUsed()); + + if (getLogger().isDebugEnabled()) { + getLogger().debug("Executing CQL delete: {}{}", result.statement().getQuery(), valuesFragment(values)); + } + + BoundStatement deleteStatement = preparedStatement.bind(values); + + try { + cassandraSession.execute(deleteStatement); + } catch (QueryExecutionException | AllNodesFailedException | DriverTimeoutException qee) { + getLogger().error("Error executing delete", qee); + throw new QueryFailureException(); + } + } + + @Override + public void update(QualifiedTableName cassandraTable, Record record, Map primaryKeyOverrides, + List updateKeys, UpdateMethod updateMethod, WriteOverrides overrides) throws QueryFailureException { + final boolean useTimestampOverride = hasTimestampOverride(overrides) && updateMethod == UpdateMethod.SET; + GeneratedResult result = generateUpdate(cassandraTable, record, primaryKeyOverrides, updateKeys, updateMethod, resolveTtlSeconds(overrides, defaultTtl), useTimestampOverride); + PreparedStatement preparedStatement = cassandraSession.prepare(result.statement()); + + Object[] values = getBindValues(cassandraTable, primaryKeyOverrides, record, preparedStatement, result.keysUsed()); + if (useTimestampOverride) { + setTimestampBindValue(values, preparedStatement, record, overrides); + } + + if (getLogger().isDebugEnabled()) { + getLogger().debug("Executing CQL {} update: {}{}", updateMethod, result.statement().getQuery(), valuesFragment(values)); + } + + BoundStatement statement = preparedStatement.bind(values); + + try { + cassandraSession.execute(statement); + } catch (QueryExecutionException | AllNodesFailedException | DriverTimeoutException qee) { + getLogger().error("Error executing update", qee); + throw new QueryFailureException(); + } + } + + /** + * Resolves each named field to its actual bind-marker position and converts its record value for that + * position's CQL type, leaving any position not named in {@code keyNamesInOrder} for the caller to set. + */ + private Object[] getBindValues(QualifiedTableName qualifiedTableName, + Map mappings, + Record record, + PreparedStatement preparedStatement, List keyNamesInOrder) { + ColumnDefinitions variableDefinitions = preparedStatement.getVariableDefinitions(); + Object[] result = new Object[variableDefinitions.size()]; + + for (String fieldName : keyNamesInOrder) { + int index = variableDefinitions.firstIndexOf(fieldName); + + RecordPath path = getRecordPathOverride(qualifiedTableName, fieldName, mappings); + Object value; + if (path != null) { + value = evaluateOverride(record, path); + } else { + value = record.getValue(fieldName); + } + DataType cqlType = variableDefinitions.get(index).getType(); + result[index] = convertForCqlType(value, cqlType); + } + + return result; + } + + /** + * Converts each positional parameter to the Java type its bind marker's declared CQL type requires, failing + * fast if the parameter count doesn't match the statement's bind markers. + */ + private Object[] toBindValues(final PreparedStatement preparedStatement, final List parameters) { + final ColumnDefinitions bindMarkers = preparedStatement.getVariableDefinitions(); + + if (bindMarkers.size() != parameters.size()) { + throw new IllegalArgumentException(String.format( + "Query declares %d bind marker(s) but %d parameter(s) were supplied", + bindMarkers.size(), parameters.size())); + } + + final Object[] values = new Object[parameters.size()]; + + for (int index = 0; index < parameters.size(); index++) { + final DataType cqlType = bindMarkers.get(index).getType(); + final Object value = parameters.get(index); + + values[index] = cqlType.equals(DataTypes.TIMESTAMP) + ? toInstant(value) + : convertForCqlType(value, cqlType); + } + + return values; + } + + /** + * Converts a value bound to a {@code timestamp} column into the {@link Instant} the driver's own codec + * expects. Unlike the other scalar types, {@code timestamp} has no {@code Flexible*Codec} registered to + * coerce a compatible-but-non-exact value, so an ISO-8601 rendering - the natural form for a parameter + * supplied as text - would otherwise fail with the driver's opaque {@code CodecNotFoundException}. + */ + static Instant toInstant(final Object value) { + if (value == null) { + return null; + } else if (value instanceof Instant instantValue) { + return instantValue; + } else if (value instanceof java.util.Date dateValue) { + return dateValue.toInstant(); + } else if (value instanceof Number numberValue) { + return Instant.ofEpochMilli(numberValue.longValue()); + } + + final String text = value.toString().trim(); + + try { + return Instant.parse(text); + } catch (final DateTimeParseException e) { + try { + return Instant.ofEpochMilli(Long.parseLong(text)); + } catch (final NumberFormatException ignored) { + throw new IllegalArgumentException(String.format( + "Value '%s' cannot be bound to a timestamp column: expected an ISO-8601 instant such as " + + "2026-08-07T14:30:00Z, or a count of milliseconds since the epoch", text), e); + } + } + } + + private Object evaluateOverride(Record record, RecordPath path) { + RecordPathResult result = path.evaluate(record); + List valueList = result.getSelectedFields().toList(); + + if (valueList.isEmpty()) { + throw new IllegalArgumentException(String.format("%s evaluated to no values.", path.getPath())); + } else if (valueList.size() > 1) { + throw new IllegalArgumentException(String.format("%s evaluated to more than one value.", path.getPath())); + } + + return valueList.get(0).getValue(); + } + + private RecordPath getRecordPathOverride(QualifiedTableName tableName, String fieldName, + Map overrides) { + final String cleanFieldName = CqlIdentifier.fromCql(fieldName).asInternal(); + final String keyspace = tableName.isQualified() ? tableName.keyspace() : defaultKeyspace; + + Optional> identifier = overrides + .entrySet() + .stream() + .filter((e) -> e.getKey().keyspace().equals(keyspace) + && e.getKey().tableName().equals(tableName.table()) + && CqlIdentifier.fromCql(e.getKey().fieldName()).asInternal().equals(cleanFieldName)) + .findFirst(); + + return identifier.map(Map.Entry::getValue).orElse(null); + } + + /** + * Fills in the slot reserved for {@link #WRITE_TIMESTAMP_BIND_MARKER} directly in the same array used for + * the single {@link PreparedStatement#bind(Object...)} call, rather than binding it separately afterward via + * a named setter on the resulting {@link BoundStatement} - the latter looked correct but did not reliably + * take effect for UPDATE statements when verified against a real cluster. + */ + private void setTimestampBindValue(Object[] values, PreparedStatement preparedStatement, Record record, WriteOverrides overrides) { + int index = preparedStatement.getVariableDefinitions().firstIndexOf(WRITE_TIMESTAMP_BIND_MARKER); + values[index] = toEpochMicros(record.getValue(overrides.timestampField())); + } + + private Object convertForCqlType(final Object value, final DataType cqlType) { + if (value == null) { + return null; + } + + // A RECORD-typed value is normally a nested Record, but DataTypeUtils#toRecord also accepts a raw Map + // as an equally valid representation, so both are converted into a UdtValue the same way here. + if (cqlType instanceof UserDefinedType udtType + && (value instanceof Record || value instanceof Map)) { + final Function fieldValueLookup = value instanceof Record nestedRecord + ? nestedRecord::getValue + : ((Map) value)::get; + + UdtValue udtValue = udtType.newValue(); + final List fieldNames = udtType.getFieldNames().stream().map(name -> name.asInternal()).toList(); + + for (int i = 0; i < fieldNames.size(); i++) { + final String fieldName = fieldNames.get(i); + final DataType fieldCqlType = udtType.getFieldTypes().get(i); + final Object fieldValue = convertForCqlType(fieldValueLookup.apply(fieldName), fieldCqlType); + try { + // A null field is set directly against its declared CQL type rather than through a codec + // lookup. Resolving the codec from the value's runtime class cannot work when there is no + // value: the lookup becomes [ <-> java.lang.Object], which no codec satisfies, + // and that made any UDT carrying an absent or optional field unwritable. + udtValue = fieldValue == null + ? udtValue.setToNull(fieldName) + : udtValue.set(fieldName, fieldValue, (Class) fieldValue.getClass()); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + return udtValue; + } + + // An ARRAY-typed value is normally a List, but Object[] is NiFi's own canonical representation + // (DataTypeUtils#toArray always returns one), so both are accepted here for list and set columns. + if (cqlType instanceof ListType listType && (value instanceof List || value instanceof Object[])) { + final DataType elementCqlType = listType.getElementType(); + return toElementStream(value).map(element -> convertForCqlType(element, elementCqlType)).toList(); + } + + if (cqlType instanceof SetType setType && (value instanceof Set || value instanceof Object[])) { + final DataType elementCqlType = setType.getElementType(); + return toElementStream(value).map(element -> convertForCqlType(element, elementCqlType)).collect(Collectors.toSet()); + } + + if (value instanceof Map map && cqlType instanceof MapType mapType) { + final DataType keyCqlType = mapType.getKeyType(); + final DataType valueCqlType = mapType.getValueType(); + final Map converted = new HashMap<>(); + + for (final Map.Entry entry : map.entrySet()) { + converted.put(convertForCqlType(entry.getKey(), keyCqlType), convertForCqlType(entry.getValue(), valueCqlType)); + } + + return converted; + } + + if (cqlType.equals(DataTypes.UUID)) { + return DataTypeUtils.toUUID(value); + } + + // The driver's own codec already rejects a non-version-1 UUID bound to a timeuuid column, but only + // via a confusing CodecNotFoundException ("Codec not found for requested operation: [TIMEUUID <-> + // java.util.UUID]") that reads like a driver/configuration problem rather than a data problem. + // Checking here instead produces a clear, attributable error before the value ever reaches bind(). + if (cqlType.equals(DataTypes.TIMEUUID)) { + final UUID uuid = DataTypeUtils.toUUID(value); + if (uuid.version() != 1) { + throw new IllegalArgumentException(String.format( + "Value '%s' is not a valid timeuuid: version %d, but timeuuid columns require a version 1 (time-based) UUID", + uuid, uuid.version())); + } + return uuid; + } + + return value; + } + + private static Stream toElementStream(final Object value) { + return (value instanceof Object[] array) ? Arrays.stream(array) : ((Collection) value).stream(); + } + + @Override + public void update(QualifiedTableName cassandraTable, List records, Map primaryKeyOverrides, + List updateKeys, UpdateMethod updateMethod, CqlBatchType batchType, + WriteOverrides overrides) throws QueryFailureException { + if (records == null || records.isEmpty()) { + return; + } + + final boolean useTimestampOverride = hasTimestampOverride(overrides) && updateMethod == UpdateMethod.SET; + BatchStatementBuilder builder = BatchStatement.builder(toDriverBatchType(batchType)); + + GeneratedResult result = generateUpdate(cassandraTable, records.get(0), primaryKeyOverrides, updateKeys, updateMethod, resolveTtlSeconds(overrides, defaultTtl), useTimestampOverride); + PreparedStatement preparedStatement = cassandraSession.prepare(result.statement()); + + if (getLogger().isDebugEnabled()) { + getLogger().debug("Executing CQL {} batch {} update of {} record(s): {}", batchType, updateMethod, records.size(), result.statement().getQuery()); + } + + int recordNumber = 0; + for (Record record : records) { + recordNumber++; + Object[] values = getBindValues(cassandraTable, primaryKeyOverrides, record, preparedStatement, result.keysUsed()); + if (useTimestampOverride) { + setTimestampBindValue(values, preparedStatement, record, overrides); + } + + if (VALUE_LOGGING_ENABLED && getLogger().isDebugEnabled()) { + getLogger().debug("Batch update record {}{}", recordNumber, valuesFragment(values)); + } + + builder.addStatement(preparedStatement.bind(values)); + } + + try { + cassandraSession.execute(builder.build()); + } catch (QueryExecutionException | AllNodesFailedException | DriverTimeoutException qee) { + getLogger().error("Error executing batch update", qee); + throw new QueryFailureException(); + } + } + + @Override + public PrimaryKey getMetadata(QualifiedTableName table) { + cacheMetadata(table); + return tableMetadataCache.get(resolveKeyspace(table)); + } + + @Override + public String getDefaultKeyspace() { + return this.defaultKeyspace; + } + + record GeneratedResult(SimpleStatement statement, List keysUsed) { + + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/CassandraUdtSchemaMapper.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/CassandraUdtSchemaMapper.java new file mode 100644 index 000000000000..701416d80b9a --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/CassandraUdtSchemaMapper.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.CqlIdentifier; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.DataTypes; +import com.datastax.oss.driver.api.core.type.ListType; +import com.datastax.oss.driver.api.core.type.MapType; +import com.datastax.oss.driver.api.core.type.SetType; +import com.datastax.oss.driver.api.core.type.UserDefinedType; +import org.apache.avro.LogicalTypes; +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; + +import java.util.List; +import java.util.Map; + +import static com.datastax.oss.driver.api.core.type.DataTypes.ASCII; + +/** + * Recursively converts a Cassandra CQL {@link DataType} into a nullable-union Avro {@link Schema}, mapping + * User Defined Types (including nested ones and those held in a list/set/map) to nested named Avro records. + * Any CQL type with no Avro equivalent ({@code DURATION}, {@code TUPLE}, {@code VECTOR}, custom types) falls + * back to a plain string so one exotic column never fails an entire query. + */ +public final class CassandraUdtSchemaMapper { + + private CassandraUdtSchemaMapper() { + } + + /** + * Converts the given CQL type into a nullable Avro union schema. + * + * @param cqlType the CQL type to convert + * @param udtSchemaCache UDT schemas already built in this conversion, keyed by "keyspace.typeName", so a + * UDT referenced more than once reuses one {@link Schema} instance (Avro rejects + * duplicate record names). Pass a fresh, empty map per top-level call. + */ + public static Schema toAvroSchema(final DataType cqlType, final Map udtSchemaCache) { + if (cqlType instanceof UserDefinedType udtType) { + return nullableUnion(toUdtRecordSchema(udtType, udtSchemaCache)); + } + + if (cqlType instanceof ListType listType) { + return nullableUnion(SchemaBuilder.array().items(toAvroSchema(listType.getElementType(), udtSchemaCache))); + } + + if (cqlType instanceof SetType setType) { + return nullableUnion(SchemaBuilder.array().items(toAvroSchema(setType.getElementType(), udtSchemaCache))); + } + + if (cqlType instanceof MapType mapType) { + return nullableUnion(SchemaBuilder.map().values(toAvroSchema(mapType.getValueType(), udtSchemaCache))); + } + + return nullableUnion(toPrimitiveAvroSchema(cqlType)); + } + + private static Schema toUdtRecordSchema(final UserDefinedType udtType, final Map udtSchemaCache) { + final String namespace = udtType.getKeyspace() == null ? "cassandra" : udtType.getKeyspace().asInternal(); + final String schemaKey = namespace + "." + udtType.getName().asInternal(); + + final Schema cached = udtSchemaCache.get(schemaKey); + if (cached != null) { + return cached; + } + + SchemaBuilder.FieldAssembler fields = SchemaBuilder.record(udtType.getName().asInternal()) + .namespace(namespace) + .fields(); + + final List fieldNames = udtType.getFieldNames(); + final List fieldTypes = udtType.getFieldTypes(); + + for (int i = 0; i < fieldNames.size(); i++) { + fields = fields.name(fieldNames.get(i).asInternal()).type(toAvroSchema(fieldTypes.get(i), udtSchemaCache)).noDefault(); + } + + final Schema schema = fields.endRecord(); + udtSchemaCache.put(schemaKey, schema); + return schema; + } + + private static Schema nullableUnion(final Schema schema) { + return SchemaBuilder.builder().unionOf().nullBuilder().endNull().and().type(schema).endUnion(); + } + + private static Schema toPrimitiveAvroSchema(final DataType cqlType) { + final SchemaBuilder.TypeBuilder typeBuilder = SchemaBuilder.builder(); + + if (cqlType.equals(ASCII) || cqlType.equals(DataTypes.TEXT)) { + return typeBuilder.stringType(); + } + + // Declared with the Avro `uuid` logical type, so AvroTypeUtil#determineDataType resolves the record + // field to the native UUID type rather than STRING - matching the java.util.UUID the driver's own + // codec decodes the column into, with no value-side conversion needed on either side. + if (cqlType.equals(DataTypes.UUID) || cqlType.equals(DataTypes.TIMEUUID)) { + final Schema schema = Schema.create(Schema.Type.STRING); + LogicalTypes.uuid().addToSchema(schema); + return schema; + } + + // TIMESTAMP/DATE/TIME, DECIMAL/VARINT and INET are all declared as a plain Avro string, and the + // driver's own decoded value (Instant/LocalDate/LocalTime, BigDecimal/BigInteger, InetAddress) is + // placed into the record unchanged. The declared type and the runtime class differ, but that is a + // supported combination rather than a defect: RecordFieldType.STRING is a *widening* type over + // DATE/TIME/TIMESTAMP/DECIMAL/BIGINT among others, so DataTypeUtils accepts each of these values for + // a string field and coerces them losslessly (a nanosecond LocalTime renders in full, for instance) + // if and when a downstream writer asks for a string. + // + // Declaring the native NiFi types instead is not an option here: + // - TIMESTAMP/DATE/TIME are backed by java.sql.*, not java.time.*, so pairing them with the + // driver's values makes DataTypeUtils#convertType throw for TIMESTAMP and silently truncate a + // nanosecond TIME to whole seconds. Converting the values to java.sql.* to suit instead loses + // that same sub-second precision, since CQL `time` is nanosecond-resolution and java.sql.Time is + // not. + // - DECIMAL would need Avro's fixed precision/scale, which CQL's arbitrary-precision `decimal` + // cannot supply up front, and a native BIGINT is unreachable through the Avro round trip at all + // (AvroTypeUtil maps record BIGINT to Avro string, which resolves back to record STRING). + // - INET has no native NiFi record type in the first place. + if (cqlType.equals(DataTypes.TIMESTAMP) + || cqlType.equals(DataTypes.DATE) + || cqlType.equals(DataTypes.TIME) + || cqlType.equals(DataTypes.INET) + || cqlType.equals(DataTypes.VARINT) + || cqlType.equals(DataTypes.DECIMAL)) { + return typeBuilder.stringType(); + } + + if (cqlType.equals(DataTypes.BOOLEAN)) { + return typeBuilder.booleanType(); + } + + if (cqlType.equals(DataTypes.INT) + // Avro has no byte/short primitive, so these widen to Avro's int + || cqlType.equals(DataTypes.TINYINT) + || cqlType.equals(DataTypes.SMALLINT)) { + return typeBuilder.intType(); + } + + if (cqlType.equals(DataTypes.BIGINT) || cqlType.equals(DataTypes.COUNTER)) { + return typeBuilder.longType(); + } + + if (cqlType.equals(DataTypes.FLOAT)) { + return typeBuilder.floatType(); + } + + if (cqlType.equals(DataTypes.DOUBLE)) { + return typeBuilder.doubleType(); + } + + if (cqlType.equals(DataTypes.BLOB)) { + return typeBuilder.bytesType(); + } + + // A CQL type with no case above - DURATION, TUPLE, VECTOR, and any future or custom type - is declared + // as STRING rather than failing schema generation. This used to throw IllegalArgumentException here, + // which failed the entire query over one exotic column; CassandraCQLExecutionService#toRecordValue + // converts the value to match via toString(), the same fallback DECIMAL/VARINT/INET already use. + return typeBuilder.stringType(); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/CharacterCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/CharacterCodec.java new file mode 100644 index 000000000000..d35019eebf2a --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/CharacterCodec.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; + +import java.nio.ByteBuffer; + +/** + * A CQL {@code text} codec that binds a {@link Character} (what {@code DataTypeUtils} produces for a + * CHAR-typed record field) as a one-character string, since Cassandra has no single-character type. + */ +public class CharacterCodec implements TypeCodec { + private final TypeCodec stringCodec; + + public CharacterCodec() { + this.stringCodec = TypeCodecs.TEXT; + } + + @Override + public GenericType getJavaType() { + return GenericType.of(Character.class); + } + + @Override + public DataType getCqlType() { + return stringCodec.getCqlType(); // maps to CQL `text` + } + + @Override + public ByteBuffer encode(Character value, ProtocolVersion protocolVersion) { + return value == null ? null : stringCodec.encode(value.toString(), protocolVersion); + } + + @Override + public Character decode(ByteBuffer bytes, ProtocolVersion protocolVersion) { + final String string = stringCodec.decode(bytes, protocolVersion); + return (string == null || string.isEmpty()) ? null : string.charAt(0); + } + + @Override + public String format(Character value) { + return value == null ? "NULL" : stringCodec.format(value.toString()); + } + + @Override + public Character parse(String value) { + final String string = stringCodec.parse(value); + return (string == null || string.isEmpty()) ? null : string.charAt(0); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleBigIntCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleBigIntCodec.java new file mode 100644 index 000000000000..83e4e4f3b824 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleBigIntCodec.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import org.apache.nifi.serialization.record.util.DataTypeUtils; + +import java.nio.ByteBuffer; + +/** + * A fallback CQL {@code bigint} codec that accepts any long-compatible value (a numeric String, a + * differently-sized {@link Number}), not just an exact {@link Long}. Declared against {@link Object} so + * the registry treats it as a fallback behind the built-in codec rather than rejecting it as a duplicate. + */ +public class FlexibleBigIntCodec implements TypeCodec { + private final TypeCodec bigIntCodec; + + public FlexibleBigIntCodec() { + this.bigIntCodec = TypeCodecs.BIGINT; + } + + @Override + public GenericType getJavaType() { + return GenericType.of(Object.class); + } + + @Override + public DataType getCqlType() { + return bigIntCodec.getCqlType(); // maps to CQL `bigint` + } + + @Override + public ByteBuffer encode(Object value, ProtocolVersion protocolVersion) { + return value == null ? null : bigIntCodec.encode(DataTypeUtils.toLong(value, getCqlType().toString()), protocolVersion); + } + + @Override + public Object decode(ByteBuffer bytes, ProtocolVersion protocolVersion) { + return bigIntCodec.decode(bytes, protocolVersion); + } + + @Override + public String format(Object value) { + return value == null ? "NULL" : bigIntCodec.format(DataTypeUtils.toLong(value, getCqlType().toString())); + } + + @Override + public Object parse(String value) { + return bigIntCodec.parse(value); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleBooleanCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleBooleanCodec.java new file mode 100644 index 000000000000..f6b323f484a3 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleBooleanCodec.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import org.apache.nifi.serialization.record.util.DataTypeUtils; + +import java.nio.ByteBuffer; + +/** + * A fallback CQL {@code boolean} codec that accepts any boolean-compatible value (such as a + * {@code "true"}/{@code "false"} String), not just an exact {@link Boolean}. Declared against + * {@link Object} so the registry treats it as a fallback behind the built-in codec rather than + * rejecting it as a duplicate. + */ +public class FlexibleBooleanCodec implements TypeCodec { + private final TypeCodec booleanCodec; + + public FlexibleBooleanCodec() { + this.booleanCodec = TypeCodecs.BOOLEAN; + } + + @Override + public GenericType getJavaType() { + return GenericType.of(Object.class); + } + + @Override + public DataType getCqlType() { + return booleanCodec.getCqlType(); // maps to CQL `boolean` + } + + @Override + public ByteBuffer encode(Object value, ProtocolVersion protocolVersion) { + return value == null ? null : booleanCodec.encode(DataTypeUtils.toBoolean(value, getCqlType().toString()), protocolVersion); + } + + @Override + public Object decode(ByteBuffer bytes, ProtocolVersion protocolVersion) { + return booleanCodec.decode(bytes, protocolVersion); + } + + @Override + public String format(Object value) { + return value == null ? "NULL" : booleanCodec.format(DataTypeUtils.toBoolean(value, getCqlType().toString())); + } + + @Override + public Object parse(String value) { + return booleanCodec.parse(value); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleCounterCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleCounterCodec.java new file mode 100644 index 000000000000..440b88c81916 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleCounterCodec.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.DataTypes; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; + +import java.nio.ByteBuffer; + +/** + * A CQL {@code counter} codec that binds any {@link Number} by narrowing it to a {@code long} and + * delegating to the built-in {@code bigint} codec, since a counter is stored as a 64-bit integer. + */ +public class FlexibleCounterCodec implements TypeCodec { + + private final TypeCodec inner = TypeCodecs.BIGINT; + + @Override + public GenericType getJavaType() { + return GenericType.of(Number.class); + } + + @Override + public DataType getCqlType() { + return DataTypes.COUNTER; + } + + @Override + public ByteBuffer encode(Number value, ProtocolVersion protocolVersion) { + if (value == null) { + return null; + } + return inner.encode(value.longValue(), protocolVersion); + } + + @Override + public Number decode(ByteBuffer bytes, ProtocolVersion protocolVersion) { + Long val = inner.decode(bytes, protocolVersion); + return val; // returns as Long, but you can cast if needed + } + + @Override + public String format(Number value) { + return inner.format(value == null ? null : value.longValue()); + } + + @Override + public Number parse(String value) { + return inner.parse(value); + } +} + diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleDateCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleDateCodec.java new file mode 100644 index 000000000000..494361cf6f0d --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleDateCodec.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.serialization.record.util.DataTypeUtils; + +import java.nio.ByteBuffer; +import java.sql.Date; +import java.time.LocalDate; +import java.util.Optional; + +/** + * A fallback CQL {@code date} codec that accepts any date-compatible value {@code DataTypeUtils} recognizes, + * including a {@link String} parsed with the fixed ISO {@code yyyy-MM-dd} pattern (what a RecordPath + * {@code format()} primary key override produces). Declared against {@link Object} so the registry treats it + * as a fallback behind the built-in {@link LocalDate} codec rather than rejecting it as a duplicate. + */ +public class FlexibleDateCodec implements TypeCodec { + private static final String DATE_PATTERN = "yyyy-MM-dd"; + + private final TypeCodec dateCodec; + + public FlexibleDateCodec() { + this.dateCodec = TypeCodecs.DATE; + } + + @Override + public GenericType getJavaType() { + return GenericType.of(Object.class); + } + + @Override + public DataType getCqlType() { + return dateCodec.getCqlType(); // maps to CQL `date` + } + + @Override + public ByteBuffer encode(Object value, ProtocolVersion protocolVersion) { + return value == null ? null : dateCodec.encode(toLocalDate(value), protocolVersion); + } + + @Override + public Object decode(ByteBuffer bytes, ProtocolVersion protocolVersion) { + return dateCodec.decode(bytes, protocolVersion); + } + + @Override + public String format(Object value) { + return value == null ? "NULL" : dateCodec.format(toLocalDate(value)); + } + + @Override + public Object parse(String value) { + return dateCodec.parse(value); + } + + private static LocalDate toLocalDate(final Object value) { + final Object converted = DataTypeUtils.convertType(value, RecordFieldType.DATE.getDataType(), + Optional.of(DATE_PATTERN), Optional.empty(), Optional.empty(), "value"); + return ((Date) converted).toLocalDate(); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleDoubleCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleDoubleCodec.java new file mode 100644 index 000000000000..c7e8cf1cd42e --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleDoubleCodec.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import org.apache.nifi.serialization.record.util.DataTypeUtils; + +import java.nio.ByteBuffer; + +/** + * A fallback CQL {@code double} codec that accepts any double-compatible value (a numeric String, a + * differently-sized {@link Number}), not just an exact {@link Double}. Declared against {@link Object} so + * the registry treats it as a fallback behind the built-in codec rather than rejecting it as a duplicate. + */ +public class FlexibleDoubleCodec implements TypeCodec { + private final TypeCodec doubleCodec; + + public FlexibleDoubleCodec() { + this.doubleCodec = TypeCodecs.DOUBLE; + } + + @Override + public GenericType getJavaType() { + return GenericType.of(Object.class); + } + + @Override + public DataType getCqlType() { + return doubleCodec.getCqlType(); // maps to CQL `double` + } + + @Override + public ByteBuffer encode(Object value, ProtocolVersion protocolVersion) { + return value == null ? null : doubleCodec.encode(DataTypeUtils.toDouble(value, getCqlType().toString()), protocolVersion); + } + + @Override + public Object decode(ByteBuffer bytes, ProtocolVersion protocolVersion) { + return doubleCodec.decode(bytes, protocolVersion); + } + + @Override + public String format(Object value) { + return value == null ? "NULL" : doubleCodec.format(DataTypeUtils.toDouble(value, getCqlType().toString())); + } + + @Override + public Object parse(String value) { + return doubleCodec.parse(value); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleFloatCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleFloatCodec.java new file mode 100644 index 000000000000..e793337c8395 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleFloatCodec.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import org.apache.nifi.serialization.record.util.DataTypeUtils; + +import java.nio.ByteBuffer; + +/** + * A fallback CQL {@code float} codec that accepts any float-compatible value (a numeric String, a + * differently-sized {@link Number}), not just an exact {@link Float}. Declared against {@link Object} so + * the registry treats it as a fallback behind the built-in codec rather than rejecting it as a duplicate. + */ +public class FlexibleFloatCodec implements TypeCodec { + private final TypeCodec floatCodec; + + public FlexibleFloatCodec() { + this.floatCodec = TypeCodecs.FLOAT; + } + + @Override + public GenericType getJavaType() { + return GenericType.of(Object.class); + } + + @Override + public DataType getCqlType() { + return floatCodec.getCqlType(); // maps to CQL `float` + } + + @Override + public ByteBuffer encode(Object value, ProtocolVersion protocolVersion) { + return value == null ? null : floatCodec.encode(DataTypeUtils.toFloat(value, getCqlType().toString()), protocolVersion); + } + + @Override + public Object decode(ByteBuffer bytes, ProtocolVersion protocolVersion) { + return floatCodec.decode(bytes, protocolVersion); + } + + @Override + public String format(Object value) { + return value == null ? "NULL" : floatCodec.format(DataTypeUtils.toFloat(value, getCqlType().toString())); + } + + @Override + public Object parse(String value) { + return floatCodec.parse(value); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleIntCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleIntCodec.java new file mode 100644 index 000000000000..e77667b40244 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleIntCodec.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import org.apache.nifi.serialization.record.util.DataTypeUtils; + +import java.nio.ByteBuffer; + +/** + * A fallback CQL {@code int} codec that accepts any int-compatible value (a numeric String, a + * differently-sized {@link Number}), not just an exact {@link Integer}. Declared against {@link Object} so + * the registry treats it as a fallback behind the built-in codec rather than rejecting it as a duplicate. + */ +public class FlexibleIntCodec implements TypeCodec { + private final TypeCodec intCodec; + + public FlexibleIntCodec() { + this.intCodec = TypeCodecs.INT; + } + + @Override + public GenericType getJavaType() { + return GenericType.of(Object.class); + } + + @Override + public DataType getCqlType() { + return intCodec.getCqlType(); // maps to CQL `int` + } + + @Override + public ByteBuffer encode(Object value, ProtocolVersion protocolVersion) { + return value == null ? null : intCodec.encode(DataTypeUtils.toInteger(value, getCqlType().toString()), protocolVersion); + } + + @Override + public Object decode(ByteBuffer bytes, ProtocolVersion protocolVersion) { + return intCodec.decode(bytes, protocolVersion); + } + + @Override + public String format(Object value) { + return value == null ? "NULL" : intCodec.format(DataTypeUtils.toInteger(value, getCqlType().toString())); + } + + @Override + public Object parse(String value) { + return intCodec.parse(value); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleSmallIntCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleSmallIntCodec.java new file mode 100644 index 000000000000..76d7ffcde9e7 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleSmallIntCodec.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import org.apache.nifi.serialization.record.util.DataTypeUtils; + +import java.nio.ByteBuffer; + +/** + * A fallback CQL {@code smallint} codec that accepts any short-compatible value (a numeric String, a + * differently-sized {@link Number}), not just an exact {@link Short}. Declared against {@link Object} so + * the registry treats it as a fallback behind the built-in codec rather than rejecting it as a duplicate. + */ +public class FlexibleSmallIntCodec implements TypeCodec { + private final TypeCodec smallIntCodec; + + public FlexibleSmallIntCodec() { + this.smallIntCodec = TypeCodecs.SMALLINT; + } + + @Override + public GenericType getJavaType() { + return GenericType.of(Object.class); + } + + @Override + public DataType getCqlType() { + return smallIntCodec.getCqlType(); // maps to CQL `smallint` + } + + @Override + public ByteBuffer encode(Object value, ProtocolVersion protocolVersion) { + return value == null ? null : smallIntCodec.encode(DataTypeUtils.toShort(value, getCqlType().toString()), protocolVersion); + } + + @Override + public Object decode(ByteBuffer bytes, ProtocolVersion protocolVersion) { + return smallIntCodec.decode(bytes, protocolVersion); + } + + @Override + public String format(Object value) { + return value == null ? "NULL" : smallIntCodec.format(DataTypeUtils.toShort(value, getCqlType().toString())); + } + + @Override + public Object parse(String value) { + return smallIntCodec.parse(value); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleTimeCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleTimeCodec.java new file mode 100644 index 000000000000..ee3e6ac0bc79 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleTimeCodec.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import org.apache.nifi.serialization.record.RecordFieldType; +import org.apache.nifi.serialization.record.util.DataTypeUtils; + +import java.nio.ByteBuffer; +import java.sql.Time; +import java.time.LocalTime; +import java.util.Optional; + +/** + * A fallback CQL {@code time} codec that accepts any time-compatible value {@code DataTypeUtils} recognizes, + * including a {@link String} parsed with the fixed {@code HH:mm:ss} pattern (what a RecordPath {@code format()} + * primary key override produces). Declared against {@link Object} so the registry treats it as a fallback + * behind the built-in {@link LocalTime} codec rather than rejecting it as a duplicate. + */ +public class FlexibleTimeCodec implements TypeCodec { + private static final String TIME_PATTERN = "HH:mm:ss"; + + private final TypeCodec timeCodec; + + public FlexibleTimeCodec() { + this.timeCodec = TypeCodecs.TIME; + } + + @Override + public GenericType getJavaType() { + return GenericType.of(Object.class); + } + + @Override + public DataType getCqlType() { + return timeCodec.getCqlType(); // maps to CQL `time` + } + + @Override + public ByteBuffer encode(Object value, ProtocolVersion protocolVersion) { + return value == null ? null : timeCodec.encode(toLocalTime(value), protocolVersion); + } + + @Override + public Object decode(ByteBuffer bytes, ProtocolVersion protocolVersion) { + return timeCodec.decode(bytes, protocolVersion); + } + + @Override + public String format(Object value) { + return value == null ? "NULL" : timeCodec.format(toLocalTime(value)); + } + + @Override + public Object parse(String value) { + return timeCodec.parse(value); + } + + private static LocalTime toLocalTime(final Object value) { + final Object converted = DataTypeUtils.convertType(value, RecordFieldType.TIME.getDataType(), + Optional.empty(), Optional.of(TIME_PATTERN), Optional.empty(), "value"); + return ((Time) converted).toLocalTime(); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleTinyIntCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleTinyIntCodec.java new file mode 100644 index 000000000000..2b15f1826de4 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/FlexibleTinyIntCodec.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; +import org.apache.nifi.serialization.record.util.DataTypeUtils; + +import java.nio.ByteBuffer; + +/** + * A fallback CQL {@code tinyint} codec that accepts any byte-compatible value (a numeric String, a + * differently-sized {@link Number}), not just an exact {@link Byte}. Declared against {@link Object} so + * the registry treats it as a fallback behind the built-in codec rather than rejecting it as a duplicate. + */ +public class FlexibleTinyIntCodec implements TypeCodec { + private final TypeCodec tinyIntCodec; + + public FlexibleTinyIntCodec() { + this.tinyIntCodec = TypeCodecs.TINYINT; + } + + @Override + public GenericType getJavaType() { + return GenericType.of(Object.class); + } + + @Override + public DataType getCqlType() { + return tinyIntCodec.getCqlType(); // maps to CQL `tinyint` + } + + @Override + public ByteBuffer encode(Object value, ProtocolVersion protocolVersion) { + return value == null ? null : tinyIntCodec.encode(DataTypeUtils.toByte(value, getCqlType().toString()), protocolVersion); + } + + @Override + public Object decode(ByteBuffer bytes, ProtocolVersion protocolVersion) { + return tinyIntCodec.decode(bytes, protocolVersion); + } + + @Override + public String format(Object value) { + return value == null ? "NULL" : tinyIntCodec.format(DataTypeUtils.toByte(value, getCqlType().toString())); + } + + @Override + public Object parse(String value) { + return tinyIntCodec.parse(value); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/JavaSQLDateCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/JavaSQLDateCodec.java new file mode 100644 index 000000000000..d13a4b7b2a72 --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/JavaSQLDateCodec.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; + +import java.nio.ByteBuffer; +import java.sql.Date; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; + +/** + * A CQL {@code date} codec that binds {@code java.sql.Date} (what {@code DataTypeUtils} produces for a + * DATE-typed record field) by converting to and from {@link LocalDate} in the system default time zone. + */ +public class JavaSQLDateCodec implements TypeCodec { + private final TypeCodec localDateCodec; + + public JavaSQLDateCodec() { + this.localDateCodec = TypeCodecs.DATE; + } + + @Override + public GenericType getJavaType() { + return GenericType.of(Date.class); + } + + @Override + public DataType getCqlType() { + return localDateCodec.getCqlType(); // maps to CQL `date` + } + + @Override + public ByteBuffer encode(Date value, ProtocolVersion protocolVersion) { + return value == null ? null : localDateCodec.encode(toLocalDate(value), protocolVersion); + } + + @Override + public Date decode(ByteBuffer bytes, ProtocolVersion protocolVersion) { + final LocalDate localDate = localDateCodec.decode(bytes, protocolVersion); + return localDate == null ? null : toSqlDate(localDate); + } + + @Override + public String format(Date value) { + return value == null ? "NULL" : localDateCodec.format(toLocalDate(value)); + } + + @Override + public Date parse(String value) { + final LocalDate localDate = localDateCodec.parse(value); + return localDate == null ? null : toSqlDate(localDate); + } + + private static LocalDate toLocalDate(final Date value) { + return LocalDate.ofInstant(Instant.ofEpochMilli(value.getTime()), ZoneId.systemDefault()); + } + + private static Date toSqlDate(final LocalDate localDate) { + final long epochMillis = localDate.atStartOfDay(ZoneId.systemDefault()).toInstant().toEpochMilli(); + return new Date(epochMillis); + } +} diff --git a/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/JavaSQLTimeCodec.java b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/JavaSQLTimeCodec.java new file mode 100644 index 000000000000..62b47ec204ff --- /dev/null +++ b/nifi-extension-bundles/nifi-cql-bundle/nifi-cql-session-provider-shared/src/main/java/org/apache/nifi/service/cassandra/mapping/JavaSQLTimeCodec.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.nifi.service.cassandra.mapping; + +import com.datastax.oss.driver.api.core.ProtocolVersion; +import com.datastax.oss.driver.api.core.type.DataType; +import com.datastax.oss.driver.api.core.type.codec.TypeCodec; +import com.datastax.oss.driver.api.core.type.codec.TypeCodecs; +import com.datastax.oss.driver.api.core.type.reflect.GenericType; + +import java.nio.ByteBuffer; +import java.sql.Time; +import java.time.LocalDate; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; + +/** + * A CQL {@code time} codec that binds {@code java.sql.Time} (what {@code DataTypeUtils} produces for a + * TIME-typed record field) by converting to and from {@link LocalTime} in the system default time zone. + */ +public class JavaSQLTimeCodec implements TypeCodec