From da9917f5a10d98e90c4c0e78e408c16ce9ff3482 Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Tue, 8 Sep 2026 18:31:44 -0500 Subject: [PATCH 1/8] chore: Yarn upgrade dependencies --- package.json | 1 + yarn.lock | 1381 ++++++++++++++++++++++++++------------------------ 2 files changed, 712 insertions(+), 670 deletions(-) diff --git a/package.json b/package.json index e2a575a6b4..cabb194c73 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "dependencies": { "@iarna/toml": "^2.2.5", "axios": "^1.18.0", + "dompurify": ">=3.3.2", "glob": "^13.0.6", "gray-matter": "^4.0.3", "jquery": "^3.7.1", diff --git a/yarn.lock b/yarn.lock index 133e9ece66..7dd95597e1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,43 +2,66 @@ # yarn lockfile v1 -"@antfu/install-pkg@^1.1.0": - version "1.1.0" - resolved "https://registry.yarnpkg.com/@antfu/install-pkg/-/install-pkg-1.1.0.tgz#78fa036be1a6081b5a77a5cf59f50c7752b6ba26" - integrity sha512-MGQsmw10ZyI+EJo45CdSER4zEb+p31LpDAFp2Z3gkSd1yqVZGi0Ebx++YTEMonJy4oChEMLsxZ64j8FH6sSqtQ== +"@antfu/install-pkg@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@antfu/install-pkg/-/install-pkg-2.0.1.tgz#bdbc3d228e6f645c583ed93545fcdd5f5a2f0ca2" + integrity sha512-iCKVQcIC0e3oDxEfs3SHQGW+ovhBMZmS1TE+bTk50rVyMCBmCfClv7Qi3HQKlumYwvjb/iIMeWCW2i67q6kFfQ== dependencies: - package-manager-detector "^1.3.0" - tinyexec "^1.0.1" + package-manager-detector "^1.7.0" + tinyexec "^1.2.4" "@babel/code-frame@^7.0.0": - version "7.29.0" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.29.0.tgz#7cd7a59f15b3cc0dcd803038f7792712a7d0b15c" - integrity sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw== + version "7.29.7" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.29.7.tgz#f2fbbfea87c44a21590ec515b778b2c26d8866e7" + integrity sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw== dependencies: - "@babel/helper-validator-identifier" "^7.28.5" + "@babel/helper-validator-identifier" "^7.29.7" js-tokens "^4.0.0" picocolors "^1.1.1" -"@babel/helper-validator-identifier@^7.28.5": - version "7.28.5" - resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz#010b6938fab7cb7df74aa2bbc06aa503b8fe5fb4" - integrity sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q== +"@babel/helper-validator-identifier@^7.29.7": + version "7.29.7" + resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz#bd87084ced0c796ec46bda492de6e83d29e89fc2" + integrity sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg== "@braintree/sanitize-url@^7.1.2": version "7.1.2" resolved "https://registry.yarnpkg.com/@braintree/sanitize-url/-/sanitize-url-7.1.2.tgz#ca2035b0fefe956a8676ff0c69af73e605fcd81f" integrity sha512-jigsZK+sMF/cuiB7sERuo9V7N9jx+dhmHHnQyDSVdpZwVutaBu7WvNYqMDLSgFgfB30n452TP3vjDAvFC973mA== +"@cacheable/memory@^2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@cacheable/memory/-/memory-2.2.0.tgz#72aeb8b051f6d597d10ac8595b94ad8302bd2239" + integrity sha512-CTLKqLItRCEixEAewD3/j9DB3/o96gpTPD4eJ1v+DGOlxZRZncRQkGYqqnAGCscYd6RNeXfGeiuCphsPtqyIfQ== + dependencies: + "@cacheable/utils" "^2.5.0" + "@keyv/bigmap" "^1.3.1" + hookified "^1.15.1" + keyv "^5.6.0" + +"@cacheable/utils@^2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@cacheable/utils/-/utils-2.5.0.tgz#534c91113aa48fe43baedb169550b6ee070ef303" + integrity sha512-buipgOVDkkPXNR5+xBpDw7Zk2n1EvU7qBJCNUcL7rhQ//kfpOXPAvQ511Os0vpLYJ1pZnvudNytkQt2hst3wqA== + dependencies: + hashery "^1.5.1" + keyv "^5.6.0" + "@chevrotain/types@~11.1.2": version "11.1.2" resolved "https://registry.yarnpkg.com/@chevrotain/types/-/types-11.1.2.tgz#e83a1a2704f0c5e49e7592b214031a0f4a34d7e5" integrity sha512-U+HFai5+zmJCkK86QsaJtoITlboZHBqrVketcO2ROv865xfCMSFpELQoz1GkX5GzME8pTa+3kbKrZHQtI0gdbw== -"@colors/colors@1.6.0", "@colors/colors@^1.6.0": +"@colors/colors@1.6.0": version "1.6.0" resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.6.0.tgz#ec6cd237440700bc23ca23087f513c75508958b0" integrity sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA== +"@colors/colors@^1.6.0": + version "1.6.1" + resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.6.1.tgz#8e2827259ada76965b3a051c18a5927fbdac38c2" + integrity sha512-dTmUJzXSuayBK+hZydEaXd2mhx61qWQwkwaBBY6LyEOVx/L9aQU5ac8eFNEsd9nrD1+zb9zvDCphLSe8g1F4Qw== + "@cypress/request@^4.0.0": version "4.0.1" resolved "https://registry.yarnpkg.com/@cypress/request/-/request-4.0.1.tgz#c3f0eec41834e7590d21cd740a4c6f7e0326368d" @@ -71,9 +94,9 @@ lodash.once "^4.1.1" "@dabh/diagnostics@^2.0.8": - version "2.0.8" - resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.8.tgz#ead97e72ca312cf0e6dd7af0d300b58993a31a5e" - integrity sha512-R4MSXTVnuMzGD7bzHdW2ZhhdPC/igELENcq5IjEverBvq5hn1SXCWcsi6eSsdWP0/Ur+SItRRjAktmdoX/8R/Q== + version "2.0.9" + resolved "https://registry.yarnpkg.com/@dabh/diagnostics/-/diagnostics-2.0.9.tgz#0de9e8718e23ab60d1fb1ce0c004079f14f5ad36" + integrity sha512-R6siwR65Hm+3yfgP7o8DKhNvputQAwfoz9zTc3kyDudnomj2/BcLmD+uGQQPICjuFUp8ounPBU+jmKsocwVVAg== dependencies: "@so-ric/colorspace" "^1.1.6" enabled "2.0.x" @@ -91,9 +114,9 @@ jsdoc-type-pratt-parser "~4.1.0" "@eslint-community/eslint-utils@^4.8.0", "@eslint-community/eslint-utils@^4.9.1": - version "4.9.1" - resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.9.1.tgz#4e90af67bc51ddee6cdef5284edf572ec376b595" - integrity sha512-phrYmNiYppR7znFEdqgfWHXR6NCkZEK7hwWDHZUjit/2/U0r6XvkDl0SYnoM51Hq7FhCGdLDT6zxCCOY1hexsQ== + version "4.10.1" + resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz#8911bd72b2c3640a543609e0400b8c4d2e7e7cb6" + integrity sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg== dependencies: eslint-visitor-keys "^3.4.3" @@ -111,10 +134,10 @@ debug "^4.3.1" minimatch "^10.2.4" -"@eslint/config-helpers@^0.6.0": - version "0.6.0" - resolved "https://registry.yarnpkg.com/@eslint/config-helpers/-/config-helpers-0.6.0.tgz#ef9a36881d39dfd5dbeac22b0da997fabfb08b03" - integrity sha512-ii6Bw9jJ2zi2cWA2Z+9/QZ/+3DX6kwaV5Q986D/CdP3Lap3w/pgQZ373FV7byY/i7L4IRH/G43I5dz1ClsCbpA== +"@eslint/config-helpers@^0.7.0": + version "0.7.0" + resolved "https://registry.yarnpkg.com/@eslint/config-helpers/-/config-helpers-0.7.0.tgz#09ee4aa07b73f059ec2d4c74bf4b2ff02b322377" + integrity sha512-DObd/KKUsU+FaFv4PLxSRenpXfQWmPXXP3pPZ6/K1PCrMu2vQpMDMuQe/BqYeoLcz8ro0bVDF1RxOJgfVEdhUw== dependencies: "@eslint/core" "^1.2.1" @@ -135,10 +158,10 @@ resolved "https://registry.yarnpkg.com/@eslint/object-schema/-/object-schema-3.0.5.tgz#88e9bf4d11d2b19c082e78ebe7ce88724a5eb091" integrity sha512-vqTaUEgxzm+YDSdElad6PiRoX4t8VGDjCtt05zn4nU810UIx/uNEV7/lZJ6KwFThKZOzOxzXy48da+No7HZaMw== -"@eslint/plugin-kit@^0.7.2": - version "0.7.2" - resolved "https://registry.yarnpkg.com/@eslint/plugin-kit/-/plugin-kit-0.7.2.tgz#4b0962f3f2c7ce8bc98b3ecfe34525c09d2cb729" - integrity sha512-+CNAzxglkrpNf/kKywqQfk74QjtceuOE7Qm+AF8miRvPF/wmmK5+OJOgVh3AVTT3RP2mH3+FOaxlE5v72owk0A== +"@eslint/plugin-kit@^0.7.3": + version "0.7.3" + resolved "https://registry.yarnpkg.com/@eslint/plugin-kit/-/plugin-kit-0.7.3.tgz#cc7268cc36405b331ef92db1bc37971f21d66fe1" + integrity sha512-IkO+/KEUvwbVpiURZg+P7zF74z5Jxe0UgJxVni+RtoHQ6IZieXaO02kmadomap/q+l6bc/jdPGGqTjhuZnuz1Q== dependencies: "@eslint/core" "^1.2.1" levn "^0.4.1" @@ -190,11 +213,11 @@ integrity sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg== "@iconify/utils@^3.0.2": - version "3.1.3" - resolved "https://registry.yarnpkg.com/@iconify/utils/-/utils-3.1.3.tgz#71efd68f9ed2ea3c91fd3a01c0032f70a87027b7" - integrity sha512-LPKOXPn/zV+zis1oOfGWogaXVpqUybF3ZS6SCZIsz8vg0ivVp9+fVqyYB7xq0aiST/VhUQYGO1qo6uoYSiEJqw== + version "3.1.7" + resolved "https://registry.yarnpkg.com/@iconify/utils/-/utils-3.1.7.tgz#fff11c528490b119a250ace3cf35a0a78e182317" + integrity sha512-JZHlwdID+dy+lTgbYC8NEC4zeugqeYsc6jewvzb4c58kHauJn+X7rNwQjxz5p2qSjqaEeQoLkCIQ9v/H4PK0/w== dependencies: - "@antfu/install-pkg" "^1.1.0" + "@antfu/install-pkg" "^2.0.1" "@iconify/types" "^2.0.0" import-meta-resolve "^4.2.0" @@ -205,17 +228,30 @@ dependencies: minipass "^7.0.4" -"@mermaid-js/parser@^1.2.0": - version "1.2.0" - resolved "https://registry.yarnpkg.com/@mermaid-js/parser/-/parser-1.2.0.tgz#266d728c54d2d4034d270f8b31d790e26296a5fa" - integrity sha512-oYPyv8A4As1yH5Bx+04iQEQxXuIQDe0GKCNSRgao6z8AM9jixXIfP0vsppRLvGf+nKIOb9/LdpWA4YuJiVvESA== +"@keyv/bigmap@^1.3.1": + version "1.3.1" + resolved "https://registry.yarnpkg.com/@keyv/bigmap/-/bigmap-1.3.1.tgz#fc82fa83947e7ff68c6798d08907db842771ef2c" + integrity sha512-WbzE9sdmQtKy8vrNPa9BRnwZh5UF4s1KTmSK0KUVLo3eff5BlQNNWDnFOouNpKfPKDnms9xynJjsMYjMaT/aFQ== + dependencies: + hashery "^1.4.0" + hookified "^1.15.0" + +"@keyv/serialize@^1.1.1": + version "1.1.1" + resolved "https://registry.yarnpkg.com/@keyv/serialize/-/serialize-1.1.1.tgz#0c01dd3a3483882af7cf3878d4e71d505c81fc4a" + integrity sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA== + +"@mermaid-js/parser@^1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@mermaid-js/parser/-/parser-1.2.1.tgz#94cc40416137bb10d2bc1b0a49c623d716e76233" + integrity sha512-n12NohV3mrUyUL2o93IgG/ifeW9FTyeJn3zDxkhwa8MJ9Fxg3HQMlA3RiGmD/3UnJvheztkjjQAjA2T4LmUcpw== dependencies: "@chevrotain/types" "~11.1.2" -"@puppeteer/browsers@2.13.0": - version "2.13.0" - resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.13.0.tgz#10f980c6d65efeff77f8a3cac6e1a7ac10604500" - integrity sha512-46BZJYJjc/WwmKjsvDFykHtXrtomsCIrwYQPOP7VfMJoZY2bsDF9oROBABR3paDjDcmkUye1Pb1BqdcdiipaWA== +"@puppeteer/browsers@2.13.2": + version "2.13.2" + resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.13.2.tgz#dcc8e7a4545d9680a8a72c53f132e80afc582284" + integrity sha512-5EUZSUIc37H6aIXyWO0Z4y8NlF8NnjgmqeQgOGiswAU7pY0HOo16ho4+alIWmSfdZnjqBRawMsP3I5YqLSn6kw== dependencies: debug "^4.4.3" extract-zip "^2.0.1" @@ -325,9 +361,9 @@ integrity sha512-fALi2aI6shfg7vM5KiR1wNJnZ7r6UuggVqtDA+xiEdPZQwy/trcQaHnwShLuLdta2rTymCNpxYTiMZX/e09F4g== "@types/d3-geo@*": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@types/d3-geo/-/d3-geo-3.1.0.tgz#b9e56a079449174f0a2c8684a9a4df3f60522440" - integrity sha512-856sckF0oP/diXtS4jNsiQw/UuK5fQG8l/a9VVLeSouf1/PPbBE1i1W852zVwKwYCBkFJJB7nCFTbk6UMEXBOQ== + version "3.1.1" + resolved "https://registry.yarnpkg.com/@types/d3-geo/-/d3-geo-3.1.1.tgz#9e283af179601c549581600b3fec25941911329d" + integrity sha512-65Emv9fQiQQqphLlRkuQ5ypPsOmWPhtBGCMv61JDPEPMvsx+gzhGf74yw1a78xFKPj6zw4AgQICJoQv0vK9M2w== dependencies: "@types/geojson" "*" @@ -359,9 +395,9 @@ integrity sha512-oUzyO1/Zm6rsxKRHA1vH0NEDG58HrT5icx/azi9MF1TWdtttWl0UIUsjEQBBh+SIkrpd21ZjEv7ptxWys1ncsg== "@types/d3-random@*": - version "3.0.3" - resolved "https://registry.yarnpkg.com/@types/d3-random/-/d3-random-3.0.3.tgz#ed995c71ecb15e0cd31e22d9d5d23942e3300cfb" - integrity sha512-Imagg1vJ3y76Y2ea0871wpabqp613+8/r0mCLEBfdtqC7xMSfj9idOnmBYyMoULfHePJyxMAw3nWhJxzc+LFwQ== + version "3.0.4" + resolved "https://registry.yarnpkg.com/@types/d3-random/-/d3-random-3.0.4.tgz#6bd3683b8332fc0f01e7059b7636bc5c7ede7337" + integrity sha512-UHYId5WTCx4L4YNel7NU00XUXXgvgpgZOvp10PuvsQENjMDXhh2RyFc0KBjO7B45ne4Ha1yVH7ii0vnzKkuzWA== "@types/d3-scale-chromatic@*": version "3.1.0" @@ -381,9 +417,9 @@ integrity sha512-bhAXu23DJWsrI45xafYpkQ4NtcKMwWnAC/vKrd2l+nxMFuvOT3XMYTIj2opv8vq8AO5Yh7Qac/nSeP/3zjTK0w== "@types/d3-shape@*": - version "3.1.8" - resolved "https://registry.yarnpkg.com/@types/d3-shape/-/d3-shape-3.1.8.tgz#d1516cc508753be06852cd06758e3bb54a22b0e3" - integrity sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w== + version "3.2.0" + resolved "https://registry.yarnpkg.com/@types/d3-shape/-/d3-shape-3.2.0.tgz#66ff342011dc243c6c20e6b899523d148aca412d" + integrity sha512-kVd74ta9eof3eJOvbNd1vGKS/XERRyQbT26Og63hIsvDO84cjD5gEOhsXf26w3FSoNlPVz84DOFcKv/oou+fMw== dependencies: "@types/d3-path" "*" @@ -454,9 +490,9 @@ "@types/d3-zoom" "*" "@types/debug@^4.0.0": - version "4.1.12" - resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.12.tgz#a155f21690871953410df4b6b6f53187f0500917" - integrity sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ== + version "4.1.13" + resolved "https://registry.yarnpkg.com/@types/debug/-/debug-4.1.13.tgz#22d1cc9d542d3593caea764f974306ab36286ee7" + integrity sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw== dependencies: "@types/ms" "*" @@ -466,9 +502,9 @@ integrity sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw== "@types/estree@^1.0.6", "@types/estree@^1.0.8": - version "1.0.8" - resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.8.tgz#958b91c991b1867ced318bedea0e215ee050726e" - integrity sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w== + version "1.0.9" + resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.9.tgz#cf3f0e876d7bee15a93ab925b82bf570a3904a24" + integrity sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg== "@types/geojson@*": version "7946.0.16" @@ -503,11 +539,11 @@ integrity sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA== "@types/node@*": - version "25.3.2" - resolved "https://registry.yarnpkg.com/@types/node/-/node-25.3.2.tgz#cbc4b963e1b3503eb2bcf7c55bf48c95204918d1" - integrity sha512-RpV6r/ij22zRRdyBPcxDeKAzH43phWVKEjL2iksqo1Vz3CuBUrgmPpPhALKiRfU7OMCmeeO9vECBMsV0hMTG8Q== + version "26.5.0" + resolved "https://registry.yarnpkg.com/@types/node/-/node-26.5.0.tgz#e6547ac7d7229d92d6e7157a6e19332cd16dc8ef" + integrity sha512-dVSGpriSoCgz8WnDNTuSSuSv1PC/ALXihO4ulRZt7Md8k9mlbdin3lGOcDE8SnWOgf513ByWlXd7BK4azmyg/A== dependencies: - undici-types "~7.18.0" + undici-types "~8.9.0" "@types/sinonjs__fake-timers@8.1.1": version "8.1.1" @@ -546,105 +582,100 @@ dependencies: "@types/node" "*" -"@typescript-eslint/eslint-plugin@8.61.1": - version "8.61.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.61.1.tgz#6e4b7fee21f1983308e9e9b634ecbaf702c86006" - integrity sha512-ZPlVl3PB3et/59Ne0fv/sci6ZXz4T4Hp4nTJ56i/Y0gR89ARb+KphojTq6j+56E5PIezmOIOOWyY+aWQFd+IkQ== +"@typescript-eslint/eslint-plugin@8.70.0": + version "8.70.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.70.0.tgz#59f635c74dd1e2bffb6aecbda557b24dadffd3dc" + integrity sha512-/v8HZt6RlyIZxB3ntehELOcUcfxKPVGWXnQdJuHRmzrqgF8nQypcC/oxGW+Ot4VGKDq81XugPKxx0n5PBtf9PA== dependencies: "@eslint-community/regexpp" "^4.12.2" - "@typescript-eslint/scope-manager" "8.61.1" - "@typescript-eslint/type-utils" "8.61.1" - "@typescript-eslint/utils" "8.61.1" - "@typescript-eslint/visitor-keys" "8.61.1" + "@typescript-eslint/scope-manager" "8.70.0" + "@typescript-eslint/type-utils" "8.70.0" + "@typescript-eslint/utils" "8.70.0" + "@typescript-eslint/visitor-keys" "8.70.0" ignore "^7.0.5" natural-compare "^1.4.0" ts-api-utils "^2.5.0" -"@typescript-eslint/parser@8.61.1": - version "8.61.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-8.61.1.tgz#881fba60b50636249cdeea2e547bf75715254c72" - integrity sha512-PJ5vePq5/ognBbrIcoC5+SHO5dfpeLPzP9FpLkzWrguoYQEeeSjlJpVwOpo1JRSTEi7dRcwNy4h4dzV70PqHcg== +"@typescript-eslint/parser@8.70.0": + version "8.70.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-8.70.0.tgz#96ce2de96c06c8442fea1a8f7b07855a56b3c5e3" + integrity sha512-zYvrmj9Yxd63UGaXw+kdt6A0F0s0qveJyuatIM77bYC2DE4pgmg7a50u8LR7PRtXd0x+h+Tl3eXabGm06SWd3Q== dependencies: - "@typescript-eslint/scope-manager" "8.61.1" - "@typescript-eslint/types" "8.61.1" - "@typescript-eslint/typescript-estree" "8.61.1" - "@typescript-eslint/visitor-keys" "8.61.1" + "@typescript-eslint/scope-manager" "8.70.0" + "@typescript-eslint/types" "8.70.0" + "@typescript-eslint/typescript-estree" "8.70.0" + "@typescript-eslint/visitor-keys" "8.70.0" debug "^4.4.3" -"@typescript-eslint/project-service@8.61.1": - version "8.61.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/project-service/-/project-service-8.61.1.tgz#fcd9739964a40867eed55f1ac318d3909f24b4af" - integrity sha512-PrC4JYGmR241lYnfhmKGTXkFqv8+ymbTFgSAY0fVXpY82/QkMw5TZPl+vGzuDDU2QYJk9fIDOBTntF+yDv9LEA== +"@typescript-eslint/project-service@8.70.0": + version "8.70.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/project-service/-/project-service-8.70.0.tgz#a62e837362f26c604ad15b20bacce1c3f4d6b552" + integrity sha512-hFHbTNqhU9G+2eKFXCBVb1tjFT/LceiJ4+HfLO4pTpDI0KHi6iajpcFFkaSQ9gXmCh7n82A0PthaayEdN6mspQ== dependencies: - "@typescript-eslint/tsconfig-utils" "^8.61.1" - "@typescript-eslint/types" "^8.61.1" + "@typescript-eslint/tsconfig-utils" "^8.70.0" + "@typescript-eslint/types" "^8.70.0" debug "^4.4.3" -"@typescript-eslint/scope-manager@8.61.1": - version "8.61.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-8.61.1.tgz#2479921a40fdb0afa18f5838fae6167264b417b2" - integrity sha512-L2bdIeoQS8FlKAvONAr20w6OcLXeB+qiDKbAooS9A0Ben+iSIkBef0FxqwKWYqt5sa0i4KJtxVyVmhMylKzF5w== +"@typescript-eslint/scope-manager@8.70.0": + version "8.70.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-8.70.0.tgz#b77628b03c9c56ef21fb5a6bc2f4a4335163b999" + integrity sha512-8nP3Kwh5hlgZ4FicGvmznAmJe8UL4sdU8tLukrPaMuQmDuk4Y8xYfzu/aYZW4xT2JCgc7H/TpDI5cGlxcWJSqQ== dependencies: - "@typescript-eslint/types" "8.61.1" - "@typescript-eslint/visitor-keys" "8.61.1" + "@typescript-eslint/types" "8.70.0" + "@typescript-eslint/visitor-keys" "8.70.0" -"@typescript-eslint/tsconfig-utils@8.61.1", "@typescript-eslint/tsconfig-utils@^8.61.1": - version "8.61.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.61.1.tgz#ca88080e0cf191d49516d7f300b67aa090d2254f" - integrity sha512-UN/H4di+OO7EWx2ovME+8t31YO+KVnK0RRKEHR3kOt21/Ay8BOq3M1OMvWs5vNiqcFCYGYoxK3MXPZzmMUE+yg== +"@typescript-eslint/tsconfig-utils@8.70.0", "@typescript-eslint/tsconfig-utils@^8.70.0": + version "8.70.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.70.0.tgz#f583ca72159c4fd8e775c153da3241de6b77974f" + integrity sha512-adnkeeNq9Sq1sUf4+FRVc0KdgYghzsgFpZSQVZVvY0LCuUuN0FnQgyGzCJeC4fW1cdXseBAjU2EOqUIjbNcZUw== -"@typescript-eslint/type-utils@8.61.1": - version "8.61.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-8.61.1.tgz#8fa18f453ee140893b47d339d1a6b64cac9b08a1" - integrity sha512-GYRicKmVK0C4fsKgaACaknOUAq9Oa2kwsjnpFhFcS/5p4Ht5IP9OVLbgIgcK4SRk92nVHFluurg1lumD9dBcLw== +"@typescript-eslint/type-utils@8.70.0": + version "8.70.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-8.70.0.tgz#7f9c01c24e56bfad2a089167926c7cdded9de5f6" + integrity sha512-NUMKIhYVaVIVLnRL9CRt+VVcuLgSHUCpXn4/+K8wql+vdInUzvx8BjUO1oJ7cG9shjFJKtF8F8Hh2kCh3/KBVw== dependencies: - "@typescript-eslint/types" "8.61.1" - "@typescript-eslint/typescript-estree" "8.61.1" - "@typescript-eslint/utils" "8.61.1" + "@typescript-eslint/types" "8.70.0" + "@typescript-eslint/typescript-estree" "8.70.0" + "@typescript-eslint/utils" "8.70.0" debug "^4.4.3" ts-api-utils "^2.5.0" -"@typescript-eslint/types@8.61.1", "@typescript-eslint/types@^8.61.1": - version "8.61.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-8.61.1.tgz#0c51f518e4e6848371a1c988e859d59eb7522d5a" - integrity sha512-G+CRlPqLv7Bz1IZVs03x5K59F1veqL0EJUROAdGhKsEq8qOiRiZbI+HUojPq5l0fEGOKModD9br6lObhB8zkoA== - -"@typescript-eslint/types@^8.11.0": - version "8.56.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-8.56.1.tgz#975e5942bf54895291337c91b9191f6eb0632ab9" - integrity sha512-dbMkdIUkIkchgGDIv7KLUpa0Mda4IYjo4IAMJUZ+3xNoUXxMsk9YtKpTHSChRS85o+H9ftm51gsK1dZReY9CVw== - -"@typescript-eslint/typescript-estree@8.61.1": - version "8.61.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-8.61.1.tgz#febbe70365ac0bf7611262b61b338fc8797965c7" - integrity sha512-u+oQD3BqYWPc8YV9Zab4vaJElJuwOLPRc10Jm1o/qS+6Qwen14HCWwx0Seo4LnSn2wxea2Ik8DxPt2/FHmuhrg== - dependencies: - "@typescript-eslint/project-service" "8.61.1" - "@typescript-eslint/tsconfig-utils" "8.61.1" - "@typescript-eslint/types" "8.61.1" - "@typescript-eslint/visitor-keys" "8.61.1" +"@typescript-eslint/types@8.70.0", "@typescript-eslint/types@^8.11.0", "@typescript-eslint/types@^8.70.0": + version "8.70.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-8.70.0.tgz#9ee52888cdeca604fe9436935219b967fa7f6053" + integrity sha512-asTOIYhDg4zdzOScCyaytrsV3cR6B4ecPQlXw/dJIm7J/MZTtCtfVII9JD8Geh4jTCrK/Xe6cg5UevoleMcoJQ== + +"@typescript-eslint/typescript-estree@8.70.0": + version "8.70.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-8.70.0.tgz#79cfcd9678ee28ea69cc13032c0f89bb1d298917" + integrity sha512-d9NmHMPEKQ7QCLLm1jI3zmoQBwT5KwFYjXBJ9ymZfKCUU+5rmTRykKAFvH5Qn/ZCds3CEAFS9OC9M/jkl0X2bA== + dependencies: + "@typescript-eslint/project-service" "8.70.0" + "@typescript-eslint/tsconfig-utils" "8.70.0" + "@typescript-eslint/types" "8.70.0" + "@typescript-eslint/visitor-keys" "8.70.0" debug "^4.4.3" minimatch "^10.2.2" semver "^7.7.3" tinyglobby "^0.2.15" ts-api-utils "^2.5.0" -"@typescript-eslint/utils@8.61.1": - version "8.61.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-8.61.1.tgz#ffd1054de7dd33b7873cd6c6713ec6b0366316d3" - integrity sha512-1+P/3Dj6jvtybE1q0HQ6yBt/gq+oKJyLdEv4HdnqasaEXRSYCAsD59mXEVQnM/ULNdQxbX77tdG4jPRjIS6knA== +"@typescript-eslint/utils@8.70.0": + version "8.70.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-8.70.0.tgz#78a78b4c52dd8523e5321993cb46ebe6d7934510" + integrity sha512-oZmtKJz/4fufZ2p3+Cn3ijEojcdfR+1zYDH2xKYrEly0dR/Q/1xUPRCOlKGxod78nWlU2UnDe09GZ3TaknBFGA== dependencies: "@eslint-community/eslint-utils" "^4.9.1" - "@typescript-eslint/scope-manager" "8.61.1" - "@typescript-eslint/types" "8.61.1" - "@typescript-eslint/typescript-estree" "8.61.1" + "@typescript-eslint/scope-manager" "8.70.0" + "@typescript-eslint/types" "8.70.0" + "@typescript-eslint/typescript-estree" "8.70.0" -"@typescript-eslint/visitor-keys@8.61.1": - version "8.61.1" - resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-8.61.1.tgz#546cf102b4efdb72a9a08e63a1b0d7d745eb66eb" - integrity sha512-6fJ9MHWtK14C1DSkiMlHUSOmrVebL7150xZJBlJiL62jjhIA4JmOq6flwBgDxIdBKKdoiZRel+dfPD5MLfny3w== +"@typescript-eslint/visitor-keys@8.70.0": + version "8.70.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-8.70.0.tgz#b451c8aea76dc97fc768b8d9d7f61019bf789628" + integrity sha512-BoC8PiO4Hkdo0TVJh9Ntxr5MxPDI7/oFsrygN5ADelFSeXG/qgNuucIGA+L5Z6JpPTE/uRfcTWtscjbUaufepQ== dependencies: - "@typescript-eslint/types" "8.61.1" + "@typescript-eslint/types" "8.70.0" eslint-visitor-keys "^5.0.0" "@upsetjs/venn.js@^2.0.0": @@ -661,14 +692,14 @@ acorn-jsx@^5.3.2: integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== acorn@^8.15.0, acorn@^8.16.0: - version "8.16.0" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.16.0.tgz#4ce79c89be40afe7afe8f3adb902a1f1ce9ac08a" - integrity sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw== + version "8.18.0" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.18.0.tgz#4faf01b2d6d326bfeed97aea1f52220b5f4c1940" + integrity sha512-lGq+9yr1/GuAWaVYIHRjvvySG5/4VfKIvC8EWxStPdcDh/Ka7FG3twP6v4d5BkravUilhIAsG4Qj83t02LWUPQ== adm-zip@^0.5.16: - version "0.5.17" - resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.17.tgz#5c0b65f37aeec5c2a94995c024f931f62e4bbc5a" - integrity sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ== + version "0.5.18" + resolved "https://registry.yarnpkg.com/adm-zip/-/adm-zip-0.5.18.tgz#283a05f2bf1e3fd315f0f31cde29b7a6e3c1619d" + integrity sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng== agent-base@6: version "6.0.2" @@ -683,9 +714,9 @@ agent-base@^7.1.0, agent-base@^7.1.2: integrity sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ== ajv@^6.14.0: - version "6.14.0" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.14.0.tgz#fd067713e228210636ebb08c60bd3765d6dbe73a" - integrity sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw== + version "6.15.0" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.15.0.tgz#07e982c74626167aa7a2495c53817892d7139492" + integrity sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw== dependencies: fast-deep-equal "^3.1.1" fast-json-stable-stringify "^2.0.0" @@ -705,9 +736,9 @@ ansi-regex@^5.0.1: integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== ansi-regex@^6.2.2: - version "6.2.2" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.2.2.tgz#60216eea464d864597ce2832000738a0589650c1" - integrity sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg== + version "6.3.0" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.3.0.tgz#247c8e7b70a1a43b10ce14c0226fcbf58e8815d5" + integrity sha512-WpDfL7NO6j7tH88IDBNVdUJxDh9nmCteAVW9dsep846XdwF4naCBK+/tGLX3KJgcpgMRXCFlTM2hKGoK9FsdrQ== ansi-styles@^4.0.0, ansi-styles@^4.1.0: version "4.3.0" @@ -721,18 +752,10 @@ ansi-styles@^6.2.1, ansi-styles@^6.2.3: resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-6.2.3.tgz#c044d5dcc521a076413472597a1acb1f103c4041" integrity sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg== -anymatch@~3.1.2: - version "3.1.3" - resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.3.tgz#790c58b19ba1720a84205b57c618d5ad8524973e" - integrity sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw== - dependencies: - normalize-path "^3.0.0" - picomatch "^2.0.4" - -arch@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/arch/-/arch-2.2.0.tgz#1bc47818f305764f23ab3306b0bfc086c5a29d11" - integrity sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ== +arch@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/arch/-/arch-3.0.0.tgz#a44e7077da4615fc5f1e3da21fbfc201d2c1817c" + integrity sha512-AmIAC+Wtm2AU8lGfTtHsw0Y9Qtftx2YXEEtiBP10xFUtMOA+sHHx6OAddyL52mUKh1vsXQ6/w1mVDptZCyUt4Q== are-docs-informative@^0.0.2: version "0.0.2" @@ -862,12 +885,12 @@ at-least-node@^1.0.0: integrity sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg== autoprefixer@>=10.2.5: - version "10.4.27" - resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-10.4.27.tgz#51ea301a5c3c5f8642f8e564759c4f573be486f2" - integrity sha512-NP9APE+tO+LuJGn7/9+cohklunJsXWiaWEfV3si4Gi/XHDwVNgkwr1J3RQYFIvPy76GmJ9/bW8vyoU1LcxwKHA== + version "10.5.5" + resolved "https://registry.yarnpkg.com/autoprefixer/-/autoprefixer-10.5.5.tgz#c2d2c2ccadda1d88536a4ff0e3e8d1417dbca388" + integrity sha512-uiRYvQYe/nNSzBJ7OUnd2/TZVsAdob3blml44teEpee9Cc1f4rGZFewO+JT3Wo8mgFOSzNqes4FHZn/Qz8WOuw== dependencies: - browserslist "^4.28.1" - caniuse-lite "^1.0.30001774" + browserslist "^4.28.9" + caniuse-lite "^1.0.30001810" fraction.js "^5.3.4" picocolors "^1.1.1" postcss-value-parser "^4.2.0" @@ -890,17 +913,17 @@ aws4@^1.8.0: integrity sha512-lHe62zvbTB5eEABUVi/AwVh0ZKY9rMMDhmm+eeyuuUQbQ3+J+fONVQOZyj+DdrvD4BY33uYniyRJ4UJIaSKAfw== axe-core@^4.10.0: - version "4.11.1" - resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.11.1.tgz#052ff9b2cbf543f5595028b583e4763b40c78ea7" - integrity sha512-BASOg+YwO2C+346x3LZOeoovTIoTrRqEsqMa6fmfAV0P+U9mFr9NsyOEpiYvFjbc64NMrSswhV50WdXzdb/Z5A== + version "4.13.0" + resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.13.0.tgz#f868ecb1bd61d982321760e51d841ab497ab86d0" + integrity sha512-UzGt8zg7Ny8djbYMhxl2zuEevVa7r2gJjYY5Lwr1xM7+XU2nd6CkIWFTVcCIbAP63vSz71NaVyyuSk9lHKcy0A== axios@^1.18.0: - version "1.18.0" - resolved "https://registry.yarnpkg.com/axios/-/axios-1.18.0.tgz#8a7f8854af280fcaae063272df2ed9f3837d2398" - integrity sha512-E32NzpYKp++W7XRe52rHiXV2ehxmh3wbdgO7MHeFM+vqxLBYHzt0ElkiImtOBxtOmyp0yoC8C6uESVV84Y2/hw== + version "1.20.0" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.20.0.tgz#515513445aa60e71d04b6521ca6210829ccb4786" + integrity sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg== dependencies: follow-redirects "^1.16.0" - form-data "^4.0.5" + form-data "^4.0.6" https-proxy-agent "^5.0.1" proxy-from-env "^2.1.0" @@ -909,10 +932,10 @@ axobject-query@^4.1.0: resolved "https://registry.yarnpkg.com/axobject-query/-/axobject-query-4.1.0.tgz#28768c76d0e3cff21bc62a9e2d0b6ac30042a1ee" integrity sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ== -b4a@^1.6.4: - version "1.8.0" - resolved "https://registry.yarnpkg.com/b4a/-/b4a-1.8.0.tgz#1ca3ba0edc9469aaabef5647e769a83d50180b1a" - integrity sha512-qRuSmNSkGQaHwNbM7J78Wwy+ghLEYF1zNrSeMxj4Kgw6y33O3mXcQ6Ie9fRvfU/YnxWkOchPXbaLb73TkIsfdg== +b4a@^1.6.4, b4a@^1.8.1: + version "1.8.1" + resolved "https://registry.yarnpkg.com/b4a/-/b4a-1.8.1.tgz#7f16334ca80127aeb26064a28841acbf174840a4" + integrity sha512-aiqre1Nr0B/6DgE2N5vwTc+2/oQZ4Wh1t4NznYY4E00y8LCt6NqdRv81so00oo27D8MVKTpUa/MwUUtBLXCoDw== bail@^2.0.0: version "2.0.2" @@ -930,14 +953,14 @@ balanced-match@^4.0.2: integrity sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA== bare-events@^2.5.4, bare-events@^2.7.0: - version "2.8.2" - resolved "https://registry.yarnpkg.com/bare-events/-/bare-events-2.8.2.tgz#7b3e10bd8e1fc80daf38bb516921678f566ab89f" - integrity sha512-riJjyv1/mHLIPX4RwiK+oW9/4c3TEUeORHKefKAKnZ5kyslbN+HXowtbaVEqt4IMUB7OXlfixcs6gsFeo/jhiQ== + version "2.9.2" + resolved "https://registry.yarnpkg.com/bare-events/-/bare-events-2.9.2.tgz#01d377b64c3c7167b28b52da7add3221aecf2d7f" + integrity sha512-AIPKioV7/Y/8KfZ3AAhjPJxLLbY49S64Ym5DakZlUg75qQiTgUq9hEJoEwa4eUezPUlXRy/i5NpsKvo9jgKmoA== -bare-fs@^4.0.1: - version "4.5.5" - resolved "https://registry.yarnpkg.com/bare-fs/-/bare-fs-4.5.5.tgz#589a8f87a32af0266aa474413c8d7d11d50e4a65" - integrity sha512-XvwYM6VZqKoqDll8BmSww5luA5eflDzY0uEFfBJtFKe4PAAtxBjU3YIxzIBzhyaEQBy1VXEQBto4cpN5RZJw+w== +bare-fs@^4.0.1, bare-fs@^4.5.5: + version "4.8.1" + resolved "https://registry.yarnpkg.com/bare-fs/-/bare-fs-4.8.1.tgz#1a946560b45844dc37120c00330eae7c8a1c44cf" + integrity sha512-N1nnXdHZAOSstz0XiHikGS4HGMH4CnSwhqWdGQQMqqdvp4Jybm9sE3R1WVnpWVd4SFkc8ryPDBLViNLwiEqECg== dependencies: bare-events "^2.5.4" bare-path "^3.0.0" @@ -945,30 +968,24 @@ bare-fs@^4.0.1: bare-url "^2.2.2" fast-fifo "^1.3.2" -bare-os@^3.0.1: - version "3.7.0" - resolved "https://registry.yarnpkg.com/bare-os/-/bare-os-3.7.0.tgz#23c60064e53400db1550ef4b2987fdc42ee399b2" - integrity sha512-64Rcwj8qlnTZU8Ps6JJEdSmxBEUGgI7g8l+lMtsJLl4IsfTcHMTfJ188u2iGV6P6YPRZrtv72B2kjn+hp+Yv3g== - bare-path@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/bare-path/-/bare-path-3.0.0.tgz#b59d18130ba52a6af9276db3e96a2e3d3ea52178" - integrity sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw== - dependencies: - bare-os "^3.0.1" + version "3.1.2" + resolved "https://registry.yarnpkg.com/bare-path/-/bare-path-3.1.2.tgz#67261bc74a9ba0105ba139b8638f218c09e8d82e" + integrity sha512-ZyKbsuuqK6Ag0K8pX6V5Txq6XeJRvY+wXucnFGRjiyVYP9YWDpIQugk/b+enRYrEYBJaqLzghRQpXPMR7341Nw== bare-stream@^2.6.4: - version "2.8.0" - resolved "https://registry.yarnpkg.com/bare-stream/-/bare-stream-2.8.0.tgz#3ac6141a65d097fd2bf6e472c848c5d800d47df9" - integrity sha512-reUN0M2sHRqCdG4lUK3Fw8w98eeUIZHL5c3H7Mbhk2yVBL+oofgaIp0ieLfD5QXwPCypBpmEEKU2WZKzbAk8GA== + version "2.13.4" + resolved "https://registry.yarnpkg.com/bare-stream/-/bare-stream-2.13.4.tgz#61d448a268d1efd992103aae6bd729332f278ac0" + integrity sha512-PcrQ8lVLbiJscNm1Kez+Yp4Gy4AHGcN1lzwjvf5NybWen7VvEgUfyfnXYJ2zNqWnzOfCb1Abq6lH8ti0syQszA== dependencies: - streamx "^2.21.0" + b4a "^1.8.1" + streamx "^2.25.0" teex "^1.0.1" bare-url@^2.2.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/bare-url/-/bare-url-2.3.2.tgz#4aef382efa662b2180a6fe4ca07a71b39bdf7ca3" - integrity sha512-ZMq4gd9ngV5aTMa5p9+UfY0b3skwhHELaDkhEHetMdX0LRkW9kzaym4oo/Eh+Ghm0CCDuMTsRIGM/ytUc1ZYmw== + version "2.5.4" + resolved "https://registry.yarnpkg.com/bare-url/-/bare-url-2.5.4.tgz#10ab50f39b50335e2aa7672d5941043dbfc7749e" + integrity sha512-Gxa7UVWBr0/edU1b+TJhn/AZvMQUj9OGspvYsaTYQrAbZA4BOTZGL3LiZxvD+CeMlDH4juwD84+eTAp/bLYW5g== dependencies: bare-path "^3.0.0" @@ -977,10 +994,10 @@ base64-js@^1.3.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -baseline-browser-mapping@^2.11.12: - version "2.11.20" - resolved "https://registry.yarnpkg.com/baseline-browser-mapping/-/baseline-browser-mapping-2.11.20.tgz#26078c7a4b08299656ea7ddceaebec955dc44303" - integrity sha512-H0ulySigv6icDJ1F7SjtdCD6PrhTpdYCmP0CactWy1+ekh0AFd0o1Wn5T8b+hnTmdBx19u9yhL6wvCylXMY7zw== +baseline-browser-mapping@^2.11.20: + version "2.11.21" + resolved "https://registry.yarnpkg.com/baseline-browser-mapping/-/baseline-browser-mapping-2.11.21.tgz#99af73cb8e54007e4f5345e132278e26c2662f2c" + integrity sha512-uh8vpY/1/YyFkunIDFH/12p7/7VdPKA1hejMVEbdkEaWnUz0Hesvx5EbiU6XxjyHZIOju+ZMbQJkRh+es3/spQ== basic-ftp@5.3.1, basic-ftp@^5.0.2: version "5.3.1" @@ -999,11 +1016,6 @@ big-integer@^1.6.48: resolved "https://registry.yarnpkg.com/big-integer/-/big-integer-1.6.52.tgz#60a887f3047614a8e1bffe5d7173490a97dc8c85" integrity sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg== -binary-extensions@^2.0.0: - version "2.3.0" - resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" - integrity sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw== - blob-util@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/blob-util/-/blob-util-2.0.2.tgz#3b4e3c281111bb7f11128518006cdc60b403a1eb" @@ -1022,30 +1034,23 @@ brace-expansion@1.1.13, brace-expansion@^1.1.7: balanced-match "^1.0.0" concat-map "0.0.1" -brace-expansion@5.0.6, brace-expansion@^5.0.2, brace-expansion@^5.0.5: +brace-expansion@5.0.6, brace-expansion@^5.0.8: version "5.0.6" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-5.0.6.tgz#ec68fe0a641a29d8711579caf641d05bae1f2285" integrity sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g== dependencies: balanced-match "^4.0.2" -braces@~3.0.2: - version "3.0.3" - resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" - integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== - dependencies: - fill-range "^7.1.1" - -browserslist@^4.28.1: - version "4.28.8" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.28.8.tgz#a3c79ceb70028527e5da7dafc887f3200b5168c0" - integrity sha512-V2NpofLblG64mfOtSgDhOJESZEGogzDMBv/q+W6oc4LXWP/q75eOXoOaaOu1EOadB9U4Bwx/e0yzbvwKH8zalA== +browserslist@^4.28.9: + version "4.28.9" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.28.9.tgz#07ce6b449b90af880eb9bfb7cd39372cc4f71c8c" + integrity sha512-EWazOblFYUvlGZcfGhPUPmYh3nikUxBVb+y9MJun5f3hBi812X+8MSQTujLBtgK3cf51fJWbWfOjyeO954d+Eg== dependencies: - baseline-browser-mapping "^2.11.12" - caniuse-lite "^1.0.30001809" - electron-to-chromium "^1.5.402" - node-releases "^2.0.53" - update-browserslist-db "^1.3.0" + baseline-browser-mapping "^2.11.20" + caniuse-lite "^1.0.30001810" + electron-to-chromium "^1.5.420" + node-releases "^2.0.54" + update-browserslist-db "^1.3.2" buffer-crc32@~0.2.3: version "0.2.13" @@ -1060,12 +1065,23 @@ buffer@^5.7.1: base64-js "^1.3.1" ieee754 "^1.1.13" +cacheable@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/cacheable/-/cacheable-2.5.0.tgz#d142d41043e5a865f6053cc70ef4e3ad068bd5ce" + integrity sha512-60cyAOytib/OzBw1JNSoSV/boK1AtHryDIjvVBk7XbN4ugfkM3+Sry7fEjNgPMGgOjuaZPAp8ruZ0Cxafwyq9g== + dependencies: + "@cacheable/memory" "^2.2.0" + "@cacheable/utils" "^2.5.0" + hookified "^1.15.0" + keyv "^5.6.0" + qified "^0.10.1" + cachedir@^2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/cachedir/-/cachedir-2.4.0.tgz#7fef9cf7367233d7c88068fe6e34ed0d355a610d" integrity sha512-9EtFOZR8g22CL7BWjJ9BUx1+A/djkofnyW3aOXZORNW2kxoUpx2h+uN2cOqwPmFhnpVmxg+KW2OjOSgChTEvsQ== -call-bind-apply-helpers@^1.0.0, call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2: +call-bind-apply-helpers@^1.0.1, call-bind-apply-helpers@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz#4b5428c222be985d79c3d82657479dbe0b59b2d6" integrity sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ== @@ -1073,14 +1089,14 @@ call-bind-apply-helpers@^1.0.0, call-bind-apply-helpers@^1.0.1, call-bind-apply- es-errors "^1.3.0" function-bind "^1.1.2" -call-bind@^1.0.7, call-bind@^1.0.8: - version "1.0.8" - resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.8.tgz#0736a9660f537e3388826f440d5ec45f744eaa4c" - integrity sha512-oKlSFMcMwpUg2ednkhQ454wfWiU/ul3CkJe/PEHcTKuiX6RpbehUiFMXu13HalGZxfUwCQzZG747YXBn1im9ww== +call-bind@^1.0.7, call-bind@^1.0.8, call-bind@^1.0.9: + version "1.0.9" + resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.9.tgz#39a644700c80bc7d0ca9102fc6d1d43b2fd7eee7" + integrity sha512-a/hy+pNsFUTR+Iz8TCJvXudKVLAnz/DyeSUo10I5yvFDQJBFU2s9uqQpoSrJlroHUKoKqzg+epxyP9lqFdzfBQ== dependencies: - call-bind-apply-helpers "^1.0.0" - es-define-property "^1.0.0" - get-intrinsic "^1.2.4" + call-bind-apply-helpers "^1.0.2" + es-define-property "^1.0.1" + get-intrinsic "^1.3.0" set-function-length "^1.2.2" call-bound@^1.0.2, call-bound@^1.0.3, call-bound@^1.0.4: @@ -1096,12 +1112,7 @@ callsites@^3.0.0: resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== -caniuse-lite@^1.0.30001774: - version "1.0.30001774" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001774.tgz#0e576b6f374063abcd499d202b9ba1301be29b70" - integrity sha512-DDdwPGz99nmIEv216hKSgLD+D4ikHQHjBC/seF98N9CPqRX4M5mSxT9eTV6oyisnJcuzxtZy4n17yKKQYmYQOA== - -caniuse-lite@^1.0.30001809: +caniuse-lite@^1.0.30001810: version "1.0.30001810" resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001810.tgz#4970b477dea3278374de9bc43aa8f5d39fc3cda2" integrity sha512-TITQPUkaz+aVk5GL6NhOdwk1aEaNTSDPsGFWrTuhKGtjTF70jL/Oht2W4c6rXUe5fu7Ie19VIahAXHIIiWWNeg== @@ -1129,26 +1140,26 @@ character-entities@^2.0.0: resolved "https://registry.yarnpkg.com/character-entities/-/character-entities-2.0.2.tgz#2d09c2e72cd9523076ccb21157dff66ad43fcc22" integrity sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ== -chokidar@^3.3.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.6.0.tgz#197c6cc669ef2a8dc5e7b4d97ee4e092c3eb0d5b" - integrity sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw== - dependencies: - anymatch "~3.1.2" - braces "~3.0.2" - glob-parent "~5.1.2" - is-binary-path "~2.1.0" - is-glob "~4.0.1" - normalize-path "~3.0.0" - readdirp "~3.6.0" - optionalDependencies: - fsevents "~2.3.2" +chokidar@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-5.0.0.tgz#949c126a9238a80792be9a0265934f098af369a5" + integrity sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw== + dependencies: + readdirp "^5.0.0" chownr@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/chownr/-/chownr-3.0.0.tgz#9855e64ecd240a9cc4267ce8a4aa5d24a1da15e4" integrity sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g== +chrome-remote-interface@0.33.3: + version "0.33.3" + resolved "https://registry.yarnpkg.com/chrome-remote-interface/-/chrome-remote-interface-0.33.3.tgz#d8b4339f487b460a9461af7355bda98054e8e1a4" + integrity sha512-zNnn0prUL86Teru6UCAZ1yU1XeXljHl3gj7OrfPcarEfU62OUU4IujDPdTDW3dAWwRqN3ZMG/Chhkh2gPL/wiw== + dependencies: + commander "2.11.x" + ws "^7.2.0" + chromium-bidi@14.0.0: version "14.0.0" resolved "https://registry.yarnpkg.com/chromium-bidi/-/chromium-bidi-14.0.0.tgz#15a12ab083ae519a49a724e94994ca0a9ced9c8e" @@ -1195,6 +1206,15 @@ cliui@^8.0.1: strip-ansi "^6.0.1" wrap-ansi "^7.0.0" +cliui@^9.0.1: + version "9.0.1" + resolved "https://registry.yarnpkg.com/cliui/-/cliui-9.0.1.tgz#6f7890f386f6f1f79953adc1f78dec46fcc2d291" + integrity sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w== + dependencies: + string-width "^7.2.0" + strip-ansi "^7.1.0" + wrap-ansi "^9.0.0" + color-convert@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" @@ -1210,9 +1230,9 @@ color-convert@^3.1.3: color-name "^2.0.0" color-name@^2.0.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/color-name/-/color-name-2.1.0.tgz#0b677385c1c4b4edfdeaf77e38fa338e3a40b693" - integrity sha512-1bPaDNFm0axzE4MEAzKPuqKWeRaT43U/hyxKPBdqTfmPF+d6n7FSoTFxLVULUJOmiLp01KjhIPPH+HrXZJN4Rg== + version "2.1.1" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-2.1.1.tgz#86c00ec98db36705be5bd4428097f13af1175a6d" + integrity sha512-p2FdgwVx1a9yWBHP2wI0VgShkDpgN4kZISkxdNipGBJWpa5G6b04OINlVWCyJj0JmfvcPrgqt95E9k8yvaOJFg== color-name@~1.1.4: version "1.1.4" @@ -1251,6 +1271,11 @@ combined-stream@^1.0.8, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" +commander@2.11.x: + version "2.11.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" + integrity sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ== + commander@7: version "7.2.0" resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7" @@ -1306,9 +1331,9 @@ cose-base@^2.2.0: layout-base "^2.0.0" cosmiconfig@^9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-9.0.0.tgz#34c3fc58287b915f3ae905ab6dc3de258b55ad9d" - integrity sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg== + version "9.0.2" + resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-9.0.2.tgz#9e5615163becf6a82211fb33d2f68947c25d0c5e" + integrity sha512-gtTZxTDau1wL7Y7zifc2dd8jHSK/k6BTx/2Xp/BpdlAdnlYWFVt7qhJqgwi7637yRwRQ3qL4ZidbB4I8tA5VOg== dependencies: env-paths "^2.2.1" import-fresh "^3.3.0" @@ -1325,21 +1350,22 @@ cross-spawn@^7.0.0, cross-spawn@^7.0.6: which "^2.0.1" cypress@^15.17.0: - version "15.17.0" - resolved "https://registry.yarnpkg.com/cypress/-/cypress-15.17.0.tgz#45b1807d2cb06aa1985006824ce2aa11d3cb7190" - integrity sha512-WL5Gcqi1GaDWozBwXmkSAtOPafTsVSRS764iX6xvuz3DPzvBAxbkRyEi4BreVdVWxLDpiYRgZCyJUafBw44njw== + version "15.21.1" + resolved "https://registry.yarnpkg.com/cypress/-/cypress-15.21.1.tgz#51653d58fa0e9994a11406aa3df2bee716264f9a" + integrity sha512-ogHpHMj0XNlZA5MGzjg8SWHf0eMw9lTwVQRKjpcT1GzNTxNUjwnHA8YCG6nOJ8ThU+XZaUxJgpMYZnJ//8aDaw== dependencies: "@cypress/request" "^4.0.0" "@cypress/xvfb" "^1.2.4" "@types/sinonjs__fake-timers" "8.1.1" "@types/sizzle" "^2.3.2" "@types/tmp" "^0.2.3" - arch "^2.2.0" + arch "^3.0.0" blob-util "^2.0.2" bluebird "^3.7.2" buffer "^5.7.1" cachedir "^2.4.0" chalk "^4.1.0" + chrome-remote-interface "0.33.3" ci-info "^4.1.0" cli-table3 "0.6.1" commander "^6.2.1" @@ -1365,7 +1391,6 @@ cypress@^15.17.0: systeminformation "^5.31.1" tmp "~0.2.4" tree-kill "1.2.2" - tslib "1.14.1" untildify "^4.0.0" yauzl "^3.3.1" @@ -1383,10 +1408,10 @@ cytoscape-fcose@^2.2.0: dependencies: cose-base "^2.2.0" -cytoscape@^3.33.3: - version "3.34.0" - resolved "https://registry.yarnpkg.com/cytoscape/-/cytoscape-3.34.0.tgz#5fbe2eb1cf76b070a8ecd5647c35f65aa097c9c6" - integrity sha512-62rNSrioXw93uliKFBwjukeQyeWwH2PqDrTac31r2P6464u3AUvTk0xS4LVvT251g7IgkFunrI48ZEZGjywSOg== +cytoscape@^3.34.0: + version "3.34.3" + resolved "https://registry.yarnpkg.com/cytoscape/-/cytoscape-3.34.3.tgz#1503996ba0b59b901d86310a1f612e92dc464f51" + integrity sha512-yfYGhRcGAntq6YBD583j4n0Eg3jIxvWmZtz/5uz9UYkeIStSlMxuUja+ec5j3iBD8nv1rwaOAYMW09tBdkSeaQ== "d3-array@1 - 2": version "2.12.1" @@ -1711,10 +1736,10 @@ data-view-byte-offset@^1.0.1: es-errors "^1.3.0" is-data-view "^1.0.1" -dayjs@^1.10.4, dayjs@^1.11.20: - version "1.11.21" - resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.21.tgz#57f87562e62de76f3c704bd2b8d522fc33068eb2" - integrity sha512-98IT+HOahAisibz/yjKbzuOBwYcjJ7BCLPzARyHiyEBmRz4fatF+KPJszEHXsGYjUG234aH/cOjW1wwTbKUZlA== +dayjs@^1.10.4, dayjs@^1.11.21: + version "1.11.23" + resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.23.tgz#b0a363506dde5f36cf5075e42ebe8115165a8c79" + integrity sha512-QDTCU0M0MxR3hQfnlDJfwekQiaanm1ubOD231u73WBckQ/fsamwRLiE2GBz6D3a/xF1NgfiDLJjXBa1hYOYTtQ== debug@4, debug@^4.0.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@^4.4.1, debug@^4.4.3: version "4.4.3" @@ -1770,9 +1795,9 @@ degenerator@^5.0.0: esprima "^4.0.1" delaunator@5: - version "5.0.1" - resolved "https://registry.yarnpkg.com/delaunator/-/delaunator-5.0.1.tgz#39032b08053923e924d6094fe2cde1a99cc51278" - integrity sha512-8nvh+XBe96aCESrGOqMp/84b13H9cdKbG5P2ejQCh4d4sK9RL4371qou9drQjMhvnPmhWl5hnmqbEE0fXr9Xnw== + version "5.1.0" + resolved "https://registry.yarnpkg.com/delaunator/-/delaunator-5.1.0.tgz#d13271fbf3aff6753f9ea6e235557f20901046ea" + integrity sha512-AGrQ4QSgssa1NGmWmLPqN5NY2KajF5MqxetNEO+o0n3ZwZZeTmt7bBnvzHWrmkZFxGgr4HdyFgelzgi06otLuQ== dependencies: robust-predicates "^3.0.2" @@ -1798,10 +1823,10 @@ devlop@^1.0.0, devlop@^1.1.0: dependencies: dequal "^2.0.0" -devtools-protocol@0.0.1566079: - version "0.0.1566079" - resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1566079.tgz#28049eed025a25cf3aea964f6c6bd517796c44ac" - integrity sha512-MJfAEA1UfVhSs7fbSQOG4czavUp1ajfg6prlAN0+cmfa2zNjaIbvq8VneP7do1WAQQIvgNJWSMeP6UyI90gIlQ== +devtools-protocol@0.0.1608973: + version "0.0.1608973" + resolved "https://registry.yarnpkg.com/devtools-protocol/-/devtools-protocol-0.0.1608973.tgz#56e0a2a999b06d416ee928ca06aeba95a5880515" + integrity sha512-Tpm17fxYzt+J7VrGdc1k8YdRqS3YV7se/M6KeemEqvUbq/n7At1rWVuXMxQgpWkdwSdIEKYbU//Bve+Shm4YNQ== discontinuous-range@1.0.0: version "1.0.0" @@ -1816,9 +1841,9 @@ doctrine@^2.1.0: esutils "^2.0.2" dompurify@>=3.3.2, dompurify@^3.3.3: - version "3.4.13" - resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-3.4.13.tgz#fc28949d59f92d62e28a3a764bcbeee35897a1be" - integrity sha512-2vmYIoqjze2d+kakP8S/nS5shfsl587kzwEjcGlTdiksUVgFHnFCsLYDVj/JNqJVOQZGSYBTmuycv0PodwmnMQ== + version "3.4.15" + resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-3.4.15.tgz#30351b34513894f428c13bddb28bff0a027ae804" + integrity sha512-EUBjM+B+lkDE41iE82DDSCfkoPGfXx8IxFxPMjNzm/Uk4xDet77rTN9wqlxlVg71kK7XGuUMv6wUxJUwwv+Xyw== optionalDependencies: "@types/trusted-types" "^2.0.7" @@ -1839,10 +1864,10 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" -electron-to-chromium@^1.5.402: - version "1.5.419" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.419.tgz#96fb638258642b37d2538390028d207437712873" - integrity sha512-nHMPn8x4yCxCI0iSnL+LlHL5sUoUfjLXkcRIagZ4GBdrfFLFaiLNvzJWbJqZhFT9IAhw5tUSNlhggWN+otvp/A== +electron-to-chromium@^1.5.420: + version "1.5.425" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.5.425.tgz#7ba6d039fbb547080de3992974f6bf435f0ce705" + integrity sha512-QvPtl41EUOnuT1HBvMKgxXRIaHNcagBPs50u7VULzhZXaGfqTbZyE16LQsctZ/RQHlGu+FOWeDTR4mY6YbeF1g== emoji-regex@^10.3.0: version "10.6.0" @@ -1888,10 +1913,20 @@ error-ex@^1.3.1: dependencies: is-arrayish "^0.2.1" -es-abstract@^1.23.2, es-abstract@^1.23.3, es-abstract@^1.23.5, es-abstract@^1.23.9, es-abstract@^1.24.0: - version "1.24.1" - resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.24.1.tgz#f0c131ed5ea1bb2411134a8dd94def09c46c7899" - integrity sha512-zHXBLhP+QehSSbsS9Pt23Gg964240DPd6QCf8WpkqEXxQ7fhdZzYsocOr5u7apWonsS5EjZDmTF+/slGMyasvw== +es-abstract-get@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/es-abstract-get/-/es-abstract-get-1.0.0.tgz#1eae87101f42bedeb6a740e8c5051271aef89088" + integrity sha512-6PMWXpdhshVvFp+FoWYs1EvG1Nj0tvk0dZM+XcK0xMEM1czRVcP6ohqPWHy6qPagSpC8j4+p89WXlT+xXJs/fg== + dependencies: + es-errors "^1.3.0" + es-object-atoms "^1.1.2" + is-callable "^1.2.7" + object-inspect "^1.13.4" + +es-abstract@^1.23.2, es-abstract@^1.23.3, es-abstract@^1.23.5, es-abstract@^1.23.9, es-abstract@^1.24.0, es-abstract@^1.24.2: + version "1.24.2" + resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.24.2.tgz#2dbd38c180735ee983f77585140a2706a963ed9a" + integrity sha512-2FpH9Q5i2RRwyEP1AylXe6nYLR5OhaJTZwmlcP0dL/+JCbgg7yyEo/sEK6HeGZRf3dFpWwThaRHVApXSkW3xeg== dependencies: array-buffer-byte-length "^1.0.2" arraybuffer.prototype.slice "^1.0.4" @@ -1958,10 +1993,10 @@ es-errors@^1.3.0: resolved "https://registry.yarnpkg.com/es-errors/-/es-errors-1.3.0.tgz#05f75a25dab98e4fb1dcd5e1472c0546d5057c8f" integrity sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw== -es-object-atoms@^1.0.0, es-object-atoms@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.1.1.tgz#1c4f2c4837327597ce69d2ca190a7fdd172338c1" - integrity sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA== +es-object-atoms@^1.0.0, es-object-atoms@^1.1.1, es-object-atoms@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/es-object-atoms/-/es-object-atoms-1.1.2.tgz#a2d0b373205724dfa525d23b0c3e1b1ca582c99b" + integrity sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw== dependencies: es-errors "^1.3.0" @@ -1983,18 +2018,21 @@ es-shim-unscopables@^1.0.2, es-shim-unscopables@^1.1.0: hasown "^2.0.2" es-to-primitive@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.3.0.tgz#96c89c82cc49fd8794a24835ba3e1ff87f214e18" - integrity sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g== + version "1.3.4" + resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.3.4.tgz#0c854291cf0d7b439d6b9e5771ea837ffaf1f991" + integrity sha512-yPDz7wqpg1/mmHLmS3tcfTfbw5f1eryXvyghYBffGdERwe+mV7ZcWzTR8LR17Kvqt3qfPurjlonmnq3MKXIOXw== dependencies: + es-abstract-get "^1.0.0" + es-define-property "^1.0.1" + es-errors "^1.3.0" is-callable "^1.2.7" - is-date-object "^1.0.5" - is-symbol "^1.0.4" + is-date-object "^1.1.0" + is-symbol "^1.1.1" es-toolkit@^1.45.1: - version "1.46.1" - resolved "https://registry.yarnpkg.com/es-toolkit/-/es-toolkit-1.46.1.tgz#38ca27191a98a867fc544b81cf1477a68947fb06" - integrity sha512-5eNtXOs3tbfxXOj04tjjseeWkRWaoCjdEI+96DgwzZoe6c9juL49pXlzAFTI72aWC9Y8p7168g6XIKjh7k6pyQ== + version "1.52.0" + resolved "https://registry.yarnpkg.com/es-toolkit/-/es-toolkit-1.52.0.tgz#71eaf1a8b18834ef77637eccbb885ba4c03cd6dd" + integrity sha512-XTNEJQh1tY1ZJVcf6ayP/2n4ZPyaHlW2FWs7xvw5ddPuhUVjLD3olQVQS7kf58JbAB48iL0uL/jerTrjtV3lDA== escalade@^3.1.1, escalade@^3.2.0: version "3.2.0" @@ -2028,18 +2066,18 @@ eslint-config-prettier@^10.1.5: integrity sha512-82GZUjRS0p/jganf6q1rEO25VSoHH0hKPCTrgillPjdI/3bgBhAE1QzHrHTizjpRvy6pGAvKjDJtk2pF9NDq8w== eslint-import-resolver-node@^0.3.9: - version "0.3.9" - resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz#d4eaac52b8a2e7c3cd1903eb00f7e053356118ac" - integrity sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g== + version "0.3.10" + resolved "https://registry.yarnpkg.com/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.10.tgz#84ce3005abfc300588cf23bbac1aabec1fc6e8c1" + integrity sha512-tRrKqFyCaKict5hOd244sL6EQFNycnMQnBe+j8uqGNXYzsImGbGUU4ibtoaBmv5FLwJwcFJNeg1GeVjQfbMrDQ== dependencies: debug "^3.2.7" - is-core-module "^2.13.0" - resolve "^1.22.4" + is-core-module "^2.16.1" + resolve "^2.0.0-next.6" eslint-module-utils@^2.12.1: - version "2.12.1" - resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.12.1.tgz#f76d3220bfb83c057651359295ab5854eaad75ff" - integrity sha512-L8jSWTze7K2mTg0vos/RuLRS5soomksDPoJLXIslC7c8Wmut3bx7CPpJijDcBZtxQ5lrbUdM+s0OlNbz0DCDNw== + version "2.14.0" + resolved "https://registry.yarnpkg.com/eslint-module-utils/-/eslint-module-utils-2.14.0.tgz#611f8e1c6ceb29a93eb949e1cc670b8287764819" + integrity sha512-W2WCRZ9Dqntd+2u8jJcVMV2PKulc6RdLgUUoh/yQr3uB6lo/ZOeGx11sv60/8S4QFFKNslAlWhr9u0Ef7ZW6Ig== dependencies: debug "^3.2.7" @@ -2131,16 +2169,16 @@ eslint-visitor-keys@^5.0.0, eslint-visitor-keys@^5.0.1: integrity sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA== eslint@^10.5.0: - version "10.5.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-10.5.0.tgz#5fca69d6b41fe7e00ba22d4100b2e44efe439ad5" - integrity sha512-1y+7C+vi12bUK1IpZeaV3gsH9fHLBmPvYmPx42pvT/E9yG0IC8g3PUZZgp0+JLJl7ZDK0flc2gc+Aw9dpCvIsQ== + version "10.10.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-10.10.0.tgz#14b1d1849eedb2ee6805f3759050479ce2b23d71" + integrity sha512-NPXn6r5zl4uET1DAVPaOwzX3rut4c0wcmw3dWJAfOsTM5+TogXo0DDjz8pwm/hL8cyVNpHqeK4JpN0NjnyFFNw== dependencies: "@eslint-community/eslint-utils" "^4.8.0" "@eslint-community/regexpp" "^4.12.2" "@eslint/config-array" "^0.23.5" - "@eslint/config-helpers" "^0.6.0" + "@eslint/config-helpers" "^0.7.0" "@eslint/core" "^1.2.1" - "@eslint/plugin-kit" "^0.7.2" + "@eslint/plugin-kit" "^0.7.3" "@humanfs/node" "^0.16.6" "@humanwhocodes/module-importer" "^1.0.1" "@humanwhocodes/retry" "^0.4.2" @@ -2155,14 +2193,14 @@ eslint@^10.5.0: esquery "^1.7.0" esutils "^2.0.2" fast-deep-equal "^3.1.3" - file-entry-cache "^8.0.0" + file-entry-cache "11.1.5 || >11.1.6 <12" find-up "^5.0.0" glob-parent "^6.0.2" ignore "^5.2.0" imurmurhash "^0.1.4" is-glob "^4.0.0" json-stable-stringify-without-jsonify "^1.0.1" - minimatch "^10.2.4" + minimatch "^10.2.5" natural-compare "^1.4.0" optionator "^0.9.3" @@ -2305,6 +2343,13 @@ fast-levenshtein@^2.0.6: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== +fastdom@1.0.12: + version "1.0.12" + resolved "https://registry.yarnpkg.com/fastdom/-/fastdom-1.0.12.tgz#ae43d55af017252ae499b2e186511ab97412de39" + integrity sha512-LB+xjSTEbjHE1cWsxu+tN2Xqr1kpi+V9aADI7sVM5ZMaXyYGPHULQMzpJMYqOTULK/73pUkWVzzObFRBkPr+hg== + dependencies: + strictdom "^1.0.1" + fault@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/fault/-/fault-2.0.1.tgz#d47ca9f37ca26e4bd38374a7c500b5a384755b6c" @@ -2329,19 +2374,12 @@ fecha@^4.2.0: resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd" integrity sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw== -file-entry-cache@^8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-8.0.0.tgz#7787bddcf1131bffb92636c69457bbc0edd6d81f" - integrity sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ== - dependencies: - flat-cache "^4.0.0" - -fill-range@^7.1.1: - version "7.1.1" - resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.1.1.tgz#44265d3cac07e3ea7dc247516380643754a05292" - integrity sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg== +"file-entry-cache@11.1.5 || >11.1.6 <12": + version "11.1.5" + resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-11.1.5.tgz#c8210eb055de63e68685ccfb6a017e386d4577d0" + integrity sha512-+PFTHITI08JIGhnNpGNI8T8inUpgZfk3GNEqfT9R2zZV2iFXg3CvqzSl/uEhs7TSGujYRELEANyDvS8Fj7+S7Q== dependencies: - to-regex-range "^5.0.1" + flat-cache "^6.1.23" find-up@^5.0.0: version "5.0.0" @@ -2351,18 +2389,19 @@ find-up@^5.0.0: locate-path "^6.0.0" path-exists "^4.0.0" -flat-cache@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-4.0.1.tgz#0ece39fcb14ee012f4b0410bd33dd9c1f011127c" - integrity sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw== +flat-cache@^6.1.23: + version "6.1.23" + resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-6.1.23.tgz#735dc888c271868d7301b3bbb8edba5028afb61d" + integrity sha512-f++BY9pTk+983xK1FLzlLpmM0i0z+jHmx3QESGkURMXujQZz1k5wzwX6hjnQ8goaD0B+sYnDK1yZ6MTyZfUaqA== dependencies: - flatted "^3.2.9" - keyv "^4.5.4" + cacheable "^2.5.0" + flatted "^3.4.2" + hookified "^1.15.0" -flatted@^3.2.9: - version "3.4.2" - resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.4.2.tgz#f5c23c107f0f37de8dbdf24f13722b3b98d52726" - integrity sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA== +flatted@^3.4.2: + version "3.4.4" + resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.4.4.tgz#aeeca2a506303f0cee61c59e6c9f2a88d2f29fc6" + integrity sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q== fn.name@1.x.x: version "1.1.0" @@ -2386,7 +2425,7 @@ forever-agent@~0.6.1: resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" integrity sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw== -form-data@^4.0.5, form-data@~4.0.4: +form-data@^4.0.6, form-data@~4.0.4: version "4.0.6" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.6.tgz#28e864e1b786dbebb68db1f452f9635278665827" integrity sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ== @@ -2407,15 +2446,6 @@ fraction.js@^5.3.4: resolved "https://registry.yarnpkg.com/fraction.js/-/fraction.js-5.3.4.tgz#8c0fcc6a9908262df4ed197427bdeef563e0699a" integrity sha512-1X1NTtiJphryn/uLQz3whtY6jK3fTqoE3ohKs0tT+Ujr1W59oopxmoEh7Lu5p6vBaPbgoM0bzveAW4Qi5RyWDQ== -fs-extra@^11.0.0: - version "11.3.3" - resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-11.3.3.tgz#a27da23b72524e81ac6c3815cc0179b8c74c59ee" - integrity sha512-VWSRii4t0AFm6ixFFmLLx1t7wS1gh+ckoa84aOeapGum0h+EZd1EhEumSB+ZdDLnEPuucsVB9oB7cxJHap6Afg== - dependencies: - graceful-fs "^4.2.0" - jsonfile "^6.0.1" - universalify "^2.0.0" - fs-extra@^9.1.0: version "9.1.0" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d" @@ -2426,32 +2456,25 @@ fs-extra@^9.1.0: jsonfile "^6.0.1" universalify "^2.0.0" -fsevents@2.3.2: - version "2.3.2" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" - integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== - -fsevents@~2.3.2: - version "2.3.3" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" - integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== - function-bind@^1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== function.prototype.name@^1.1.6, function.prototype.name@^1.1.8: - version "1.1.8" - resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.8.tgz#e68e1df7b259a5c949eeef95cdbde53edffabb78" - integrity sha512-e5iwyodOHhbMr/yNrc7fDYG4qlbIvI5gajyzPnb5TCwyhjApznQh1BMFou9b30SevY43gCJKXycoCBjMbsuW0Q== + version "1.2.0" + resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.2.0.tgz#758f3e84fa542672454bd5e14cb081a5ce07f70c" + integrity sha512-jObKIik1P2QjPHP5nz5BaOtUlfgS0fWo8IUByNXkM+o+02sJOi94em77GwJKQSJ3gfPHdgzLNrHc1uokV4P/ew== dependencies: - call-bind "^1.0.8" - call-bound "^1.0.3" - define-properties "^1.2.1" + call-bind "^1.0.9" + call-bound "^1.0.4" + es-define-property "^1.0.1" + es-errors "^1.3.0" functions-have-names "^1.2.3" - hasown "^2.0.2" + has-property-descriptors "^1.0.2" + hasown "^2.0.4" is-callable "^1.2.7" + is-document.all "^1.0.0" functions-have-names@^1.2.3: version "1.2.3" @@ -2536,13 +2559,6 @@ glob-parent@^6.0.2: dependencies: is-glob "^4.0.3" -glob-parent@~5.1.2: - version "5.1.2" - resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" - integrity sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow== - dependencies: - is-glob "^4.0.1" - glob@^13.0.6: version "13.0.6" resolved "https://registry.yarnpkg.com/glob/-/glob-13.0.6.tgz#078666566a425147ccacfbd2e332deb66a2be71d" @@ -2641,20 +2657,30 @@ hasha@5.2.2: is-stream "^2.0.0" type-fest "^0.8.0" -hasown@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" - integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== +hashery@^1.4.0, hashery@^1.5.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/hashery/-/hashery-1.5.1.tgz#4ba82ad54911ac617467870845d57a9fe508a400" + integrity sha512-iZyKG96/JwPz1N55vj2Ie2vXbhu440zfUfJvSwEqEbeLluk7NnapfGqa7LH0mOsnDxTF85Mx8/dyR6HfqcbmbQ== dependencies: - function-bind "^1.1.2" + hookified "^1.15.0" -hasown@^2.0.4: +hasown@^2.0.2, hasown@^2.0.3, hasown@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.4.tgz#8c62d8cb90beb2aad5d0a5b67581ad9854c3f003" integrity sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A== dependencies: function-bind "^1.1.2" +hookified@^1.15.0, hookified@^1.15.1: + version "1.15.1" + resolved "https://registry.yarnpkg.com/hookified/-/hookified-1.15.1.tgz#b1fafeaa5489cdc29cb85546a8f837ed4ffbbcb6" + integrity sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg== + +hookified@^2.1.1: + version "2.2.0" + resolved "https://registry.yarnpkg.com/hookified/-/hookified-2.2.0.tgz#1d024ac1668973dd5bcc4a96ab9ccdb7639ef8d4" + integrity sha512-p/LgFzRN5FeoD3DLS6bkUapeye6E4SI6yJs6KetENd18S+FBthqYq2amJUWpt5z0EQwwHemidjY5OqJGEKm5uA== + http-proxy-agent@^7.0.0, http-proxy-agent@^7.0.1: version "7.0.2" resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz#9a8b1f246866c028509486585f62b8f2c18c270e" @@ -2719,9 +2745,9 @@ ignore@^5.2.0: integrity sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g== ignore@^7.0.5: - version "7.0.5" - resolved "https://registry.yarnpkg.com/ignore/-/ignore-7.0.5.tgz#4cb5f6cd7d4c7ab0365738c7aea888baa6d7efd9" - integrity sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg== + version "7.0.9" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-7.0.9.tgz#475b2197ade916edab05ade35c691518e8543382" + integrity sha512-brTTsvFRt5C1gGHtPst/281UjPD5t9fBqbgoMPlVWy11ZLTPfu7HxK4ZYqO9H7o/yC9rSTCI85EaQ4OoY12qYw== import-fresh@^3.3.0: version "3.3.1" @@ -2770,10 +2796,10 @@ internmap@^1.0.0: resolved "https://registry.yarnpkg.com/internmap/-/internmap-1.0.1.tgz#0017cc8a3b99605f0302f2b198d272e015e5df95" integrity sha512-lDB5YccMydFBtasVtxnZ3MRBHuaoE8GKsppq+EchKL2U4nK/DmEpPHNH8MZe5HkMtpSiTSOZwfN0tzYjO/lJEw== -ip-address@^10.0.1: - version "10.4.0" - resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-10.4.0.tgz#c5910bc541b6eae287765d1e4846be0308a05d93" - integrity sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ== +ip-address@^10.1.1: + version "10.7.0" + resolved "https://registry.yarnpkg.com/ip-address/-/ip-address-10.7.0.tgz#9713429f16787ede8f642f6255b41fb515c825d9" + integrity sha512-BGFsyJd5mpXp3rK6jIdADLNgpJUK1jnjzvYF8lK+VyDab9JAmqN0YOKDdP17HlgKb2+ehPgDc8EtnRLbGCAMhA== is-array-buffer@^3.0.4, is-array-buffer@^3.0.5: version "3.0.5" @@ -2807,13 +2833,6 @@ is-bigint@^1.1.0: dependencies: has-bigints "^1.0.2" -is-binary-path@~2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09" - integrity sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw== - dependencies: - binary-extensions "^2.0.0" - is-boolean-object@^1.2.1: version "1.2.2" resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.2.2.tgz#7067f47709809a393c71ff5bb3e135d8a9215d9e" @@ -2827,12 +2846,12 @@ is-callable@^1.2.7: resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055" integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA== -is-core-module@^2.13.0, is-core-module@^2.16.1: - version "2.16.1" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.16.1.tgz#2a98801a849f43e2add644fbb6bc6229b19a4ef4" - integrity sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w== +is-core-module@^2.16.1, is-core-module@^2.16.2: + version "2.16.2" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.16.2.tgz#3e07450a8080ebce3fbf0cac494f4d2ab324e082" + integrity sha512-evOr8xfXKxE6qSR0hSXL2r3sd7ALj8+7jQEUvPYcm5sgZFdJ+AYzT6yNmJenvIYQBgIGwfwz08sL8zoL7yq2BA== dependencies: - hasown "^2.0.2" + hasown "^2.0.3" is-data-view@^1.0.1, is-data-view@^1.0.2: version "1.0.2" @@ -2843,7 +2862,7 @@ is-data-view@^1.0.1, is-data-view@^1.0.2: get-intrinsic "^1.2.6" is-typed-array "^1.1.13" -is-date-object@^1.0.5, is-date-object@^1.1.0: +is-date-object@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.1.0.tgz#ad85541996fc7aa8b2729701d27b7319f95d82f7" integrity sha512-PwwhEakHVKTdRNVOw+/Gyh0+MzlCl4R6qKvkhuvLtPMggI1WAHt9sOwZxQLSGpUaDnrdyDsomoRgNnCfKNSXXg== @@ -2851,6 +2870,13 @@ is-date-object@^1.0.5, is-date-object@^1.1.0: call-bound "^1.0.2" has-tostringtag "^1.0.2" +is-document.all@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/is-document.all/-/is-document.all-1.0.0.tgz#163a4bfb362c6ed7b118ce46cdecc4e37dee3195" + integrity sha512-+XSoyS05OdBbhFuELhgTCpFNHkpBOJqtsZfUFFpe5QTw+9Sjbh8zitxhQkYAo6wV7e1Vb8cAPvpCk9jGam/82g== + dependencies: + call-bound "^1.0.4" + is-extendable@^0.1.0: version "0.1.1" resolved "https://registry.yarnpkg.com/is-extendable/-/is-extendable-0.1.1.tgz#62b110e289a471418e3ec36a617d472e301dfc89" @@ -2891,7 +2917,7 @@ is-generator-function@^1.0.10: has-tostringtag "^1.0.2" safe-regex-test "^1.1.0" -is-glob@^4.0.0, is-glob@^4.0.1, is-glob@^4.0.3, is-glob@~4.0.1: +is-glob@^4.0.0, is-glob@^4.0.3: version "4.0.3" resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-4.0.3.tgz#64f61e42cbbb2eec2071a9dac0b28ba1e65d5084" integrity sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg== @@ -2924,11 +2950,6 @@ is-number-object@^1.1.1: call-bound "^1.0.3" has-tostringtag "^1.0.2" -is-number@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" - integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== - is-path-inside@^3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" @@ -2974,7 +2995,7 @@ is-string@^1.1.1: call-bound "^1.0.3" has-tostringtag "^1.0.2" -is-symbol@^1.0.4, is-symbol@^1.1.1: +is-symbol@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.1.1.tgz#f47761279f532e2b05a7024a7506dbbedacd0634" integrity sha512-9gGx6GTtCQM73BgmHQXfDmLtfjjTUDSyoxTCbp5WtoixAhfgsDirWIcVQ/IHpvI5Vgd5i/J5F7B9cN/WlVbC/w== @@ -3041,9 +3062,9 @@ jquery@^3.7.1: integrity sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg== js-cookie@^3.0.7: - version "3.0.7" - resolved "https://registry.yarnpkg.com/js-cookie/-/js-cookie-3.0.7.tgz#0a53abfc459c8e89c85d7a38eb6cb68714965b8c" - integrity sha512-z/wZZgDrkNV1eA0ULjM/F9/50Ya8fbzgKneSpoPsXSGd0KnpdtHfOZWK+GcwLk+EZbS4F9RBhU+K2RgzuDaItw== + version "3.0.8" + resolved "https://registry.yarnpkg.com/js-cookie/-/js-cookie-3.0.8.tgz#444e6f4b27a5d844594fef61c9d6bca5f0787688" + integrity sha512-yeJd4aNAdYZQjaon2bpD/Gb0B/omw7HQOsynXXcOiWVCacbBcPlgn8S/d1X6blFSaHao7ozqtW7NZW19xpCtIw== js-tokens@^4.0.0: version "4.0.0" @@ -3074,11 +3095,6 @@ jsdoc-type-pratt-parser@~4.1.0: resolved "https://registry.yarnpkg.com/jsdoc-type-pratt-parser/-/jsdoc-type-pratt-parser-4.1.0.tgz#ff6b4a3f339c34a6c188cbf50a16087858d22113" integrity sha512-Hicd6JK5Njt2QB6XYFS7ok9e37O8AYk3jTcppG4YVQnYjOemymvTcmc7OWsmq/Qqj5TdRFO5/x/tIPmBeRtGHg== -json-buffer@3.0.1: - version "3.0.1" - resolved "https://registry.yarnpkg.com/json-buffer/-/json-buffer-3.0.1.tgz#9338802a30d3b6605fbe0613e094008ca8c05a13" - integrity sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ== - json-parse-even-better-errors@^2.3.0: version "2.3.1" resolved "https://registry.yarnpkg.com/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz#7c47805a94319928e05777405dc12e1f7a4ee02d" @@ -3112,18 +3128,18 @@ json5@^1.0.2: minimist "^1.2.0" jsonfile@^6.0.1: - version "6.2.0" - resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.2.0.tgz#7c265bd1b65de6977478300087c99f1c84383f62" - integrity sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg== + version "6.2.1" + resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-6.2.1.tgz#b6e31717f22cc37330b081ce0051ed5de53af2f6" + integrity sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q== dependencies: universalify "^2.0.0" optionalDependencies: graceful-fs "^4.1.6" jsox@^1.2.119: - version "1.2.125" - resolved "https://registry.yarnpkg.com/jsox/-/jsox-1.2.125.tgz#33e68144b222db69dd494245dc72728ddb666f09" - integrity sha512-HIf1uwublnXZsy7p3yHTrhzMzrLO6xKnqXytT9pEil5QxaXi8eyer7Is4luF5hYSV4kD3v03Y32FWoAeVYTghQ== + version "1.2.128" + resolved "https://registry.yarnpkg.com/jsox/-/jsox-1.2.128.tgz#0e7ba27c63033248e9341f3802d6c9d3513c9af1" + integrity sha512-F73+D0uUYVLU+A0YqKMz7XSlX9kcoitt6NcU7azv3Fa6X4dt4+ihXfjcWUVJDcQQ874evDM1BQOHvRa338pxNA== jsprim@^2.0.2: version "2.0.2" @@ -3145,19 +3161,19 @@ jsx-ast-utils@^3.3.5: object.assign "^4.1.4" object.values "^1.1.6" -katex@^0.16.45: +katex@^0.16.47: version "0.16.47" resolved "https://registry.yarnpkg.com/katex/-/katex-0.16.47.tgz#0a13a42c2deb4f74e61f162d440b9165a548030f" integrity sha512-Eeo8Ys1doU1z+x8AZsPpQu+p/QcZBI5PeOo7QGQdy2x2m0MU/hYagBbGOmXwr5KVbEfVuWv9LpnQWeehogurjg== dependencies: commander "^8.3.0" -keyv@^4.5.4: - version "4.5.4" - resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93" - integrity sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw== +keyv@^5.6.0: + version "5.6.0" + resolved "https://registry.yarnpkg.com/keyv/-/keyv-5.6.0.tgz#03044074c6b4d072d0a62c7b9fa649537baf0105" + integrity sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw== dependencies: - json-buffer "3.0.1" + "@keyv/serialize" "^1.1.1" khroma@^2.1.0: version "2.1.0" @@ -3351,9 +3367,9 @@ longest-streak@^3.0.0: integrity sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g== lru-cache@^11.0.0: - version "11.2.6" - resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-11.2.6.tgz#356bf8a29e88a7a2945507b31f6429a65a192c58" - integrity sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ== + version "11.5.2" + resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-11.5.2.tgz#00e16665c90c620fba14a3c368732a976493f760" + integrity sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g== lru-cache@^7.14.1: version "7.18.3" @@ -3361,9 +3377,9 @@ lru-cache@^7.14.1: integrity sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA== lucide-static@^1.28.0: - version "1.28.0" - resolved "https://registry.yarnpkg.com/lucide-static/-/lucide-static-1.28.0.tgz#cf222b19159cdbfab424649bd255a6d794502cef" - integrity sha512-dC3VJwRFsjEVX7Iaq4rY88pm7Fi2OmOb8P0WRzXsUMgbt7sCmFX8bLhaDBeNW6JdRjuele+jKqqFaam4yr+Ygg== + version "1.43.0" + resolved "https://registry.yarnpkg.com/lucide-static/-/lucide-static-1.43.0.tgz#efc55edcd4f055121aa0a9e38b683041d66b156d" + integrity sha512-w7vdVFqh4vv7RxWHN4sSeylA0s3GHV5vpyGqZI0q22MINKJyhMNWSMd90RyTUYWBRNYqdZVZzrMmijVxU3Cl9A== markdown-link@^0.1.1: version "0.1.1" @@ -3526,25 +3542,26 @@ merge-stream@^2.0.0: integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w== mermaid@^11.16.1: - version "11.16.1" - resolved "https://registry.yarnpkg.com/mermaid/-/mermaid-11.16.1.tgz#57ae2342f6c45b967113b04c9258430bdd057ee8" - integrity sha512-TQsq6u22fAn3rek5VOubrhKPo1g5hwC3FXUN9hiyupTckcYiGuuKGkNQrKYwGJkXUxZdojwRG46gsSCFZMDp4g== + version "11.17.2" + resolved "https://registry.yarnpkg.com/mermaid/-/mermaid-11.17.2.tgz#e3caf3717582c0e44e5d04cff043e025caec7cba" + integrity sha512-V6K3C8EBdEsPFZXSKMJe6ppQOENxuHARr9GvHX4hh47lAbhMRD9qf4oEK7LoaRQxULMa80/qt5gHO73aCleBBg== dependencies: "@braintree/sanitize-url" "^7.1.2" "@iconify/utils" "^3.0.2" - "@mermaid-js/parser" "^1.2.0" + "@mermaid-js/parser" "^1.2.1" "@types/d3" "^7.4.3" "@upsetjs/venn.js" "^2.0.0" - cytoscape "^3.33.3" + cytoscape "^3.34.0" cytoscape-cose-bilkent "^4.1.0" cytoscape-fcose "^2.2.0" d3 "^7.9.0" d3-sankey "^0.12.3" dagre-d3-es "7.0.14" - dayjs "^1.11.20" + dayjs "^1.11.21" dompurify "^3.3.3" es-toolkit "^1.45.1" - katex "^0.16.45" + fastdom "1.0.12" + katex "^0.16.47" khroma "^2.1.0" marked "^16.3.0" roughjs "^4.6.6" @@ -3857,19 +3874,12 @@ mimic-function@^5.0.0: resolved "https://registry.yarnpkg.com/mimic-function/-/mimic-function-5.0.1.tgz#acbe2b3349f99b9deaca7fb70e48b83e94e67076" integrity sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA== -minimatch@^10.2.2: - version "10.2.4" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.2.4.tgz#465b3accbd0218b8281f5301e27cedc697f96fde" - integrity sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg== - dependencies: - brace-expansion "^5.0.2" - -minimatch@^10.2.4: - version "10.2.5" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.2.5.tgz#bd48687a0be38ed2961399105600f832095861d1" - integrity sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg== +minimatch@^10.2.2, minimatch@^10.2.4, minimatch@^10.2.5: + version "10.2.6" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-10.2.6.tgz#fd956bbe0b77241e9f15ac5dccb1c638060968ef" + integrity sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A== dependencies: - brace-expansion "^5.0.5" + brace-expansion "^5.0.8" minimatch@^3.1.2: version "3.1.5" @@ -3901,19 +3911,19 @@ mitt@^3.0.1: integrity sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw== moo@^0.5.0: - version "0.5.2" - resolved "https://registry.yarnpkg.com/moo/-/moo-0.5.2.tgz#f9fe82473bc7c184b0d32e2215d3f6e67278733c" - integrity sha512-iSAJLHYKnX41mKcJKjqvnAN9sf0LMDTXDEvFv+ffuRR9a1MIuXLjMNL6EsnDHSkKLTWNqQQ5uo61P4EbU4NU+Q== + version "0.5.3" + resolved "https://registry.yarnpkg.com/moo/-/moo-0.5.3.tgz#dfcdb40ff6f1a03c34af5df7f9717cecfa88c38c" + integrity sha512-m2fmM2dDm7GZQsY7KK2cme8agi+AAljILjQnof7p1ZMDe6dQ4bdnSMx0cPppudoeNv5hEFQirN6u+O4fDE0IWA== ms@^2.1.1, ms@^2.1.3: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== -nanoid@^3.3.16: - version "3.3.17" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.17.tgz#f1c3aa253c52547956a52c50bff754316f61037a" - integrity sha512-xQLf0A3HOMlgHq0n247/LRuAOYmB7dXJ/DvAxGvsSBij45XtBSmQycu+F8ODbHwns/XyFZagyL1+J0Offw1E0g== +nanoid@^3.3.18: + version "3.3.18" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.18.tgz#f66a2de1199ffde0fcf21c8a5f13106b1c081913" + integrity sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w== natural-compare@^1.4.0: version "1.4.0" @@ -3931,11 +3941,21 @@ nearley@^2.20.1: randexp "0.4.6" netmask@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/netmask/-/netmask-2.0.2.tgz#8b01a07644065d536383835823bc52004ebac5e7" - integrity sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg== + version "2.1.1" + resolved "https://registry.yarnpkg.com/netmask/-/netmask-2.1.1.tgz#80043d265b53aa521b3bd01e8fcdf353f9e1e81e" + integrity sha512-eonl3sLUha+S1GzTPxychyhnUzKyeQkZ7jLjKrBagJgPla13F+uQ71HgpFefyHgqrjEbCPkDArxYsjY8/+gLKA== -node-releases@^2.0.53: +node-exports-info@^1.6.0: + version "1.6.2" + resolved "https://registry.yarnpkg.com/node-exports-info/-/node-exports-info-1.6.2.tgz#243a3105677d6781cd26e6a72350fd928a5cb46f" + integrity sha512-kXs9Go0cah0qHVV2v389IXQLdLCeE1xfFtjOAF+iobu0OIoG1pje8At2vMHyaPMiPMnG/LWP50twML21eMcAag== + dependencies: + array.prototype.flatmap "^1.3.3" + es-errors "^1.3.0" + object.entries "^1.1.9" + semver "^6.3.1" + +node-releases@^2.0.54: version "2.0.54" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.54.tgz#09af17d5647aa9f221ec5cf2becb95b68a981afe" integrity sha512-YHs7BmmcsdAI5Ozuf8JZo6PT0mv2GIWC9vMfvUC3dp65M8hn7Ux8CPL+2oBI7juNuj9d0ndhTcznq2ODBps9cQ== @@ -3947,11 +3967,6 @@ node-sql-parser@^4.12.0: dependencies: big-integer "^1.6.48" -normalize-path@^3.0.0, normalize-path@~3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65" - integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA== - npm-run-path@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea" @@ -3981,6 +3996,16 @@ object.assign@^4.1.4, object.assign@^4.1.7: has-symbols "^1.1.0" object-keys "^1.1.1" +object.entries@^1.1.9: + version "1.1.9" + resolved "https://registry.yarnpkg.com/object.entries/-/object.entries-1.1.9.tgz#e4770a6a1444afb61bd39f984018b5bede25f8b3" + integrity sha512-8u/hfXFRBD1O0hPUjioLhoWFHRmt6tKA4/vZPyckBr18l1KE9uHrFaFaUi8MDRTpi4uak2goyPTSNJLXX2k2Hw== + dependencies: + call-bind "^1.0.8" + call-bound "^1.0.4" + define-properties "^1.2.1" + es-object-atoms "^1.1.1" + object.fromentries@^2.0.8: version "2.0.8" resolved "https://registry.yarnpkg.com/object.fromentries/-/object.fromentries-2.0.8.tgz#f7195d8a9b97bd95cbc1999ea939ecd1a2b00c65" @@ -4056,11 +4081,12 @@ ospath@^1.2.2: integrity sha512-o6E5qJV5zkAbIDNhGSIlyOhScKXgQrSRMilfph0clDfM0nEnBOlKlH4sWDmG95BW/CvwNz0vmm7dJVtU2KlMiA== own-keys@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/own-keys/-/own-keys-1.0.1.tgz#e4006910a2bf913585289676eebd6f390cf51358" - integrity sha512-qFOyK5PjiWZd+QQIh+1jhdb9LpxTF0qs7Pm8o5QHYZ0M3vKqSqzsZaEB6oWlxZ+q2sJBMI/Ktgd2N5ZwQoRHfg== + version "1.0.2" + resolved "https://registry.yarnpkg.com/own-keys/-/own-keys-1.0.2.tgz#31448ec1f781ecb1447f6f6aa0d6534222af59de" + integrity sha512-19YVAg7T+WTrxggPukVq7DjTv6+PJ867TmhCvBsYwmbFCsZd344rq2Ld1p0wo8f8Qrrhgp82c6FJRqdXWtSEhg== dependencies: - get-intrinsic "^1.2.6" + call-bound "^1.0.4" + get-intrinsic "^1.3.0" object-keys "^1.1.1" safe-push-apply "^1.0.0" @@ -4107,10 +4133,10 @@ pac-resolver@^7.0.1: degenerator "^5.0.0" netmask "^2.0.2" -package-manager-detector@^1.3.0: - version "1.6.0" - resolved "https://registry.yarnpkg.com/package-manager-detector/-/package-manager-detector-1.6.0.tgz#70d0cf0aa02c877eeaf66c4d984ede0be9130734" - integrity sha512-61A5ThoTiDG/C8s8UMZwSorAGwMJ0ERVGj2OjoW5pAalsNOg15+iQiPzrLJ4jhZ1HJzmC2PIHT2oEiH3R5fzNA== +package-manager-detector@^1.7.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/package-manager-detector/-/package-manager-detector-1.8.0.tgz#70c9a2c4bd1a513dcd6cad006a9fcebec22a1253" + integrity sha512-yQA4H19AmPEoMUeavPMDIe1higySl/gH/yaQrkT/s07Qp+7pp2hYz30N3z2l5BkjVkF9Ow6o0wjJamm2y7Sn0A== parent-module@^1.0.0: version "1.0.1" @@ -4184,17 +4210,17 @@ picocolors@^1.0.0, picocolors@^1.1.1: resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.1.1.tgz#3d321af3eab939b083c8f929a1d12cda81c26b6b" integrity sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA== -picomatch@2.3.2, picomatch@^2.0.4, picomatch@^2.2.1: +picomatch@2.3.2: version "2.3.2" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.2.tgz#5a942915e26b372dc0f0e6753149a16e6b1c5601" integrity sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA== -picomatch@4.0.4, picomatch@^4.0.3: +picomatch@4.0.4, picomatch@^4.0.4: version "4.0.4" resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-4.0.4.tgz#fd6f5e00a143086e074dffe4c924b8fb293b0589" integrity sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A== -pify@^2.2.0, pify@^2.3.0: +pify@^2.2.0: version "2.3.0" resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c" integrity sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog== @@ -4206,19 +4232,17 @@ pixelmatch@^6.0.0: dependencies: pngjs "^7.0.0" -playwright-core@1.58.2: - version "1.58.2" - resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.58.2.tgz#ac5f5b4b10d29bcf934415f0b8d133b34b0dcb13" - integrity sha512-yZkEtftgwS8CsfYo7nm0KE8jsvm6i/PTgVtB8DL726wNf6H2IMsDuxCpJj59KDaxCtSnrWan2AeDqM7JBaultg== +playwright-core@1.63.0: + version "1.63.0" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.63.0.tgz#e57665bc32846c213ac39a1e4d5bc6228e76b376" + integrity sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg== playwright@^1.58.1: - version "1.58.2" - resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.58.2.tgz#afe547164539b0bcfcb79957394a7a3fa8683cfd" - integrity sha512-vA30H8Nvkq/cPBnNw4Q8TWz1EJyqgpuinBcHET0YVJVFldr8JDNiU9LaWAE1KqSkRYazuaBhTpB5ZzShOezQ6A== + version "1.63.0" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.63.0.tgz#99b56f9f69b1b70c44f00bf84b2fe52348ae2511" + integrity sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg== dependencies: - playwright-core "1.58.2" - optionalDependencies: - fsevents "2.3.2" + playwright-core "1.63.0" pngjs@^7.0.0: version "7.0.0" @@ -4238,35 +4262,33 @@ points-on-path@^0.2.1: path-data-parser "0.1.0" points-on-curve "0.2.0" -possible-typed-array-names@^1.0.0: +possible-typed-array-names@^1.0.0, possible-typed-array-names@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz#93e3582bc0e5426586d9d07b79ee40fc841de4ae" integrity sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg== postcss-cli@>=9.1.0: - version "11.0.1" - resolved "https://registry.yarnpkg.com/postcss-cli/-/postcss-cli-11.0.1.tgz#341188ff7b26b19b206ca923ae2bd979751e7da7" - integrity sha512-0UnkNPSayHKRe/tc2YGW6XnSqqOA9eqpiRMgRlV1S6HdGi16vwJBx7lviARzbV1HpQHqLLRH3o8vTcB0cLc+5g== + version "12.0.0" + resolved "https://registry.yarnpkg.com/postcss-cli/-/postcss-cli-12.0.0.tgz#eedc5d69fd4f3b83de261b604e4f158c92754d10" + integrity sha512-57J2u1NuXuqGXftaK9JFEr/J3vtLzWpdN+RHs8GeeKUlvX7PU18VQJA57t0Rzf4CmbSfQApCwJIPrSFanQu5tQ== dependencies: - chokidar "^3.3.0" + chokidar "^5.0.0" dependency-graph "^1.0.0" - fs-extra "^11.0.0" picocolors "^1.0.0" - postcss-load-config "^5.0.0" + postcss-load-config "^6.0.0" postcss-reporter "^7.0.0" pretty-hrtime "^1.0.3" read-cache "^1.0.0" slash "^5.0.0" tinyglobby "^0.2.12" - yargs "^17.0.0" + yargs "^18.0.0" -postcss-load-config@^5.0.0: - version "5.1.0" - resolved "https://registry.yarnpkg.com/postcss-load-config/-/postcss-load-config-5.1.0.tgz#4ded23410da973e05edae9d41fa99bb5c1d5477f" - integrity sha512-G5AJ+IX0aD0dygOE0yFZQ/huFFMSNneyfp0e3/bT05a8OfPC5FUoZRPfGijUdGOJNMewJiwzcHJXFafFzeKFVA== +postcss-load-config@^6.0.0: + version "6.0.1" + resolved "https://registry.yarnpkg.com/postcss-load-config/-/postcss-load-config-6.0.1.tgz#6fd7dcd8ae89badcf1b2d644489cbabf83aa8096" + integrity sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g== dependencies: lilconfig "^3.1.1" - yaml "^2.4.2" postcss-reporter@^7.0.0: version "7.1.0" @@ -4282,11 +4304,11 @@ postcss-value-parser@^4.2.0: integrity sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ== postcss@>=8.5.23: - version "8.5.23" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.5.23.tgz#3493550116f478487298301d2c2e8dc5a56e6594" - integrity sha512-g50586zr4bZmwFiTlflMu8E0bDTb5I5gertgwAKmsdUlTQIhZtunzUlD1WSzwcVWPoAVpsrA6vlfCD7oXvRwgg== + version "8.5.28" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.5.28.tgz#da4563a99a06e62d6c1cd1acae363224bcaed6e9" + integrity sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A== dependencies: - nanoid "^3.3.16" + nanoid "^3.3.18" picocolors "^1.1.1" source-map-js "^1.2.1" @@ -4306,9 +4328,9 @@ prettier-plugin-sql@^0.18.0: tslib "^2.6.2" prettier@^3.2.5: - version "3.8.1" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.8.1.tgz#edf48977cf991558f4fcbd8a3ba6015ba2a3a173" - integrity sha512-UOnG6LftzbdaHZcKoPFtOcCKztrQ57WkHDeRD9t/PTQtmT0NHSeWWepj6pS0z/N7+08BHFDQVUrfmfMRcZwbMg== + version "3.9.6" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.9.6.tgz#b3ea5146515d40fc53f18aa63f74dfab1e10dbf6" + integrity sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g== pretty-bytes@^5.6.0: version "5.6.0" @@ -4360,9 +4382,9 @@ proxy-from-env@^2.1.0: integrity sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA== pump@^3.0.0: - version "3.0.3" - resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.3.tgz#151d979f1a29668dc0025ec589a455b53282268d" - integrity sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA== + version "3.0.4" + resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.4.tgz#1f313430527fa8b905622ebd22fe1444e757ab3c" + integrity sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA== dependencies: end-of-stream "^1.1.0" once "^1.3.1" @@ -4372,37 +4394,45 @@ punycode@^2.1.0: resolved "https://registry.yarnpkg.com/punycode/-/punycode-2.3.1.tgz#027422e2faec0b25e1549c3e1bd8309b9133b6e5" integrity sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg== -puppeteer-core@24.37.5: - version "24.37.5" - resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.37.5.tgz#b957f424717c13ff15765bc664a9b97808e5cbb4" - integrity sha512-ybL7iE78YPN4T6J+sPLO7r0lSByp/0NN6PvfBEql219cOnttoTFzCWKiBOjstXSqi/OKpwae623DWAsL7cn2MQ== +puppeteer-core@24.43.1: + version "24.43.1" + resolved "https://registry.yarnpkg.com/puppeteer-core/-/puppeteer-core-24.43.1.tgz#c10a5d398b69911c324e04c85ee2b236b9ec940e" + integrity sha512-T5ScUMAsmhdNbgDR41AGESYeS6V9MSgetkSnVhhW+gXvzC42VesKCn5ld87gAZDJ6vLHL9GkRvY9WtQWSnwFbw== dependencies: - "@puppeteer/browsers" "2.13.0" + "@puppeteer/browsers" "2.13.2" chromium-bidi "14.0.0" debug "^4.4.3" - devtools-protocol "0.0.1566079" - typed-query-selector "^2.12.0" + devtools-protocol "0.0.1608973" + typed-query-selector "^2.12.2" webdriver-bidi-protocol "0.4.1" - ws "^8.19.0" + ws "^8.20.0" puppeteer@^24.35.0: - version "24.37.5" - resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-24.37.5.tgz#c61932cdb2bc53d18970671be18d547216d295f0" - integrity sha512-3PAOIQLceyEmn1Fi76GkGO2EVxztv5OtdlB1m8hMUZL3f8KDHnlvXbvCXv+Ls7KzF1R0KdKBqLuT/Hhrok12hQ== + version "24.43.1" + resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-24.43.1.tgz#86cad9170ce2dec2db3b8d1d34c0c91424bbe3e3" + integrity sha512-/FSOViCrqRdb1HDocpsM9Z1giA71gTQPUt3SpHGVRALKAy/rJr1fLFYZW9F23qPxqVxTHQnbh/5B5opJST3kAw== dependencies: - "@puppeteer/browsers" "2.13.0" + "@puppeteer/browsers" "2.13.2" chromium-bidi "14.0.0" cosmiconfig "^9.0.0" - devtools-protocol "0.0.1566079" - puppeteer-core "24.37.5" - typed-query-selector "^2.12.0" + devtools-protocol "0.0.1608973" + puppeteer-core "24.43.1" + typed-query-selector "^2.12.2" + +qified@^0.10.1: + version "0.10.1" + resolved "https://registry.yarnpkg.com/qified/-/qified-0.10.1.tgz#0640bf21bbe6ca540db290ad8f5fde4d870b8bda" + integrity sha512-+Owyggi9IxT1ePKGafcI87ubSmxol6smwJ+RAHDQlx9+9cPwFWDiKFFCPuWhr9ignlGpZ9vDQLw67N4dcTVFEA== + dependencies: + hookified "^2.1.1" qs@^6.15.2: - version "6.15.2" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.15.2.tgz#fd55426d710403ddccc45e0f9eab16db7727ece9" - integrity sha512-Rzq0KEyX/w/tEybncDgdkZrJgVUsUMk3xjh3t5bv3S1HTAtg+uOYt72+ZfwiQwKdysThkTBdL/rTi6HDmX9Ddw== + version "6.16.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.16.0.tgz#c22c723a28a920f3aacdce8289fabd43eccb79fd" + integrity sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA== dependencies: - side-channel "^1.1.0" + es-define-property "^1.0.1" + side-channel "^1.1.1" railroad-diagrams@^1.0.0: version "1.0.0" @@ -4425,11 +4455,9 @@ randombytes@^2.1.0: safe-buffer "^5.1.0" read-cache@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/read-cache/-/read-cache-1.0.0.tgz#e664ef31161166c9751cdbe8dbcf86b5fb58f774" - integrity sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA== - dependencies: - pify "^2.3.0" + version "1.0.2" + resolved "https://registry.yarnpkg.com/read-cache/-/read-cache-1.0.2.tgz#3bfa40e27574216bd893276ced75d81e6649ca52" + integrity sha512-/peqiBB/n07gQGLsWaHho3WfvUyRscw0gYTsEFMhrIe/nWLkYaf5SbKYjGYqtRV3aPwykJgF2VEMo1ac4bnsGA== readable-stream@^3.4.0, readable-stream@^3.6.2: version "3.6.2" @@ -4440,14 +4468,12 @@ readable-stream@^3.4.0, readable-stream@^3.6.2: string_decoder "^1.1.1" util-deprecate "^1.0.1" -readdirp@~3.6.0: - version "3.6.0" - resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" - integrity sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA== - dependencies: - picomatch "^2.2.1" +readdirp@^5.0.0: + version "5.1.1" + resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-5.1.1.tgz#520bca06f9d1ae1b96cc0800dbe84b983d19422c" + integrity sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA== -reflect.getprototypeof@^1.0.6, reflect.getprototypeof@^1.0.9: +reflect.getprototypeof@^1.0.10, reflect.getprototypeof@^1.0.9: version "1.0.10" resolved "https://registry.yarnpkg.com/reflect.getprototypeof/-/reflect.getprototypeof-1.0.10.tgz#c629219e78a3316d8b604c765ef68996964e7bf9" integrity sha512-00o4I+DVrefhv+nX0ulyi3biSHCPDe+yLv5o/p6d/UVlirijB8E16FtfwSAi4g3tcqrQ4lRAqQSoFEZJehYEcw== @@ -4541,12 +4567,15 @@ resolve-from@^4.0.0: resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== -resolve@^1.22.4: - version "1.22.11" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.11.tgz#aad857ce1ffb8bfa9b0b1ac29f1156383f68c262" - integrity sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ== +resolve@^2.0.0-next.6: + version "2.0.0-next.7" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-2.0.0-next.7.tgz#ba3b035d4b1ee7c522426eee73cabcb0fd5515dd" + integrity sha512-tqt+NBWwyaMgw3zDsnygx4CByWjQEJHOPMdslYhppaQSJUtL/D4JO9CcBBlhPoI8lz9oJIDXkwXfhF4aWqP8xQ== dependencies: - is-core-module "^2.16.1" + es-errors "^1.3.0" + is-core-module "^2.16.2" + node-exports-info "^1.6.0" + object-keys "^1.1.1" path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" @@ -4569,9 +4598,9 @@ rfdc@^1.4.1: integrity sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA== robust-predicates@^3.0.2: - version "3.0.2" - resolved "https://registry.yarnpkg.com/robust-predicates/-/robust-predicates-3.0.2.tgz#d5b28528c4824d20fc48df1928d41d9efa1ad771" - integrity sha512-IXgzBWvWQwE6PrDI05OvmXUIruQTcoMDzRsOd5CDvHCVLcLHMTSYvOK5Cm46kWqlV3yAbuSpBZdJ5oP5OUoStg== + version "3.0.3" + resolved "https://registry.yarnpkg.com/robust-predicates/-/robust-predicates-3.0.3.tgz#1099061b3349e2c5abec6c2ab0acd440d24d4062" + integrity sha512-NS3levdsRIUOmiJ8FZWCP7LG3QpJyrs/TE0Zpf1yvZu8cAJJ6QMW92H1c7kWpdIHo8RvmLxN/o2JXTKHp74lUA== roughjs@^4.6.6: version "4.6.6" @@ -4589,13 +4618,13 @@ rw@1: integrity sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ== safe-array-concat@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.1.3.tgz#c9e54ec4f603b0bbb8e7e5007a5ee7aecd1538c3" - integrity sha512-AURm5f0jYEOydBj7VQlVvDrjeFgthDdEF5H1dP+6mNpoXOMo1quQqJ4wvJDyRZ9+pO3kGWoOdmV08cSv2aJV6Q== + version "1.1.4" + resolved "https://registry.yarnpkg.com/safe-array-concat/-/safe-array-concat-1.1.4.tgz#a54cc9b61a57f33b42abad3cbdda3a2b38cc5719" + integrity sha512-wtZlHyOje6OZTGqAoaDKxFkgRtkF9CnHAVnCHKfuj200wAgL+bSJhdsCD2l0Qx/2ekEXjPWcyKkfGb5CPboslg== dependencies: - call-bind "^1.0.8" - call-bound "^1.0.2" - get-intrinsic "^1.2.6" + call-bind "^1.0.9" + call-bound "^1.0.4" + get-intrinsic "^1.3.0" has-symbols "^1.1.0" isarray "^2.0.5" @@ -4645,9 +4674,9 @@ semver@^6.3.1: integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== semver@^7.7.2, semver@^7.7.3, semver@^7.7.4: - version "7.7.4" - resolved "https://registry.yarnpkg.com/semver/-/semver-7.7.4.tgz#28464e36060e991fa7a11d0279d2d3f3b57a7e8a" - integrity sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA== + version "7.8.5" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.8.5.tgz#39b646037dd50c14fb451e7e4cac58ed8b863f69" + integrity sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA== serialize-javascript@^6.0.2: version "6.0.2" @@ -4699,13 +4728,13 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== -side-channel-list@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/side-channel-list/-/side-channel-list-1.0.0.tgz#10cb5984263115d3b7a0e336591e290a830af8ad" - integrity sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA== +side-channel-list@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/side-channel-list/-/side-channel-list-1.0.1.tgz#c2e0b5a14a540aebee3bbc6c3f8666cc9b509127" + integrity sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w== dependencies: es-errors "^1.3.0" - object-inspect "^1.13.3" + object-inspect "^1.13.4" side-channel-map@^1.0.1: version "1.0.1" @@ -4728,14 +4757,14 @@ side-channel-weakmap@^1.0.2: object-inspect "^1.13.3" side-channel-map "^1.0.1" -side-channel@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.1.0.tgz#c3fcff9c4da932784873335ec9765fa94ff66bc9" - integrity sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw== +side-channel@^1.1.0, side-channel@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.1.1.tgz#ea02c62e05dc4bea67d4442f0fb71ee192f8e0ab" + integrity sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ== dependencies: es-errors "^1.3.0" - object-inspect "^1.13.3" - side-channel-list "^1.0.0" + object-inspect "^1.13.4" + side-channel-list "^1.0.1" side-channel-map "^1.0.1" side-channel-weakmap "^1.0.2" @@ -4785,11 +4814,11 @@ socks-proxy-agent@^8.0.5: socks "^2.8.3" socks@^2.8.3: - version "2.8.7" - resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.7.tgz#e2fb1d9a603add75050a2067db8c381a0b5669ea" - integrity sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A== + version "2.8.10" + resolved "https://registry.yarnpkg.com/socks/-/socks-2.8.10.tgz#23aab4dccc1fdae110aaf4fa51cd190552432dc5" + integrity sha512-e0VyvkVTwVYViNovRkZ9aodhxVlyoMn7eJhVUPxZ+eK9P/7CBkxvvsBOHqFPEH416726W8tLXXXjKwqgTErrCQ== dependencies: - ip-address "^10.0.1" + ip-address "^10.1.1" smart-buffer "^4.2.0" source-map-js@^1.2.1: @@ -4821,9 +4850,9 @@ spdx-license-ids@^3.0.0: integrity sha512-CWLcCCH7VLu13TgOH+r8p1O/Znwhqv/dbb6lqWy67G+pT1kHmeD/+V36AVb/vq8QMIQwVShJ6Ssl5FPh0fuSdw== sql-formatter@^15.0.2: - version "15.7.2" - resolved "https://registry.yarnpkg.com/sql-formatter/-/sql-formatter-15.7.2.tgz#d385a4f9ddf95b7916c66286c93374b1417bc70b" - integrity sha512-b0BGoM81KFRVSpZFwPpIPU5gng4YD8DI/taLD96NXCFRf5af3FzSE4aSwjKmxcyTmf/MfPu91j75883nRrWDBw== + version "15.8.2" + resolved "https://registry.yarnpkg.com/sql-formatter/-/sql-formatter-15.8.2.tgz#ac0684864b3ac850e455dc915adbca78dd51aeb8" + integrity sha512-kTYRg5FIcvsDtYUG2Qn9pYT6xKwiLJN5TTIvc5Mur6hIg4pSfdpHu8Yyu5bqESLHnVM3mXzD446cb2+uEaKZXg== dependencies: argparse "^2.0.1" nearley "^2.20.1" @@ -4856,15 +4885,20 @@ stop-iteration-iterator@^1.1.0: es-errors "^1.3.0" internal-slot "^1.1.0" -streamx@^2.12.5, streamx@^2.15.0, streamx@^2.21.0: - version "2.23.0" - resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.23.0.tgz#7d0f3d00d4a6c5de5728aecd6422b4008d66fd0b" - integrity sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg== +streamx@^2.12.5, streamx@^2.15.0, streamx@^2.25.0: + version "2.28.1" + resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.28.1.tgz#376cd42a089505a69bec0efdb5189aa45586141f" + integrity sha512-zEzXb0s5Cds7tqMH6rhZ05lcJydCWiQPEwiNngVqzsxCc962vLY4Uw+mW7od8kDH258k2Uz/JrOkdIAAhSh9VA== dependencies: events-universal "^1.0.0" fast-fifo "^1.3.2" text-decoder "^1.1.0" +strictdom@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/strictdom/-/strictdom-1.0.1.tgz#189de91649f73d44d59b8432efa68ef9d2659460" + integrity sha512-cEmp9QeXXRmjj/rVp9oyiqcvyocWab/HaoN4+bwFeZ7QzykJD6L3yD4v12K1x0tHpqRqVpJevN3gW7kyM39Bqg== + string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" @@ -4874,7 +4908,7 @@ string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: is-fullwidth-code-point "^3.0.0" strip-ansi "^6.0.1" -string-width@^7.0.0: +string-width@^7.0.0, string-width@^7.2.0: version "7.2.0" resolved "https://registry.yarnpkg.com/string-width/-/string-width-7.2.0.tgz#b5bb8e2165ce275d4d43476dd2700ad9091db6dc" integrity sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ== @@ -4883,10 +4917,10 @@ string-width@^7.0.0: get-east-asian-width "^1.0.0" strip-ansi "^7.1.0" -string-width@^8.2.0: - version "8.2.1" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-8.2.1.tgz#165089cfa527cc88fbc23dd73313f5e334af1ea1" - integrity sha512-IIaP0g3iy9Cyy18w3M9YcaDudujEAVHKt3a3QJg1+sr/oX96TbaGUubG0hJyCjCBThFH+tFpcIyoUHUn1ogaLA== +string-width@^8.2.0, string-width@^8.2.1: + version "8.2.2" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-8.2.2.tgz#7310516493df575742fe98af6fae87d85d5ed0ac" + integrity sha512-GaPUh5gfdrYzqeVNZvUfT23vYYxXzKYidUcnMtJg/3rxRV63EFZy3k6xfKlmfeJD0176lnUV/Usr3XcwSvFzpg== dependencies: get-east-asian-width "^1.5.0" strip-ansi "^7.1.2" @@ -4901,27 +4935,28 @@ string.prototype.includes@^2.0.1: es-abstract "^1.23.3" string.prototype.trim@^1.2.10: - version "1.2.10" - resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.2.10.tgz#40b2dd5ee94c959b4dcfb1d65ce72e90da480c81" - integrity sha512-Rs66F0P/1kedk5lyYyH9uBzuiI/kNRmwJAR9quK6VOtIpZ2G+hMZd+HQbbv25MgCA6gEffoMZYxlTod4WcdrKA== + version "1.2.11" + resolved "https://registry.yarnpkg.com/string.prototype.trim/-/string.prototype.trim-1.2.11.tgz#e6bd19cda3985d05a42dda31f3ddf4d35d3430e3" + integrity sha512-PwvK7BU+CMTJGYQCTZb5RWXIML92lftJLhQz1tBzgKiqGxJaMlBAa48POXaNAC2s4y8jr3EFqrkF9+44neS46w== dependencies: - call-bind "^1.0.8" - call-bound "^1.0.2" + call-bind "^1.0.9" + call-bound "^1.0.4" define-data-property "^1.1.4" define-properties "^1.2.1" - es-abstract "^1.23.5" - es-object-atoms "^1.0.0" + es-abstract "^1.24.2" + es-object-atoms "^1.1.2" has-property-descriptors "^1.0.2" + safe-regex-test "^1.1.0" string.prototype.trimend@^1.0.9: - version "1.0.9" - resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.9.tgz#62e2731272cd285041b36596054e9f66569b6942" - integrity sha512-G7Ok5C6E/j4SGfyLCloXTrngQIQU3PWtXGst3yM7Bea9FRURf1S42ZHlZZtsNque2FN2PoUhfZXYLNWwEr4dLQ== + version "1.0.10" + resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.10.tgz#be6bcf4f3fe0460bdeccdb2cf4f971b310f8346e" + integrity sha512-2+3aDAOmPTmuFwjDnmJG2ctEkQKVki7vOSqaxkv42Mowj1V6PnvuwFCRrR5lChUux1TBskPjfkeTOhqczDMxTw== dependencies: - call-bind "^1.0.8" - call-bound "^1.0.2" + call-bind "^1.0.9" + call-bound "^1.0.4" define-properties "^1.2.1" - es-object-atoms "^1.0.0" + es-object-atoms "^1.1.2" string.prototype.trimstart@^1.0.8: version "1.0.8" @@ -4969,9 +5004,9 @@ strip-final-newline@^2.0.0: integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA== stylis@^4.3.6: - version "4.3.6" - resolved "https://registry.yarnpkg.com/stylis/-/stylis-4.3.6.tgz#7c7b97191cb4f195f03ecab7d52f7902ed378320" - integrity sha512-yQ3rwFWRfwNUY7H5vpU0wfdkNSnvnJinhF9830Swlaxl03zsOjCfmX0ugac+3LtK0lYSgwL/KXc8oYL3mG4YFQ== + version "4.4.0" + resolved "https://registry.yarnpkg.com/stylis/-/stylis-4.4.0.tgz#c5846c9345f4bfc51bd0cbd7ca35a0744f485a5d" + integrity sha512-5Z9ZpRzfuH6l/UAvCPAPUo3665Nk2wLaZU3x+TLHKVzIz33+sbJqbtrYoC3KD4/uVOr2Zp+L0LySezP9OHV9yA== supports-color@^7.1.0: version "7.2.0" @@ -4993,14 +5028,14 @@ supports-preserve-symlinks-flag@^1.0.0: integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== systeminformation@^5.31.1: - version "5.31.7" - resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-5.31.7.tgz#32009a8790af048299ea46d01e9f306adc1abb45" - integrity sha512-/8NC53e5nP9nmhn42/ncdOkyJnOoue/Vy+tJOyUGd1Yv66G069wK4rrziwhrqDETgk78CudTQupw5z19S5uoZw== + version "5.33.10" + resolved "https://registry.yarnpkg.com/systeminformation/-/systeminformation-5.33.10.tgz#5de4f2c627f2e27e14d528e28417b6d2ba475fac" + integrity sha512-/NXbMVASt2UbSVgmWto4aBTIAeuYY7zOELpZybmNxqpsrbo5uYHpuEPf5nkyrPng0uG5YOb+Yv6s0FyKY4mDQg== tar-fs@^3.1.1: - version "3.1.1" - resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.1.1.tgz#4f164e59fb60f103d472360731e8c6bb4a7fe9ef" - integrity sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg== + version "3.1.3" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.1.3.tgz#05668cc68a30741c3813f9c16593b8dec7dcbcd1" + integrity sha512-/hU4AXnIdZu+Gvl1pk0oI5f5HxWsCJRtY2aFaJdk9VvyL48DWU6iU5WAIPG+wIi1YvWA6eTJvIviP/tMAZZNwQ== dependencies: pump "^3.0.0" tar-stream "^3.1.5" @@ -5009,11 +5044,12 @@ tar-fs@^3.1.1: bare-path "^3.0.0" tar-stream@^3.1.5: - version "3.1.7" - resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-3.1.7.tgz#24b3fb5eabada19fe7338ed6d26e5f7c482e792b" - integrity sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ== + version "3.2.1" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-3.2.1.tgz#952d72f7aba68ce5cb802ef2e0b19f1bd989140d" + integrity sha512-nqsEO8zLZJvrOMdEwkA0QdCLFbetHMn95Zqu4fKwX+hkaTWJPZZOrxx/PwtxoK0MMGQmBQNRW3CPs8IFYQz4cQ== dependencies: b4a "^1.6.4" + bare-fs "^4.5.5" fast-fifo "^1.2.0" streamx "^2.15.0" @@ -5048,27 +5084,27 @@ text-hex@1.0.x: integrity sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg== thenby@^1.3.4: - version "1.3.4" - resolved "https://registry.yarnpkg.com/thenby/-/thenby-1.3.4.tgz#81581f6e1bb324c6dedeae9bfc28e59b1a2201cc" - integrity sha512-89Gi5raiWA3QZ4b2ePcEwswC3me9JIg+ToSgtE0JWeCynLnLxNr/f9G+xfo9K+Oj4AFdom8YNJjibIARTJmapQ== + version "1.4.1" + resolved "https://registry.yarnpkg.com/thenby/-/thenby-1.4.1.tgz#5b2bcf5c323038274712ebec9e2c681c915c9d98" + integrity sha512-D5a/bO0KdalOE3q8MlrRmSxjbKZHT3MQmXkJP+r97Vw8MMwOZKOwUSEyTtK7eSMj2y0kyAjpYMRMZmmLw1FtNQ== throttleit@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/throttleit/-/throttleit-1.0.1.tgz#304ec51631c3b770c65c6c6f76938b384000f4d5" integrity sha512-vDZpf9Chs9mAdfY046mcPt8fg5QSZr37hEH4TXYBnDF+izxgrbRGUAAaBvIk/fJm9aOFCGFd1EsNg5AZCbnQCQ== -tinyexec@^1.0.1: - version "1.0.2" - resolved "https://registry.yarnpkg.com/tinyexec/-/tinyexec-1.0.2.tgz#bdd2737fe2ba40bd6f918ae26642f264b99ca251" - integrity sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg== +tinyexec@^1.2.4: + version "1.3.1" + resolved "https://registry.yarnpkg.com/tinyexec/-/tinyexec-1.3.1.tgz#16a2e3c6e23fafce72640e678e651db36145b9c6" + integrity sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA== tinyglobby@^0.2.12, tinyglobby@^0.2.15: - version "0.2.15" - resolved "https://registry.yarnpkg.com/tinyglobby/-/tinyglobby-0.2.15.tgz#e228dd1e638cea993d2fdb4fcd2d4602a79951c2" - integrity sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ== + version "0.2.17" + resolved "https://registry.yarnpkg.com/tinyglobby/-/tinyglobby-0.2.17.tgz#562a9a6c9eb2b3b123d39719f9af5bb44fcd7631" + integrity sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g== dependencies: fdir "^6.5.0" - picomatch "^4.0.3" + picomatch "^4.0.4" tldts-core@^6.1.86: version "6.1.86" @@ -5087,13 +5123,6 @@ tmp@~0.2.4: resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.2.7.tgz#26f4db11d1601ce8012dcb8a798ece1c06a99059" integrity sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw== -to-regex-range@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/to-regex-range/-/to-regex-range-5.0.1.tgz#1648c44aae7c8d988a326018ed72f5b4dd0392e4" - integrity sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ== - dependencies: - is-number "^7.0.0" - tough-cookie@^5.0.0: version "5.1.2" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-5.1.2.tgz#66d774b4a1d9e12dc75089725af3ac75ec31bed7" @@ -5122,9 +5151,9 @@ ts-api-utils@^2.5.0: integrity sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA== ts-dedent@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/ts-dedent/-/ts-dedent-2.2.0.tgz#39e4bd297cd036292ae2394eb3412be63f563bb5" - integrity sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ== + version "2.3.0" + resolved "https://registry.yarnpkg.com/ts-dedent/-/ts-dedent-2.3.0.tgz#8fac36c7902b541c154ac13a27ac467997af11f8" + integrity sha512-JfJeIHke7y2egdGGgRAvpCwYFUsHlM2gPcrVOxFkznt/4uzQ7HFmvE63iFHVLBJNDuyDOQgijDK/tXH/f6Msjg== tsconfig-paths@^3.15.0: version "3.15.0" @@ -5136,11 +5165,6 @@ tsconfig-paths@^3.15.0: minimist "^1.2.6" strip-bom "^3.0.0" -tslib@1.14.1: - version "1.14.1" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" - integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== - tslib@^2.0.1, tslib@^2.6.2: version "2.8.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.8.1.tgz#612efe4ed235d567e8aba5f2a5fab70280ade83f" @@ -5204,31 +5228,31 @@ typed-array-byte-offset@^1.0.4: reflect.getprototypeof "^1.0.9" typed-array-length@^1.0.7: - version "1.0.7" - resolved "https://registry.yarnpkg.com/typed-array-length/-/typed-array-length-1.0.7.tgz#ee4deff984b64be1e118b0de8c9c877d5ce73d3d" - integrity sha512-3KS2b+kL7fsuk/eJZ7EQdnEmQoaho/r6KUef7hxvltNA5DR8NAUM+8wJMbJyZ4G9/7i3v5zPBIMN5aybAh2/Jg== + version "1.0.8" + resolved "https://registry.yarnpkg.com/typed-array-length/-/typed-array-length-1.0.8.tgz#0b70e982c9e9dafe2def6d6458ff4b3f2d2b6d70" + integrity sha512-phPGCwqr2+Qo0fwniCE8e4pKnGu/yFb5nD5Y8bf0EEeiI5GklnACYA9GFy/DrAeRrKHXvHn+1SUsOWgJp6RO+g== dependencies: - call-bind "^1.0.7" - for-each "^0.3.3" - gopd "^1.0.1" - is-typed-array "^1.1.13" - possible-typed-array-names "^1.0.0" - reflect.getprototypeof "^1.0.6" + call-bind "^1.0.9" + for-each "^0.3.5" + gopd "^1.2.0" + is-typed-array "^1.1.15" + possible-typed-array-names "^1.1.0" + reflect.getprototypeof "^1.0.10" -typed-query-selector@^2.12.0: - version "2.12.1" - resolved "https://registry.yarnpkg.com/typed-query-selector/-/typed-query-selector-2.12.1.tgz#04423bfb71b8f3aee3df1c29598ed6c7c8f55284" - integrity sha512-uzR+FzI8qrUEIu96oaeBJmd9E7CFEiQ3goA5qCVgc4s5llSubcfGHq9yUstZx/k4s9dXHVKsE35YWoFyvEqEHA== +typed-query-selector@^2.12.2: + version "2.12.2" + resolved "https://registry.yarnpkg.com/typed-query-selector/-/typed-query-selector-2.12.2.tgz#65e2462ac6b0aecfae1bfac1a4f3027070dbabaa" + integrity sha512-EOPFbyIub4ngnEdqi2yOcNeDLaX/0jcE1JoAXQDDMIthap7FoN795lc/SHfIq2d416VufXpM8z/lD+WRm2gfOQ== typescript-eslint@^8.61.1: - version "8.61.1" - resolved "https://registry.yarnpkg.com/typescript-eslint/-/typescript-eslint-8.61.1.tgz#7c224a9a643b7f42d295c67a75c1e30fee8c3eaa" - integrity sha512-V7PayAfJokV3pEHgN7/v03D1SpujhRfQtYLbLIiBfDDncdg4PAiRBfoS4cnCANK4jmAPncczi59QO3afiXUlNw== + version "8.70.0" + resolved "https://registry.yarnpkg.com/typescript-eslint/-/typescript-eslint-8.70.0.tgz#6c90c532079132ba7d4a386bf0882c56d82fe5af" + integrity sha512-P/W5cz70/cQAuKfY3xwQMWWTV7BvJ0mAQmi+9mBcsVPaBUpd6Ohpa+fECv9rBFrQcig86jAiNBFNWUqnTjr4pw== dependencies: - "@typescript-eslint/eslint-plugin" "8.61.1" - "@typescript-eslint/parser" "8.61.1" - "@typescript-eslint/typescript-estree" "8.61.1" - "@typescript-eslint/utils" "8.61.1" + "@typescript-eslint/eslint-plugin" "8.70.0" + "@typescript-eslint/parser" "8.70.0" + "@typescript-eslint/typescript-estree" "8.70.0" + "@typescript-eslint/utils" "8.70.0" typescript@^5.8.3: version "5.9.3" @@ -5245,10 +5269,10 @@ unbox-primitive@^1.1.0: has-symbols "^1.1.0" which-boxed-primitive "^1.1.1" -undici-types@~7.18.0: - version "7.18.2" - resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-7.18.2.tgz#29357a89e7b7ca4aef3bf0fd3fd0cd73884229e9" - integrity sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w== +undici-types@~8.9.0: + version "8.9.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-8.9.0.tgz#e240d97c8b5d85e5347ce73d25865c7906c1ec9f" + integrity sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg== unified@^11.0.0, unified@^11.0.5: version "11.0.5" @@ -5304,7 +5328,7 @@ untildify@^4.0.0: resolved "https://registry.yarnpkg.com/untildify/-/untildify-4.0.0.tgz#2bc947b953652487e4600949fb091e3ae8cd919b" integrity sha512-KK8xQ1mkzZeg9inewmFVDNkg3l5LUhoq9kN6iWYB/CC9YMG8HA+c1Q8HwDe6dEX7kErrEVNVBO3fWsVq5iDgtw== -update-browserslist-db@^1.3.0: +update-browserslist-db@^1.3.2: version "1.3.2" resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.3.2.tgz#9d99fbff56c50bb11ba5fd35cece5916da595836" integrity sha512-UQ+MSxlhRm1bzjhU+DcuXfjFO1FzNtqhK5+9Yvlp90ItDLk5vT932A0rFu619nf7RVS+Y/VeaUW1jaRDqZ8VJw== @@ -5325,9 +5349,9 @@ util-deprecate@^1.0.1: integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== "uuid@^11.1.0 || ^12 || ^13 || ^14.0.0": - version "14.0.0" - resolved "https://registry.yarnpkg.com/uuid/-/uuid-14.0.0.tgz#0af883220163d264ffe0c084f6b8a89b9666966d" - integrity sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg== + version "14.0.2" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-14.0.2.tgz#d5ae03e4db0881c87271f8b0b9bd7312d02c799a" + integrity sha512-xZe/16rV4aa+HGSOCiY2YeLT1OybRLrrkL/Rqaq7p7GMVXjFh+6wN4oMYgjFmnSnhY8t6Xpdl2l9qmnHYuMHwQ== vanillajs-datepicker@^1.3.4: version "1.3.4" @@ -5405,12 +5429,12 @@ which-collection@^1.0.2: is-weakset "^2.0.3" which-typed-array@^1.1.16, which-typed-array@^1.1.19: - version "1.1.20" - resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.20.tgz#3fdb7adfafe0ea69157b1509f3a1cd892bd1d122" - integrity sha512-LYfpUkmqwl0h9A2HL09Mms427Q1RZWuOHsukfVcKRq9q95iQxdw0ix1JQrqbcDR9PH1QDwf5Qo8OZb5lksZ8Xg== + version "1.1.22" + resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.22.tgz#8f3cc78aefb40b437346dd40a1dbfa5d1da43fe9" + integrity sha512-fvO4ExWMFsqyhG3AiPAObMuY1lxaqgYcxbc49CNdWDDECOJNgQyvsOWVwbZc+qf3rzRtxojBK+CMEv0Ld5CYpw== dependencies: available-typed-arrays "^1.0.7" - call-bind "^1.0.8" + call-bind "^1.0.9" call-bound "^1.0.4" for-each "^0.3.5" get-proto "^1.0.1" @@ -5478,7 +5502,7 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== -ws@8.21.0, ws@^8.19.0: +ws@8.21.0, ws@^7.2.0, ws@^8.20.0: version "8.21.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.21.0.tgz#012e413fc07429945121b0c153158c4343086951" integrity sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g== @@ -5493,7 +5517,7 @@ yallist@^5.0.0: resolved "https://registry.yarnpkg.com/yallist/-/yallist-5.0.0.tgz#00e2de443639ed0d78fd87de0d27469fbcffb533" integrity sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw== -yaml@2.9.0, yaml@^2.4.2: +yaml@2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.9.0.tgz#78274afd93598a1dfdd6130df6a566defcbf9aa4" integrity sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA== @@ -5503,10 +5527,15 @@ yargs-parser@^21.1.1: resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.1.1.tgz#9096bceebf990d21bb31fa9516e0ede294a77d35" integrity sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw== -yargs@^17.0.0, yargs@^17.7.2: - version "17.7.2" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" - integrity sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w== +yargs-parser@^22.0.0: + version "22.0.0" + resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-22.0.0.tgz#87b82094051b0567717346ecd00fd14804b357c8" + integrity sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw== + +yargs@^17.7.2: + version "17.7.3" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.3.tgz#779dffe6bcafec596a7172e983289a588647faaa" + integrity sha512-GZtjxm/J/4TSxuL3FNYjCmLktBTnIw/rVmKSIyKeYAZpmJB2ig9VauCC5xsa82GNKVKDAqpOn3KVzNt0zmrU0g== dependencies: cliui "^8.0.1" escalade "^3.1.1" @@ -5516,6 +5545,18 @@ yargs@^17.0.0, yargs@^17.7.2: y18n "^5.0.5" yargs-parser "^21.1.1" +yargs@^18.0.0: + version "18.1.0" + resolved "https://registry.yarnpkg.com/yargs/-/yargs-18.1.0.tgz#cd7e98c703ef51695bbbf062ed58f28e94291b56" + integrity sha512-2rAgRKu54VsHkqI0/tYkmluGXHD4KW7yZoycuqDQ15QOTnc2VVfy0nN/1eMhnQLO00A+dwtK20xuCnc1YGeUyg== + dependencies: + cliui "^9.0.1" + escalade "^3.1.1" + get-caller-file "^2.0.5" + string-width "^8.2.1" + y18n "^5.0.5" + yargs-parser "^22.0.0" + yauzl@^2.10.0: version "2.10.0" resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" From cc30e91317aa259d2aa8b242bbc4a475904c92ff Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Wed, 2 Sep 2026 11:24:56 -0500 Subject: [PATCH 2/8] docs(openapi): update Core and Enterprise specs to v3.11.2 Generated from docs-tooling derive extraction + merge pipeline. Source: influxdata/docs-tooling reports/openapi/public/ --- .../core/influxdb3-core-openapi.yaml | 12 +- .../influxdb3-enterprise-openapi.yaml | 352 +++++++++++++++++- 2 files changed, 340 insertions(+), 24 deletions(-) diff --git a/api-docs/influxdb3/core/influxdb3-core-openapi.yaml b/api-docs/influxdb3/core/influxdb3-core-openapi.yaml index 978f0bdc0c..b0b0f555ad 100644 --- a/api-docs/influxdb3/core/influxdb3-core-openapi.yaml +++ b/api-docs/influxdb3/core/influxdb3-core-openapi.yaml @@ -18,7 +18,7 @@ info: - `/api/v2/write`: Compatibility endpoint for InfluxDB v2 workloads and clients [Download the OpenAPI specification](/openapi/influxdb3-core-openapi.yaml) - version: v3.10.0 + version: v3.11.2 license: name: MIT url: https://opensource.org/licenses/MIT @@ -26,7 +26,7 @@ info: name: InfluxData url: https://www.influxdata.com email: support@influxdata.com - x-source-hash: sha256:d0a62a892334fbe16b4a8d816a99bc02e5454f3232c270d33c8af09bab4360a5 + x-source-hash: sha256:7894710bb3bf5f946c2e855f8bb9c0235abbb2faec6f0d56068aa89c09b99ade x-influxdata-download-url: /openapi/influxdb3-core-openapi.yaml servers: - url: https://{baseurl} @@ -2942,10 +2942,14 @@ components: - log - retry - disable - description: | + description: > How the trigger responds when the plugin returns an error: + - `log`: Log the error and keep the trigger active. - - `retry`: Retry the failed invocation with exponential backoff (up to 5 attempts), then log the failure. Errors identified as permanent aren't retried. + + - `retry`: Retry the failed invocation with exponential backoff (up to 5 attempts), then log the failure. + Errors identified as permanent aren't retried. + - `disable`: Disable the trigger until you manually re-enable it. required: - run_async diff --git a/api-docs/influxdb3/enterprise/influxdb3-enterprise-openapi.yaml b/api-docs/influxdb3/enterprise/influxdb3-enterprise-openapi.yaml index 4fbe5e7715..c3ecb2c8df 100644 --- a/api-docs/influxdb3/enterprise/influxdb3-enterprise-openapi.yaml +++ b/api-docs/influxdb3/enterprise/influxdb3-enterprise-openapi.yaml @@ -18,7 +18,7 @@ info: - `/api/v2/write`: Compatibility endpoint for InfluxDB v2 workloads and clients [Download the OpenAPI specification](/openapi/influxdb3-enterprise-openapi.yaml) - version: v3.10.0 + version: v3.11.2 license: name: MIT url: https://opensource.org/licenses/MIT @@ -26,7 +26,7 @@ info: name: InfluxData url: https://www.influxdata.com email: support@influxdata.com - x-source-hash: sha256:d0a62a892334fbe16b4a8d816a99bc02e5454f3232c270d33c8af09bab4360a5 + x-source-hash: sha256:7894710bb3bf5f946c2e855f8bb9c0235abbb2faec6f0d56068aa89c09b99ade x-influxdata-download-url: /openapi/influxdb3-enterprise-openapi.yaml servers: - url: https://{baseurl} @@ -1960,6 +1960,118 @@ paths: tags: - Server information - Enterprise + /api/v3/enterprise/upgrade/parquet_cleanup: + post: + operationId: PostEnterpriseUpgradeParquetCleanup + summary: Request Parquet cleanup after a storage engine upgrade + description: >- + Requests removal of the Parquet-era data left behind by a completed Parquet-to-PachaTree storage engine upgrade. + Any node in the cluster accepts the request. The compactor node executes it. + + + Requires the cluster to be fully on the upgraded (PachaTree) storage engine, with the Parquet-to-PachaTree + upgrade already complete. Use `dry_run` to count and size the data a real cleanup would delete without deleting + anything. + + + This endpoint is only available in InfluxDB 3 Enterprise. + + + #### Related + + + - [influxdb3 manage cleanup-parquet](/influxdb3/enterprise/reference/cli/influxdb3/manage/cleanup-parquet/) + x-enterprise-only: true + requestBody: + required: false + content: + application/json: + schema: + $ref: "#/components/schemas/ParquetCleanupRequest" + responses: + "202": + description: Success. The cleanup started, is already running, or already completed. + content: + application/json: + schema: + $ref: "#/components/schemas/ParquetCleanupResponse" + "401": + $ref: "#/components/responses/Unauthorized" + "409": + description: >- + The cluster isn't on the upgraded storage engine, no Parquet-to-PachaTree upgrade was ever recorded, the + upgrade hasn't completed, or a cleanup is already active in a different mode (dry run vs. delete). + tags: + - Server information + - Enterprise + x-influxdb-introduced-in: v3.11.1 + get: + operationId: GetEnterpriseUpgradeParquetCleanup + summary: Get Parquet cleanup status + description: |- + Returns the current or most recent Parquet cleanup state for the cluster. + + This endpoint is only available in InfluxDB 3 Enterprise. + + #### Related + + - [influxdb3 manage cleanup-parquet](/influxdb3/enterprise/reference/cli/influxdb3/manage/cleanup-parquet/) + x-enterprise-only: true + responses: + "200": + description: Success. Returns the current Parquet cleanup state. + content: + application/json: + schema: + $ref: "#/components/schemas/ParquetCleanupStatusSnapshot" + "401": + $ref: "#/components/responses/Unauthorized" + "404": + description: No Parquet cleanup has ever been requested on this cluster. + tags: + - Server information + - Enterprise + x-influxdb-introduced-in: v3.11.1 + /api/v3/enterprise/upgrade/retry_parquet_to_pacha_tree: + post: + operationId: PostEnterpriseUpgradeRetryParquetToPachaTree + summary: Retry a failed Parquet-to-PachaTree upgrade + description: >- + Resets a Parquet-to-PachaTree storage engine upgrade that latched to a `failed` state, so it resumes the next + time a compactor node starts with `--upgrade-pacha-tree`. + + + Re-checks every previously failed upgrade source against the catalog and object store: sources that are still + readable are requeued, sources whose table or database was dropped are recorded as skipped, and sources that + can't be read back are left unresolved for the next retry. + + + This endpoint is only available in InfluxDB 3 Enterprise. + + + #### Related + + + - [influxdb3 manage + retry-upgrade-to-pacha-tree](/influxdb3/enterprise/reference/cli/influxdb3/manage/retry-upgrade-to-pacha-tree/) + x-enterprise-only: true + responses: + "200": + description: Success. Returns a summary of what the retry changed. + content: + application/json: + schema: + $ref: "#/components/schemas/UpgradeRetryResponse" + "401": + $ref: "#/components/responses/Unauthorized" + "409": + description: >- + The upgrade isn't in a state this request can act on (for example, it isn't `failed`), or another request + already reset it. + tags: + - Server information + - Enterprise + x-influxdb-introduced-in: v3.11.2 /api/v3/enterprise/configure/query_groups: post: operationId: PostEnterpriseConfigureQueryGroups @@ -2305,8 +2417,10 @@ paths: Creates a resource (fine-grained permissions) token. A resource token is a token that has access to specific resources in the system. - Resource tokens are available in InfluxDB 3 Enterprise and InfluxDB 3 Cloud. - They are not available in InfluxDB 3 Core. + This endpoint is only available in InfluxDB 3 Enterprise. + InfluxDB 3 Cloud supports a narrower set of database-scoped read and write tokens instead of resource + tokens---see [Manage tokens in InfluxDB 3 Cloud](https://docs.influxdata.com/influxdb3/cloud/admin/tokens/). + InfluxDB 3 Core doesn't support resource tokens. responses: "201": description: | @@ -2815,8 +2929,8 @@ paths: operationId: GetLoadcapProfile summary: Retrieve load capture profile details description: > - Returns file counts, total size, and an anonymized catalog summary for a load capture profile. - The response is the same as the profile preview. + Returns file counts, total size, and an anonymized catalog summary for a load capture profile. The response is + the same as the profile preview. This endpoint requires an admin token. @@ -2843,10 +2957,9 @@ paths: delete: operationId: DeleteLoadcapProfile summary: Delete a load capture profile - description: > + description: | Deletes a load capture profile and its stored artifacts from object storage. - This endpoint requires an admin token. responses: "200": @@ -2859,6 +2972,8 @@ paths: description: >- The profile does not exist, or load capture is unavailable because this node does not use the performance upgrade preview with an explicit `--mode` setting that includes `query`. + "409": + description: The profile is still capturing. Wait for it to finish (or its status to catch up to `complete`), then retry. tags: - Load capture - Enterprise @@ -4430,14 +4545,6 @@ paths: - Write data components: parameters: - LoadCaptureProfileId: - name: profile_id - in: path - required: true - schema: - type: string - format: uuid - description: Identifier of the load capture profile. AcceptQueryHeader: name: Accept in: header @@ -4568,6 +4675,15 @@ components: Password for v1 compatibility authentication. For query string authentication, pass a database token with write permissions as this parameter. InfluxDB 3 checks that the `p` value is an authorized token. + LoadCaptureProfileId: + x-enterprise-only: true + name: profile_id + in: path + required: true + schema: + type: string + format: uuid + description: Identifier of the load capture profile. requestBodies: lineProtocolRequestBody: required: true @@ -5078,10 +5194,14 @@ components: - log - retry - disable - description: | + description: > How the trigger responds when the plugin returns an error: + - `log`: Log the error and keep the trigger active. - - `retry`: Retry the failed invocation with exponential backoff (up to 5 attempts), then log the failure. Errors identified as permanent aren't retried. + + - `retry`: Retry the failed invocation with exponential backoff (up to 5 attempts), then log the failure. + Errors identified as permanent aren't retried. + - `disable`: Disable the trigger until you manually re-enable it. required: - run_async @@ -5438,8 +5558,8 @@ components: type: object nullable: true description: >- - Anonymized catalog snapshot taken when the capture started, including databases, tables, columns, and - node metadata. Null for `query`-only captures. + Anonymized catalog snapshot taken when the capture started, including databases, tables, columns, and node + metadata. Null for `query`-only captures. LicenseResponse: type: object properties: @@ -5584,6 +5704,198 @@ components: - node_id example: node_id: node-1 + ParquetCleanupRequest: + type: object + description: | + Request body for `POST /api/v3/enterprise/upgrade/parquet_cleanup`. + An empty body is equivalent to the defaults (a real deletion). + properties: + dry_run: + type: boolean + description: | + When `true`, only count and size the Parquet-era data a real cleanup would delete. + Delete nothing. + default: false + example: + dry_run: false + ParquetCleanupResponse: + type: object + description: Response body for `POST /api/v3/enterprise/upgrade/parquet_cleanup`. + properties: + state: + type: string + enum: + - started + - already_running + - already_completed + description: The outcome of this cleanup request. + status: + $ref: "#/components/schemas/ParquetCleanupStatusSnapshot" + required: + - state + - status + ParquetCleanupStatusSnapshot: + type: object + description: | + Point-in-time view of the durable Parquet cleanup state. + Returned by `GET /api/v3/enterprise/upgrade/parquet_cleanup` and embedded in the POST response. + properties: + status: + type: string + enum: + - requested + - running + - completed + - failed + description: The cleanup's current status. + mode: + type: string + enum: + - delete + - dry_run + description: Whether this cleanup deletes data or only reports what it would delete. + requested_by: + type: string + nullable: true + description: The node that accepted the cleanup request, if recorded. + files_scanned: + type: integer + format: int64 + description: Number of files the cleanup has scanned so far. + files_matched: + type: integer + format: int64 + description: Number of files identified as Parquet-era data to delete. + bytes_matched: + type: integer + format: int64 + description: Total size, in bytes, of the matched files. + files_deleted: + type: integer + format: int64 + description: | + Number of files deleted so far. + Always `0` in `dry_run` mode. + bytes_deleted: + type: integer + format: int64 + description: | + Total size, in bytes, of the deleted files. + Always `0` in `dry_run` mode. + files_failed: + type: integer + format: int64 + description: Number of files the cleanup failed to delete. + requested_at_ns: + type: integer + format: int64 + nullable: true + description: When the cleanup was requested, in nanoseconds since the Unix epoch. + started_at_ns: + type: integer + format: int64 + nullable: true + description: When the cleanup started running, in nanoseconds since the Unix epoch. + completed_at_ns: + type: integer + format: int64 + nullable: true + description: When the cleanup finished, in nanoseconds since the Unix epoch. + updated_at_ns: + type: integer + format: int64 + nullable: true + description: When this status was last updated, in nanoseconds since the Unix epoch. + last_message: + type: string + nullable: true + description: The most recent human-readable status message, if any. + required: + - status + - mode + - files_scanned + - files_matched + - bytes_matched + - files_deleted + - bytes_deleted + - files_failed + example: + status: completed + mode: delete + requested_by: node-1 + files_scanned: 1024 + files_matched: 812 + bytes_matched: 4294967296 + files_deleted: 812 + bytes_deleted: 4294967296 + files_failed: 0 + requested_at_ns: 1735689600000000000 + started_at_ns: 1735689601000000000 + completed_at_ns: 1735689900000000000 + updated_at_ns: 1735689900000000000 + last_message: null + UpgradeRetryResponse: + type: object + description: Response body for `POST /api/v3/enterprise/upgrade/retry_parquet_to_pacha_tree`. + properties: + entries_inspected: + type: integer + format: int64 + description: Terminal failures the retry re-checked against the catalog and object store. + entries_skipped_source_missing: + type: integer + format: int64 + description: Entries skipped because their source object no longer exists. + entries_skipped_table_dropped: + type: integer + format: int64 + description: Entries skipped because their table was dropped. + entries_requeued: + type: integer + format: int64 + description: Entries returned to the queue because their source is still readable. + entries_already_skipped: + type: integer + format: int64 + description: Terminal failures that were already recorded as skips. + entries_already_converted: + type: integer + format: int64 + description: | + Entries whose sequence already held converted data. + Recorded as imported instead of skipped. + entries_unresolved: + type: integer + format: int64 + description: | + Entries left untouched because the object holding their sequence couldn't be read back. + Run this endpoint again to re-attempt them. + upgrade_status: + type: string + description: The status the enterprise upgrade state was left in after the retry. + message: + type: string + description: What to do next. + required: + - entries_inspected + - entries_skipped_source_missing + - entries_skipped_table_dropped + - entries_requeued + - entries_already_skipped + - entries_already_converted + - entries_unresolved + - upgrade_status + - message + example: + entries_inspected: 3 + entries_skipped_source_missing: 1 + entries_skipped_table_dropped: 0 + entries_requeued: 2 + entries_already_skipped: 0 + entries_already_converted: 0 + entries_unresolved: 0 + upgrade_status: upgrading + message: Migration state reset. Restart the compactor node with --upgrade-pacha-tree to resume the upgrade. AbortDeleteRequestBody: type: object description: Request body for aborting a row-delete request. From 46b21bc109b593977e4137abd74587edb8cbd896 Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Fri, 28 Aug 2026 05:19:10 -0500 Subject: [PATCH 3/8] docs(api): document Parquet cleanup and upgrade retry endpoints Adds API reference entries for the two Enterprise storage-engine upgrade maintenance endpoints (parquet_cleanup, retry_parquet_to_pacha_tree) and CLI reference pages for the corresponding `influxdb3 manage` subcommands, which had no documentation. Also fixes two schema errors found while adding this content: the load capture profile's request-type field is named `type`, not `capture_type`, and its node identifier is a number, not a string. Adds the missing load capture profile detail, preview, and download reference entries, and adds a note about the upgraded storage engine requirement to the Parquet export endpoints. Corrects the resource token endpoint's description of InfluxDB 3 Cloud support. --- .../reference/cli/influxdb3/_index.md | 1 + .../reference/cli/influxdb3/manage/_index.md | 44 ++++++++++ .../cli/influxdb3/manage/cleanup-parquet.md | 85 +++++++++++++++++++ .../manage/retry-upgrade-to-pacha-tree.md | 57 +++++++++++++ 4 files changed, 187 insertions(+) create mode 100644 content/influxdb3/enterprise/reference/cli/influxdb3/manage/_index.md create mode 100644 content/influxdb3/enterprise/reference/cli/influxdb3/manage/cleanup-parquet.md create mode 100644 content/influxdb3/enterprise/reference/cli/influxdb3/manage/retry-upgrade-to-pacha-tree.md diff --git a/content/influxdb3/enterprise/reference/cli/influxdb3/_index.md b/content/influxdb3/enterprise/reference/cli/influxdb3/_index.md index 71a893c97f..c012b6dcaf 100644 --- a/content/influxdb3/enterprise/reference/cli/influxdb3/_index.md +++ b/content/influxdb3/enterprise/reference/cli/influxdb3/_index.md @@ -32,6 +32,7 @@ influxdb3 [GLOBAL-OPTIONS] [COMMAND] | [import](/influxdb3/enterprise/reference/cli/influxdb3/import/) | Manage bulk Parquet imports | | [install](/influxdb3/enterprise/reference/cli/influxdb3/install/) | Install plugins | | [loadcap](/influxdb3/enterprise/reference/cli/influxdb3/loadcap/) | Capture and manage workload profiles | +| [manage](/influxdb3/enterprise/reference/cli/influxdb3/manage/) | Perform cluster maintenance operations | | [query](/influxdb3/enterprise/reference/cli/influxdb3/query/) | Query {{% product-name %}} | | [remove](/influxdb3/enterprise/reference/cli/influxdb3/remove/) | Remove stopped nodes | | [serve](/influxdb3/enterprise/reference/cli/influxdb3/serve/) | Run the {{% product-name %}} server | diff --git a/content/influxdb3/enterprise/reference/cli/influxdb3/manage/_index.md b/content/influxdb3/enterprise/reference/cli/influxdb3/manage/_index.md new file mode 100644 index 0000000000..d971dd1a50 --- /dev/null +++ b/content/influxdb3/enterprise/reference/cli/influxdb3/manage/_index.md @@ -0,0 +1,44 @@ +--- +title: influxdb3 manage +introduced: v3.11.1 +description: > + The `influxdb3 manage` command performs maintenance operations on an + InfluxDB 3 Enterprise cluster, such as cleaning up Parquet data after a + storage engine upgrade. +menu: + influxdb3_enterprise: + parent: influxdb3 + name: influxdb3 manage +weight: 300 +related: + - /influxdb3/enterprise/reference/internals/storage-engine/ +--- + +Use `influxdb3 manage` to perform maintenance operations on an InfluxDB 3 Enterprise cluster. + +> [!Note] +> These commands act on the [upgraded storage engine](/influxdb3/enterprise/reference/internals/storage-engine/) (PachaTree) migration. +> They have no effect on a cluster that has never run the storage engine upgrade. + +## Usage + + + +```bash +influxdb3 manage +``` + +## Subcommands + +| Subcommand | Description | +| :--------- | :---------- | +| [cleanup-parquet](/influxdb3/enterprise/reference/cli/influxdb3/manage/cleanup-parquet/) | Remove pre-upgrade Parquet data after a storage engine upgrade | +| [retry-upgrade-to-pacha-tree](/influxdb3/enterprise/reference/cli/influxdb3/manage/retry-upgrade-to-pacha-tree/) | Retry a failed storage engine upgrade | +| help | Print command help or the help of a subcommand | + +## Options + +| Option | | Description | +| :----- | :----------- | :------------------------------ | +| `-h` | `--help` | Print help information | +| | `--help-all` | Print detailed help information | diff --git a/content/influxdb3/enterprise/reference/cli/influxdb3/manage/cleanup-parquet.md b/content/influxdb3/enterprise/reference/cli/influxdb3/manage/cleanup-parquet.md new file mode 100644 index 0000000000..8c466b7bad --- /dev/null +++ b/content/influxdb3/enterprise/reference/cli/influxdb3/manage/cleanup-parquet.md @@ -0,0 +1,85 @@ +--- +title: influxdb3 manage cleanup-parquet +introduced: v3.11.1 +description: > + The `influxdb3 manage cleanup-parquet` command permanently removes + pre-upgrade Parquet data left behind by a completed Parquet-to-PachaTree + storage engine upgrade in InfluxDB 3 Enterprise. +menu: + influxdb3_enterprise: + parent: influxdb3 manage + name: cleanup-parquet +weight: 301 +related: + - /influxdb3/enterprise/reference/internals/storage-engine/ +--- + +Use `influxdb3 manage cleanup-parquet` to permanently remove Parquet-era data that a completed [storage engine upgrade](/influxdb3/enterprise/reference/internals/storage-engine/) (Parquet-to-PachaTree) left behind. +Any node in the cluster accepts the request, but the compactor node executes the cleanup. + +> [!Important] +> #### This deletes data permanently +> +> - Parquet files superseded by the PachaTree migration are deleted. +> - The operation is irreversible. +> - After cleanup, `influxdb3 manage downgrade-to-parquet` is no longer possible. +> +> Run with `--dry-run` first to see what a real cleanup would delete. + +## Usage + + + +```bash +influxdb3 manage cleanup-parquet [OPTIONS] +``` + +## Options + +| Option | | Description | Default | Environment variable | +| :----- | :-- | :---------- | :------ | :------------------- | +| | `--dry-run` | Report what would be deleted without deleting anything | | | +| | `--yes` | Skip the confirmation prompt. Conflicts with `--dry-run` | | | +| | `--wait` | Poll every 5 seconds until the cleanup completes or fails. Exits non-zero on failure | | | +| | `--status-only` | Print the status of the current or last cleanup, then exit. Conflicts with `--dry-run`, `--yes`, and `--wait` | | | +| `-H` | `--host ` | InfluxDB 3 Enterprise server URL | `http://127.0.0.1:8181` | `INFLUXDB3_HOST_URL` | +| | `--token ` | Authentication token | | `INFLUXDB3_AUTH_TOKEN` | +| | `--tls-ca ` | Path to a custom TLS certificate authority | | `INFLUXDB3_TLS_CA` | +| | `--tls-no-verify` | Disable TLS certificate verification | | `INFLUXDB3_TLS_NO_VERIFY` | +| `-h` | `--help` | Print help information | | | + +## Examples + +### Preview what a cleanup would delete + + + +```bash { placeholders="AUTH_TOKEN" } +influxdb3 manage cleanup-parquet \ + --host http://localhost:8181 \ + --token AUTH_TOKEN \ + --dry-run +``` + +### Run a cleanup and wait for it to finish + + + +```bash { placeholders="AUTH_TOKEN" } +influxdb3 manage cleanup-parquet \ + --host http://localhost:8181 \ + --token AUTH_TOKEN \ + --yes \ + --wait +``` + +### Check the status of the current or last cleanup + + + +```bash { placeholders="AUTH_TOKEN" } +influxdb3 manage cleanup-parquet \ + --host http://localhost:8181 \ + --token AUTH_TOKEN \ + --status-only +``` diff --git a/content/influxdb3/enterprise/reference/cli/influxdb3/manage/retry-upgrade-to-pacha-tree.md b/content/influxdb3/enterprise/reference/cli/influxdb3/manage/retry-upgrade-to-pacha-tree.md new file mode 100644 index 0000000000..25046c532a --- /dev/null +++ b/content/influxdb3/enterprise/reference/cli/influxdb3/manage/retry-upgrade-to-pacha-tree.md @@ -0,0 +1,57 @@ +--- +title: influxdb3 manage retry-upgrade-to-pacha-tree +introduced: v3.11.2 +description: > + The `influxdb3 manage retry-upgrade-to-pacha-tree` command resets a + Parquet-to-PachaTree storage engine upgrade that latched to a failed state + in InfluxDB 3 Enterprise, so it resumes on the next compactor restart. +menu: + influxdb3_enterprise: + parent: influxdb3 manage + name: retry-upgrade-to-pacha-tree +weight: 302 +related: + - /influxdb3/enterprise/reference/internals/storage-engine/ + - /influxdb3/enterprise/reference/cli/influxdb3/manage/cleanup-parquet/ +--- + +Use `influxdb3 manage retry-upgrade-to-pacha-tree` to reset a [storage engine upgrade](/influxdb3/enterprise/reference/internals/storage-engine/) (Parquet-to-PachaTree) that latched to a `failed` state. + +The command re-checks every previously failed upgrade source against the catalog and object store: + +- Sources that are still readable are requeued. +- Sources whose table or database was dropped are recorded as skipped. +- Sources that can't be read back are left unresolved for the next retry. + +The reset itself only changes durable state. +The migration resumes the next time a compactor node starts with `--upgrade-pacha-tree`. + +## Usage + + + +```bash +influxdb3 manage retry-upgrade-to-pacha-tree [OPTIONS] +``` + +## Options + +| Option | | Description | Default | Environment variable | +| :----- | :-- | :---------- | :------ | :------------------- | +| `-H` | `--host ` | InfluxDB 3 Enterprise server URL | `http://127.0.0.1:8181` | `INFLUXDB3_HOST_URL` | +| | `--token ` | Authentication token | | `INFLUXDB3_AUTH_TOKEN` | +| | `--tls-ca ` | Path to a custom TLS certificate authority | | `INFLUXDB3_TLS_CA` | +| | `--tls-no-verify` | Disable TLS certificate verification | | `INFLUXDB3_TLS_NO_VERIFY` | +| `-h` | `--help` | Print help information | | | + +## Example + + + +```bash { placeholders="AUTH_TOKEN" } +influxdb3 manage retry-upgrade-to-pacha-tree \ + --host http://localhost:8181 \ + --token AUTH_TOKEN +``` + +If any sources are left unresolved, run the command again to re-attempt them. From 2c4be4e229a95190190d88f03acc7208de52e4af Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Fri, 28 Aug 2026 16:10:24 -0500 Subject: [PATCH 4/8] docs(api): document deleting a profile while it's still capturing The delete load capture profile endpoint and the corresponding influxdb3 loadcap delete command didn't document that deleting a profile while it's still capturing returns an error. Adds a 409 response to the API reference and a note to the CLI reference page. Verified against a running InfluxDB 3 Enterprise 3.11.2 instance: starting a capture and immediately deleting it by profile id returns 409 Conflict from both the HTTP API and the CLI ("a capture is already in progress"); deleting again after the capture finishes succeeds. --- .../enterprise/reference/cli/influxdb3/loadcap/delete.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/content/influxdb3/enterprise/reference/cli/influxdb3/loadcap/delete.md b/content/influxdb3/enterprise/reference/cli/influxdb3/loadcap/delete.md index a1a65997cf..2724b961db 100644 --- a/content/influxdb3/enterprise/reference/cli/influxdb3/loadcap/delete.md +++ b/content/influxdb3/enterprise/reference/cli/influxdb3/loadcap/delete.md @@ -14,6 +14,8 @@ related: --- Use `influxdb3 loadcap delete` to delete a workload capture profile. +You can't delete a profile while it's still capturing. +Wait for the capture to finish, then retry. > [!Note] > Load capture requires the [upgraded storage engine](/influxdb3/enterprise/reference/internals/storage-engine/)—the default for new clusters. From ac9113eadab6db22d433344875ab0b783f956109 Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Tue, 8 Sep 2026 18:57:15 -0500 Subject: [PATCH 5/8] fix(plugins): populate discovered plugin articles What changed: - Generate shared README content for every discovered official plugin and backfill missing pages. - Preserve NWS capitalization in generated stubs and normalize generated Markdown for validation. Why: - Registry discovery previously created product stubs without article content for unmapped plugins. Impact: - Official plugin pages, including NWS Weather, now render generated article content in Core and Enterprise. Verification: - yarn test:sync-plugins - yarn lint-codeblocks content/shared/influxdb3-plugins/plugins-library/official/*.md content/influxdb3/core/plugins/library/official/*.md content/influxdb3/enterprise/plugins/library/official/*.md - npx hugo --quiet --- .../library/official/amqp-subscriber.md | 16 + .../library/official/bird-data-simulator.md | 16 + .../library/official/chronos-forecasting.md | 16 + .../library/official/earthquake-sampler.md | 16 + .../core/plugins/library/official/gapfill.md | 16 + .../library/official/geo-enrichment.md | 16 + .../core/plugins/library/official/import.md | 16 + .../library/official/kafka-subscriber.md | 16 + .../library/official/mqtt-subscriber.md | 16 + .../library/official/nori-regression.md | 16 + .../plugins/library/official/nws-weather.md | 16 + .../core/plugins/library/official/opcua.md | 16 + .../plugins/library/official/resampler.md | 16 + .../official/river-anomaly-detector.md | 16 + .../library/official/river-auto-profiler.md | 16 + .../library/official/river-forecaster.md | 16 + .../plugins/library/official/sagemaker.md | 16 + .../library/official/schema-validator.md | 16 + .../plugins/library/official/signal-filter.md | 16 + .../library/official/signal-generator.md | 16 + .../official/simple-data-replicator.md | 16 + .../plugins/library/official/stock-plugin.md | 16 + .../library/official/synthefy-forecasting.md | 16 + .../plugins/library/official/valuecounter.md | 16 + .../library/official/amqp-subscriber.md | 16 + .../library/official/bird-data-simulator.md | 16 + .../library/official/chronos-forecasting.md | 16 + .../library/official/earthquake-sampler.md | 16 + .../plugins/library/official/gapfill.md | 16 + .../library/official/geo-enrichment.md | 16 + .../plugins/library/official/import.md | 16 + .../library/official/kafka-subscriber.md | 16 + .../library/official/mqtt-subscriber.md | 16 + .../library/official/nori-regression.md | 16 + .../plugins/library/official/nws-weather.md | 16 + .../plugins/library/official/opcua.md | 16 + .../plugins/library/official/resampler.md | 16 + .../official/river-anomaly-detector.md | 16 + .../library/official/river-auto-profiler.md | 16 + .../library/official/river-forecaster.md | 16 + .../plugins/library/official/sagemaker.md | 16 + .../library/official/schema-validator.md | 16 + .../plugins/library/official/signal-filter.md | 16 + .../library/official/signal-generator.md | 16 + .../official/simple-data-replicator.md | 16 + .../plugins/library/official/stock-plugin.md | 16 + .../library/official/synthefy-forecasting.md | 16 + .../plugins/library/official/valuecounter.md | 16 + .../official/amqp-subscriber.md | 500 ++++++++++ .../official/basic-transformation.md | 18 +- .../official/bird-data-simulator.md | 171 ++++ .../official/chronos-forecasting.md | 299 ++++++ .../plugins-library/official/downsampler.md | 80 +- .../official/earthquake-sampler.md | 285 ++++++ .../official/forecast-error-evaluator.md | 164 ++-- .../plugins-library/official/gapfill.md | 375 ++++++++ .../official/geo-enrichment.md | 789 +++++++++++++++ .../plugins-library/official/import.md | 840 ++++++++++++++++ .../official/influxdb-to-iceberg.md | 25 +- .../official/kafka-subscriber.md | 825 ++++++++++++++++ .../plugins-library/official/mad-check.md | 103 +- .../official/mqtt-subscriber.md | 543 +++++++++++ .../official/nori-regression.md | 558 +++++++++++ .../plugins-library/official/notifier.md | 13 +- .../plugins-library/official/nws-weather.md | 415 ++++++++ .../plugins-library/official/opcua.md | 895 ++++++++++++++++++ .../official/prophet-forecasting.md | 200 ++-- .../plugins-library/official/resampler.md | 265 ++++++ .../official/river-anomaly-detector.md | 273 ++++++ .../official/river-auto-profiler.md | 271 ++++++ .../official/river-forecaster.md | 231 +++++ .../plugins-library/official/sagemaker.md | 620 ++++++++++++ .../official/schema-validator.md | 279 ++++++ .../plugins-library/official/signal-filter.md | 401 ++++++++ .../official/signal-generator.md | 559 +++++++++++ .../official/simple-data-replicator.md | 257 +++++ .../plugins-library/official/state-change.md | 60 +- .../official/stateless-adtk-detector.md | 84 +- .../plugins-library/official/stock-plugin.md | 282 ++++++ .../official/synthefy-forecasting.md | 402 ++++++++ .../official/system-metrics.md | 82 +- .../official/threshold-deadman-checks.md | 58 +- .../plugins-library/official/valuecounter.md | 266 ++++++ helper-scripts/influxdb3-plugins/README.md | 20 +- .../influxdb3-plugins/port_to_docs.js | 92 +- helper-scripts/influxdb3-plugins/reporting.js | 7 +- .../influxdb3-plugins/stub-template.js | 16 +- .../test/region-writer.test.js | 25 +- .../influxdb3-plugins/test/reporting.test.js | 5 +- .../influxdb3-plugins/test/stubs.test.js | 27 + .../test/sync-results.test.js | 24 + 91 files changed, 12118 insertions(+), 354 deletions(-) create mode 100644 content/influxdb3/core/plugins/library/official/amqp-subscriber.md create mode 100644 content/influxdb3/core/plugins/library/official/bird-data-simulator.md create mode 100644 content/influxdb3/core/plugins/library/official/chronos-forecasting.md create mode 100644 content/influxdb3/core/plugins/library/official/earthquake-sampler.md create mode 100644 content/influxdb3/core/plugins/library/official/gapfill.md create mode 100644 content/influxdb3/core/plugins/library/official/geo-enrichment.md create mode 100644 content/influxdb3/core/plugins/library/official/import.md create mode 100644 content/influxdb3/core/plugins/library/official/kafka-subscriber.md create mode 100644 content/influxdb3/core/plugins/library/official/mqtt-subscriber.md create mode 100644 content/influxdb3/core/plugins/library/official/nori-regression.md create mode 100644 content/influxdb3/core/plugins/library/official/nws-weather.md create mode 100644 content/influxdb3/core/plugins/library/official/opcua.md create mode 100644 content/influxdb3/core/plugins/library/official/resampler.md create mode 100644 content/influxdb3/core/plugins/library/official/river-anomaly-detector.md create mode 100644 content/influxdb3/core/plugins/library/official/river-auto-profiler.md create mode 100644 content/influxdb3/core/plugins/library/official/river-forecaster.md create mode 100644 content/influxdb3/core/plugins/library/official/sagemaker.md create mode 100644 content/influxdb3/core/plugins/library/official/schema-validator.md create mode 100644 content/influxdb3/core/plugins/library/official/signal-filter.md create mode 100644 content/influxdb3/core/plugins/library/official/signal-generator.md create mode 100644 content/influxdb3/core/plugins/library/official/simple-data-replicator.md create mode 100644 content/influxdb3/core/plugins/library/official/stock-plugin.md create mode 100644 content/influxdb3/core/plugins/library/official/synthefy-forecasting.md create mode 100644 content/influxdb3/core/plugins/library/official/valuecounter.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/amqp-subscriber.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/bird-data-simulator.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/chronos-forecasting.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/earthquake-sampler.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/gapfill.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/geo-enrichment.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/import.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/kafka-subscriber.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/mqtt-subscriber.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/nori-regression.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/nws-weather.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/opcua.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/resampler.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/river-anomaly-detector.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/river-auto-profiler.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/river-forecaster.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/sagemaker.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/schema-validator.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/signal-filter.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/signal-generator.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/simple-data-replicator.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/stock-plugin.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/synthefy-forecasting.md create mode 100644 content/influxdb3/enterprise/plugins/library/official/valuecounter.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/amqp-subscriber.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/bird-data-simulator.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/chronos-forecasting.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/earthquake-sampler.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/gapfill.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/geo-enrichment.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/import.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/kafka-subscriber.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/mqtt-subscriber.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/nori-regression.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/nws-weather.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/opcua.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/resampler.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/river-anomaly-detector.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/river-auto-profiler.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/river-forecaster.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/sagemaker.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/schema-validator.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/signal-filter.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/signal-generator.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/simple-data-replicator.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/stock-plugin.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/synthefy-forecasting.md create mode 100644 content/shared/influxdb3-plugins/plugins-library/official/valuecounter.md diff --git a/content/influxdb3/core/plugins/library/official/amqp-subscriber.md b/content/influxdb3/core/plugins/library/official/amqp-subscriber.md new file mode 100644 index 0000000000..a6ebfc2217 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/amqp-subscriber.md @@ -0,0 +1,16 @@ +--- +title: Amqp subscriber plugin +description: Enables real-time ingestion of AMQP messages into InfluxDB 3. Subscribe to queues and transform messages into time-series data with support for JSON, Line Protocol, and custom text formats. +menu: + influxdb3_core: + name: Amqp subscriber + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/amqp_subscriber, Amqp subscriber plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/amqp-subscriber.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/bird-data-simulator.md b/content/influxdb3/core/plugins/library/official/bird-data-simulator.md new file mode 100644 index 0000000000..452cc72440 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/bird-data-simulator.md @@ -0,0 +1,16 @@ +--- +title: Bird data simulator plugin +description: Generates synthetic bird tracking telemetry with persistent birds, ranges, simulated movement, speed, heading, and body temperature. Exposes only simple volume controls. +menu: + influxdb3_core: + name: Bird data simulator + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/bird_data_simulator, Bird data simulator plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/bird-data-simulator.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/chronos-forecasting.md b/content/influxdb3/core/plugins/library/official/chronos-forecasting.md new file mode 100644 index 0000000000..b9ad6246a0 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/chronos-forecasting.md @@ -0,0 +1,16 @@ +--- +title: Chronos forecasting plugin +description: Enables zero-shot time-series forecasting with Amazon Chronos models from HuggingFace, supporting scheduled batch forecasts and on-demand HTTP forecasts. +menu: + influxdb3_core: + name: Chronos forecasting + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/chronos_forecasting, Chronos forecasting plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/chronos-forecasting.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/earthquake-sampler.md b/content/influxdb3/core/plugins/library/official/earthquake-sampler.md new file mode 100644 index 0000000000..3a83638295 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/earthquake-sampler.md @@ -0,0 +1,16 @@ +--- +title: Earthquake sampler plugin +description: Ingests USGS earthquake events on a schedule and writes normalized records for dashboards and alerting, with optional canonical quake-schema output. +menu: + influxdb3_core: + name: Earthquake sampler + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/earthquake_sampler, Earthquake sampler plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/earthquake-sampler.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/gapfill.md b/content/influxdb3/core/plugins/library/official/gapfill.md new file mode 100644 index 0000000000..f097286132 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/gapfill.md @@ -0,0 +1,16 @@ +--- +title: Gapfill plugin +description: Detects gaps in time series and fills them with imputed values using configurable interpolation methods, with optional fill marking and per-gap quality reports. Supports scheduled and HTTP backfill. +menu: + influxdb3_core: + name: Gapfill + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/gapfill, Gapfill plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/gapfill.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/geo-enrichment.md b/content/influxdb3/core/plugins/library/official/geo-enrichment.md new file mode 100644 index 0000000000..25b7a2806a --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/geo-enrichment.md @@ -0,0 +1,16 @@ +--- +title: Geo enrichment plugin +description: Resolves lat/lon into location attributes—country and city, a zone or site you define, or a grid cell—merged into the source rows or written to a target table. HTTP endpoint backfills history. +menu: + influxdb3_core: + name: Geo enrichment + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/geo_enrichment, Geo enrichment plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/geo-enrichment.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/import.md b/content/influxdb3/core/plugins/library/official/import.md new file mode 100644 index 0000000000..46682b9e77 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/import.md @@ -0,0 +1,16 @@ +--- +title: Import plugin +description: Enables seamless data import from InfluxDB v1, v2, or v3 instances to InfluxDB 3 Core/Enterprise +menu: + influxdb3_core: + name: Import + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/import, Import plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/import.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/kafka-subscriber.md b/content/influxdb3/core/plugins/library/official/kafka-subscriber.md new file mode 100644 index 0000000000..3579a11e40 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/kafka-subscriber.md @@ -0,0 +1,16 @@ +--- +title: Kafka subscriber plugin +description: Ingests Kafka messages into InfluxDB 3 with JSON, Line Protocol, text, Avro, JSON Schema, and Protobuf decoding, plus Schema Registry and local schema support. +menu: + influxdb3_core: + name: Kafka subscriber + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/kafka_subscriber, Kafka subscriber plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/kafka-subscriber.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/mqtt-subscriber.md b/content/influxdb3/core/plugins/library/official/mqtt-subscriber.md new file mode 100644 index 0000000000..6f5f5e6483 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/mqtt-subscriber.md @@ -0,0 +1,16 @@ +--- +title: Mqtt subscriber plugin +description: Enables real-time ingestion of MQTT messages into InfluxDB 3. Subscribe to MQTT topics and transform messages into time-series data with support for JSON, Line Protocol, and custom text formats. +menu: + influxdb3_core: + name: Mqtt subscriber + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/mqtt_subscriber, Mqtt subscriber plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/mqtt-subscriber.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/nori-regression.md b/content/influxdb3/core/plugins/library/official/nori-regression.md new file mode 100644 index 0000000000..0d5d25288c --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/nori-regression.md @@ -0,0 +1,16 @@ +--- +title: Nori regression plugin +description: Predict a numeric InfluxDB 3 field from other columns with Synthefy's Nori in-context tabular regression model, via the Synthefy inference gateway. Imputes rows where the target field is null. +menu: + influxdb3_core: + name: Nori regression + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/nori_regression, Nori regression plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/nori-regression.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/nws-weather.md b/content/influxdb3/core/plugins/library/official/nws-weather.md new file mode 100644 index 0000000000..640fc896d1 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/nws-weather.md @@ -0,0 +1,16 @@ +--- +title: NWS weather plugin +description: Provides real-time weather data from the National Weather Service API for demonstration and sample data purposes. Fetches live observations from multiple weather stations across the United States. +menu: + influxdb3_core: + name: NWS weather + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/nws_weather, NWS weather plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/nws-weather.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/opcua.md b/content/influxdb3/core/plugins/library/official/opcua.md new file mode 100644 index 0000000000..551bb3cef1 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/opcua.md @@ -0,0 +1,16 @@ +--- +title: Opcua plugin +description: Enables periodic ingestion of OPC UA node values into InfluxDB 3. Connect to OPC UA servers and read current values with auto-discovery browse mode, data type detection, and namespace resolution. +menu: + influxdb3_core: + name: Opcua + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/opcua, Opcua plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/opcua.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/resampler.md b/content/influxdb3/core/plugins/library/official/resampler.md new file mode 100644 index 0000000000..c09f196b28 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/resampler.md @@ -0,0 +1,16 @@ +--- +title: Resampler plugin +description: Resamples time series with non-uniform timestamps onto a uniform time grid via configurable interpolation, or snaps timestamps to grid nodes keeping values and types unchanged. +menu: + influxdb3_core: + name: Resampler + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/resampler, Resampler plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/resampler.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/river-anomaly-detector.md b/content/influxdb3/core/plugins/library/official/river-anomaly-detector.md new file mode 100644 index 0000000000..a53851b5af --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/river-anomaly-detector.md @@ -0,0 +1,16 @@ +--- +title: River anomaly detector plugin +description: Detects time-series anomalies on writes using River ML models, combining rolling statistics with HalfSpaceTrees for per-series anomaly scoring and optional auto-tuning from River Auto Profiler output. +menu: + influxdb3_core: + name: River anomaly detector + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/river_anomaly_detector, River anomaly detector plugin on GitHub +source: "/shared/influxdb3-plugins/plugins-library/official/river\u002danomaly\u002ddetector.md" +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/river-auto-profiler.md b/content/influxdb3/core/plugins/library/official/river-auto-profiler.md new file mode 100644 index 0000000000..95c2b814e3 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/river-auto-profiler.md @@ -0,0 +1,16 @@ +--- +title: River auto profiler plugin +description: Incrementally profiles time-series data with River ML streaming statistics and writes per-series recommendations for anomaly detection tuning, including thresholds, fading factors, and maturity. +menu: + influxdb3_core: + name: River auto profiler + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/river_auto_profiler, River auto profiler plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/river-auto-profiler.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/river-forecaster.md b/content/influxdb3/core/plugins/library/official/river-forecaster.md new file mode 100644 index 0000000000..31b0b52c2e --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/river-forecaster.md @@ -0,0 +1,16 @@ +--- +title: River forecaster plugin +description: Provides online time-series forecasting with River ML SNARIMAX models, learning from writes and producing scheduled multi-step forecasts with per-series model state and optional auto-horizon support. +menu: + influxdb3_core: + name: River forecaster + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/river_forecaster, River forecaster plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/river-forecaster.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/sagemaker.md b/content/influxdb3/core/plugins/library/official/sagemaker.md new file mode 100644 index 0000000000..a33da5625f --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/sagemaker.md @@ -0,0 +1,16 @@ +--- +title: Sagemaker plugin +description: Runs scheduled inference against Amazon SageMaker endpoints using recent InfluxDB 3 rows and writes prediction results back to InfluxDB. +menu: + influxdb3_core: + name: Sagemaker + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/sagemaker, Sagemaker plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/sagemaker.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/schema-validator.md b/content/influxdb3/core/plugins/library/official/schema-validator.md new file mode 100644 index 0000000000..bb3e5aa8b0 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/schema-validator.md @@ -0,0 +1,16 @@ +--- +title: Schema validator plugin +description: Validates incoming line protocol data against a JSON schema. +menu: + influxdb3_core: + name: Schema validator + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/schema_validator, Schema validator plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/schema-validator.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/signal-filter.md b/content/influxdb3/core/plugins/library/official/signal-filter.md new file mode 100644 index 0000000000..ba44953936 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/signal-filter.md @@ -0,0 +1,16 @@ +--- +title: Signal filter plugin +description: Applies streaming digital IIR filters to numeric fields +menu: + influxdb3_core: + name: Signal filter + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/signal_filter, Signal filter plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/signal-filter.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/signal-generator.md b/content/influxdb3/core/plugins/library/official/signal-generator.md new file mode 100644 index 0000000000..b43e033822 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/signal-generator.md @@ -0,0 +1,16 @@ +--- +title: Signal generator plugin +description: Generates scheduled, configurable waveform time-series data for demos, testing, dashboards, alerts, and plugin validation without external sources. Simulates outage gaps by default. +menu: + influxdb3_core: + name: Signal generator + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/signal_generator, Signal generator plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/signal-generator.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/simple-data-replicator.md b/content/influxdb3/core/plugins/library/official/simple-data-replicator.md new file mode 100644 index 0000000000..e255bf044b --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/simple-data-replicator.md @@ -0,0 +1,16 @@ +--- +title: Simple data replicator plugin +description: Replicates data between InfluxDB 3 instances over HTTP with table and field filtering, renaming, scheduler/write triggers, and compressed retry queue buffering. +menu: + influxdb3_core: + name: Simple data replicator + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/simple_data_replicator, Simple data replicator plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/simple-data-replicator.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/stock-plugin.md b/content/influxdb3/core/plugins/library/official/stock-plugin.md new file mode 100644 index 0000000000..49232fd186 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/stock-plugin.md @@ -0,0 +1,16 @@ +--- +title: Stock plugin +description: Tracks stock, ETF, and mutual fund portfolio values from Yahoo Finance with market-hours gating and rollups. +menu: + influxdb3_core: + name: Stock plugin + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/stock_plugin, Stock plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/stock-plugin.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/synthefy-forecasting.md b/content/influxdb3/core/plugins/library/official/synthefy-forecasting.md new file mode 100644 index 0000000000..f124f6282d --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/synthefy-forecasting.md @@ -0,0 +1,16 @@ +--- +title: Synthefy forecasting plugin +description: Integrates Synthefy Forecasting API with InfluxDB 3 for on-demand time series forecasting via HTTP. Reads data from InfluxDB, generates forecasts with Synthefy models, and writes results back. +menu: + influxdb3_core: + name: Synthefy forecasting + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/synthefy_forecasting, Synthefy forecasting plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/synthefy-forecasting.md +canonical: self +--- + + diff --git a/content/influxdb3/core/plugins/library/official/valuecounter.md b/content/influxdb3/core/plugins/library/official/valuecounter.md new file mode 100644 index 0000000000..379cdddc33 --- /dev/null +++ b/content/influxdb3/core/plugins/library/official/valuecounter.md @@ -0,0 +1,16 @@ +--- +title: Valuecounter plugin +description: Counts unique field values from data-write or scheduled inputs and writes rollup measurements with per-value counts. +menu: + influxdb3_core: + name: Valuecounter + parent: Official plugins +weight: 100 +influxdb3/core/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/valuecounter, Valuecounter plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/valuecounter.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/amqp-subscriber.md b/content/influxdb3/enterprise/plugins/library/official/amqp-subscriber.md new file mode 100644 index 0000000000..428f188405 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/amqp-subscriber.md @@ -0,0 +1,16 @@ +--- +title: Amqp subscriber plugin +description: Enables real-time ingestion of AMQP messages into InfluxDB 3. Subscribe to queues and transform messages into time-series data with support for JSON, Line Protocol, and custom text formats. +menu: + influxdb3_enterprise: + name: Amqp subscriber + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/amqp_subscriber, Amqp subscriber plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/amqp-subscriber.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/bird-data-simulator.md b/content/influxdb3/enterprise/plugins/library/official/bird-data-simulator.md new file mode 100644 index 0000000000..fc4d44f871 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/bird-data-simulator.md @@ -0,0 +1,16 @@ +--- +title: Bird data simulator plugin +description: Generates synthetic bird tracking telemetry with persistent birds, ranges, simulated movement, speed, heading, and body temperature. Exposes only simple volume controls. +menu: + influxdb3_enterprise: + name: Bird data simulator + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/bird_data_simulator, Bird data simulator plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/bird-data-simulator.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/chronos-forecasting.md b/content/influxdb3/enterprise/plugins/library/official/chronos-forecasting.md new file mode 100644 index 0000000000..bbecfc7289 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/chronos-forecasting.md @@ -0,0 +1,16 @@ +--- +title: Chronos forecasting plugin +description: Enables zero-shot time-series forecasting with Amazon Chronos models from HuggingFace, supporting scheduled batch forecasts and on-demand HTTP forecasts. +menu: + influxdb3_enterprise: + name: Chronos forecasting + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/chronos_forecasting, Chronos forecasting plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/chronos-forecasting.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/earthquake-sampler.md b/content/influxdb3/enterprise/plugins/library/official/earthquake-sampler.md new file mode 100644 index 0000000000..9ca25432fc --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/earthquake-sampler.md @@ -0,0 +1,16 @@ +--- +title: Earthquake sampler plugin +description: Ingests USGS earthquake events on a schedule and writes normalized records for dashboards and alerting, with optional canonical quake-schema output. +menu: + influxdb3_enterprise: + name: Earthquake sampler + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/earthquake_sampler, Earthquake sampler plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/earthquake-sampler.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/gapfill.md b/content/influxdb3/enterprise/plugins/library/official/gapfill.md new file mode 100644 index 0000000000..d6a7c6a5c6 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/gapfill.md @@ -0,0 +1,16 @@ +--- +title: Gapfill plugin +description: Detects gaps in time series and fills them with imputed values using configurable interpolation methods, with optional fill marking and per-gap quality reports. Supports scheduled and HTTP backfill. +menu: + influxdb3_enterprise: + name: Gapfill + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/gapfill, Gapfill plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/gapfill.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/geo-enrichment.md b/content/influxdb3/enterprise/plugins/library/official/geo-enrichment.md new file mode 100644 index 0000000000..12152e0aac --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/geo-enrichment.md @@ -0,0 +1,16 @@ +--- +title: Geo enrichment plugin +description: Resolves lat/lon into location attributes—country and city, a zone or site you define, or a grid cell—merged into the source rows or written to a target table. HTTP endpoint backfills history. +menu: + influxdb3_enterprise: + name: Geo enrichment + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/geo_enrichment, Geo enrichment plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/geo-enrichment.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/import.md b/content/influxdb3/enterprise/plugins/library/official/import.md new file mode 100644 index 0000000000..f2c28b6367 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/import.md @@ -0,0 +1,16 @@ +--- +title: Import plugin +description: Enables seamless data import from InfluxDB v1, v2, or v3 instances to InfluxDB 3 Core/Enterprise +menu: + influxdb3_enterprise: + name: Import + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/import, Import plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/import.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/kafka-subscriber.md b/content/influxdb3/enterprise/plugins/library/official/kafka-subscriber.md new file mode 100644 index 0000000000..7128c94158 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/kafka-subscriber.md @@ -0,0 +1,16 @@ +--- +title: Kafka subscriber plugin +description: Ingests Kafka messages into InfluxDB 3 with JSON, Line Protocol, text, Avro, JSON Schema, and Protobuf decoding, plus Schema Registry and local schema support. +menu: + influxdb3_enterprise: + name: Kafka subscriber + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/kafka_subscriber, Kafka subscriber plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/kafka-subscriber.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/mqtt-subscriber.md b/content/influxdb3/enterprise/plugins/library/official/mqtt-subscriber.md new file mode 100644 index 0000000000..efa068526f --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/mqtt-subscriber.md @@ -0,0 +1,16 @@ +--- +title: Mqtt subscriber plugin +description: Enables real-time ingestion of MQTT messages into InfluxDB 3. Subscribe to MQTT topics and transform messages into time-series data with support for JSON, Line Protocol, and custom text formats. +menu: + influxdb3_enterprise: + name: Mqtt subscriber + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/mqtt_subscriber, Mqtt subscriber plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/mqtt-subscriber.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/nori-regression.md b/content/influxdb3/enterprise/plugins/library/official/nori-regression.md new file mode 100644 index 0000000000..8ff7e44bc3 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/nori-regression.md @@ -0,0 +1,16 @@ +--- +title: Nori regression plugin +description: Predict a numeric InfluxDB 3 field from other columns with Synthefy's Nori in-context tabular regression model, via the Synthefy inference gateway. Imputes rows where the target field is null. +menu: + influxdb3_enterprise: + name: Nori regression + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/nori_regression, Nori regression plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/nori-regression.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/nws-weather.md b/content/influxdb3/enterprise/plugins/library/official/nws-weather.md new file mode 100644 index 0000000000..1a1703f4ad --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/nws-weather.md @@ -0,0 +1,16 @@ +--- +title: NWS weather plugin +description: Provides real-time weather data from the National Weather Service API for demonstration and sample data purposes. Fetches live observations from multiple weather stations across the United States. +menu: + influxdb3_enterprise: + name: NWS weather + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/nws_weather, NWS weather plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/nws-weather.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/opcua.md b/content/influxdb3/enterprise/plugins/library/official/opcua.md new file mode 100644 index 0000000000..3ee131e3e2 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/opcua.md @@ -0,0 +1,16 @@ +--- +title: Opcua plugin +description: Enables periodic ingestion of OPC UA node values into InfluxDB 3. Connect to OPC UA servers and read current values with auto-discovery browse mode, data type detection, and namespace resolution. +menu: + influxdb3_enterprise: + name: Opcua + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/opcua, Opcua plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/opcua.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/resampler.md b/content/influxdb3/enterprise/plugins/library/official/resampler.md new file mode 100644 index 0000000000..119a9c5289 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/resampler.md @@ -0,0 +1,16 @@ +--- +title: Resampler plugin +description: Resamples time series with non-uniform timestamps onto a uniform time grid via configurable interpolation, or snaps timestamps to grid nodes keeping values and types unchanged. +menu: + influxdb3_enterprise: + name: Resampler + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/resampler, Resampler plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/resampler.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/river-anomaly-detector.md b/content/influxdb3/enterprise/plugins/library/official/river-anomaly-detector.md new file mode 100644 index 0000000000..25b3c2da1c --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/river-anomaly-detector.md @@ -0,0 +1,16 @@ +--- +title: River anomaly detector plugin +description: Detects time-series anomalies on writes using River ML models, combining rolling statistics with HalfSpaceTrees for per-series anomaly scoring and optional auto-tuning from River Auto Profiler output. +menu: + influxdb3_enterprise: + name: River anomaly detector + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/river_anomaly_detector, River anomaly detector plugin on GitHub +source: "/shared/influxdb3-plugins/plugins-library/official/river\u002danomaly\u002ddetector.md" +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/river-auto-profiler.md b/content/influxdb3/enterprise/plugins/library/official/river-auto-profiler.md new file mode 100644 index 0000000000..84e3cc6cf2 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/river-auto-profiler.md @@ -0,0 +1,16 @@ +--- +title: River auto profiler plugin +description: Incrementally profiles time-series data with River ML streaming statistics and writes per-series recommendations for anomaly detection tuning, including thresholds, fading factors, and maturity. +menu: + influxdb3_enterprise: + name: River auto profiler + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/river_auto_profiler, River auto profiler plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/river-auto-profiler.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/river-forecaster.md b/content/influxdb3/enterprise/plugins/library/official/river-forecaster.md new file mode 100644 index 0000000000..356be8e44d --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/river-forecaster.md @@ -0,0 +1,16 @@ +--- +title: River forecaster plugin +description: Provides online time-series forecasting with River ML SNARIMAX models, learning from writes and producing scheduled multi-step forecasts with per-series model state and optional auto-horizon support. +menu: + influxdb3_enterprise: + name: River forecaster + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/river_forecaster, River forecaster plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/river-forecaster.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/sagemaker.md b/content/influxdb3/enterprise/plugins/library/official/sagemaker.md new file mode 100644 index 0000000000..e113caef2b --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/sagemaker.md @@ -0,0 +1,16 @@ +--- +title: Sagemaker plugin +description: Runs scheduled inference against Amazon SageMaker endpoints using recent InfluxDB 3 rows and writes prediction results back to InfluxDB. +menu: + influxdb3_enterprise: + name: Sagemaker + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/sagemaker, Sagemaker plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/sagemaker.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/schema-validator.md b/content/influxdb3/enterprise/plugins/library/official/schema-validator.md new file mode 100644 index 0000000000..0512bd2eba --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/schema-validator.md @@ -0,0 +1,16 @@ +--- +title: Schema validator plugin +description: Validates incoming line protocol data against a JSON schema. +menu: + influxdb3_enterprise: + name: Schema validator + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/schema_validator, Schema validator plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/schema-validator.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/signal-filter.md b/content/influxdb3/enterprise/plugins/library/official/signal-filter.md new file mode 100644 index 0000000000..cd50ef5c85 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/signal-filter.md @@ -0,0 +1,16 @@ +--- +title: Signal filter plugin +description: Applies streaming digital IIR filters to numeric fields +menu: + influxdb3_enterprise: + name: Signal filter + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/signal_filter, Signal filter plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/signal-filter.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/signal-generator.md b/content/influxdb3/enterprise/plugins/library/official/signal-generator.md new file mode 100644 index 0000000000..d3cfba7a47 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/signal-generator.md @@ -0,0 +1,16 @@ +--- +title: Signal generator plugin +description: Generates scheduled, configurable waveform time-series data for demos, testing, dashboards, alerts, and plugin validation without external sources. Simulates outage gaps by default. +menu: + influxdb3_enterprise: + name: Signal generator + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/signal_generator, Signal generator plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/signal-generator.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/simple-data-replicator.md b/content/influxdb3/enterprise/plugins/library/official/simple-data-replicator.md new file mode 100644 index 0000000000..aa638ce281 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/simple-data-replicator.md @@ -0,0 +1,16 @@ +--- +title: Simple data replicator plugin +description: Replicates data between InfluxDB 3 instances over HTTP with table and field filtering, renaming, scheduler/write triggers, and compressed retry queue buffering. +menu: + influxdb3_enterprise: + name: Simple data replicator + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/simple_data_replicator, Simple data replicator plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/simple-data-replicator.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/stock-plugin.md b/content/influxdb3/enterprise/plugins/library/official/stock-plugin.md new file mode 100644 index 0000000000..f78f0f9126 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/stock-plugin.md @@ -0,0 +1,16 @@ +--- +title: Stock plugin +description: Tracks stock, ETF, and mutual fund portfolio values from Yahoo Finance with market-hours gating and rollups. +menu: + influxdb3_enterprise: + name: Stock plugin + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/stock_plugin, Stock plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/stock-plugin.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/synthefy-forecasting.md b/content/influxdb3/enterprise/plugins/library/official/synthefy-forecasting.md new file mode 100644 index 0000000000..84e2fcb0d8 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/synthefy-forecasting.md @@ -0,0 +1,16 @@ +--- +title: Synthefy forecasting plugin +description: Integrates Synthefy Forecasting API with InfluxDB 3 for on-demand time series forecasting via HTTP. Reads data from InfluxDB, generates forecasts with Synthefy models, and writes results back. +menu: + influxdb3_enterprise: + name: Synthefy forecasting + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/synthefy_forecasting, Synthefy forecasting plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/synthefy-forecasting.md +canonical: self +--- + + diff --git a/content/influxdb3/enterprise/plugins/library/official/valuecounter.md b/content/influxdb3/enterprise/plugins/library/official/valuecounter.md new file mode 100644 index 0000000000..70fc00a4d1 --- /dev/null +++ b/content/influxdb3/enterprise/plugins/library/official/valuecounter.md @@ -0,0 +1,16 @@ +--- +title: Valuecounter plugin +description: Counts unique field values from data-write or scheduled inputs and writes rollup measurements with per-value counts. +menu: + influxdb3_enterprise: + name: Valuecounter + parent: Official plugins +weight: 100 +influxdb3/enterprise/tags: [plugins, processing engine, python, official] +related: + - https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/valuecounter, Valuecounter plugin on GitHub +source: /shared/influxdb3-plugins/plugins-library/official/valuecounter.md +canonical: self +--- + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/amqp-subscriber.md b/content/shared/influxdb3-plugins/plugins-library/official/amqp-subscriber.md new file mode 100644 index 0000000000..70dbc2bc5c --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/amqp-subscriber.md @@ -0,0 +1,500 @@ + + +> **Note:** This plugin requires {{% product-name %}}.8.2 or later. + + +The AMQP Subscriber Plugin enables real-time ingestion of AMQP messages (RabbitMQ and other AMQP-compatible brokers) into {{% product-name %}}. Subscribe to queues and automatically transform messages into time-series data with support for JSON, Line Protocol, and custom text formats. The plugin provides flexible message acknowledgement policies and comprehensive error tracking with statistics. + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. This plugin supports TOML configuration files for complex mapping scenarios, which can be specified using the `config_file_path` parameter. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Required parameters + +| Parameter | Type | Default | Description | +|--------------|--------|---------------------------|--------------------------------------------------------------------------------------------------| +| `host` | string | required | AMQP broker hostname or IP address | +| `queues` | string | required | Space-separated list of queue names to consume messages from (for example, "sensor_data alerts") | +| `table_name` | string | required (json/text only) | Static InfluxDB measurement name. Required for `json`/`text` formats if `table_name_field` is not set. Not required for `lineprotocol`. | +| `table_name_field` | string | none | Field path to extract table name dynamically from message data. JSON: field name (auto-prefixed with `$.`). Text: regex pattern with capture group. Alternative to `table_name`. | + +### Connection parameters + +| Parameter | Type | Default | Description | +|----------------|---------|--------|-------------------------------------------------| +| `port` | integer | 5672 | AMQP broker port (5672 for non-TLS, 5671 for TLS) | +| `virtual_host` | string | "/" | AMQP virtual host | + +### Authentication parameters + +| Parameter | Type | Default | Description | +|------------------|--------|---------|-------------------------------------------------------------------| +| `auth_mechanism` | string | "plain" | Authentication mechanism: `plain` or `external` | +| `username` | string | none | AMQP broker username. Required when `auth_mechanism` is `plain` | +| `password` | string | none | AMQP broker password. Required when `auth_mechanism` is `plain` | + +**Authentication mechanisms:** +- `plain` - Username/password authentication (SASL PLAIN). Both `username` and `password` are required; the plugin no longer falls back to the RabbitMQ default `guest`/`guest` credentials. +- `external` - Client TLS certificate authentication (SASL EXTERNAL). Requires `ssl_ca_cert`, `ssl_client_cert`, and `ssl_client_key`. `username`/`password` are ignored. + +### TLS/SSL parameters + +| Parameter (CLI) | Parameter (TOML) | Type | Default | Description | +|-------------------|----------------------------|--------|---------|------------------------------------------------------------------| +| `ssl_ca_cert` | `[amqp.ssl]` `ca_cert` | string | none | Path to CA certificate file. Enables SSL when provided. | +| `ssl_client_cert` | `[amqp.ssl]` `client_cert` | string | none | Path to client certificate for mutual TLS | +| `ssl_client_key` | `[amqp.ssl]` `client_key` | string | none | Path to client private key for mutual TLS | + +**Note:** For mutual TLS, both `ssl_client_cert` and `ssl_client_key` must be provided together. SSL is automatically enabled when `ssl_ca_cert` is provided. + +### Message handling parameters + +| Parameter | Type | Default | Description | +|--------------------|---------|--------------|--------------------------------------------------------------------------| +| `ack_policy` | string | "on_success" | When to acknowledge messages: `on_success` or `always` | +| `requeue_on_failure` | string | "false" | Whether to requeue messages that fail processing. When `true`, failed messages are returned to the queue for retry. | +| `max_messages` | integer | 500 | Maximum number of messages to retrieve per scheduled call | + +**Acknowledgement policies:** +- `on_success` - Acknowledge only after successful processing. Failed messages are rejected (requeue controlled by `requeue_on_failure`). +- `always` - Acknowledge all messages at the end of processing, regardless of success or failure. + +### Logging parameters + +In TOML configuration, `enable_full_logging` is placed directly under the `[amqp]` section. + +| Parameter | Type | Default | Description | +|-----------------------|---------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `enable_full_logging` | boolean | false | When `true`, full exception messages are written to logs. When `false` (default), only the exception type is logged, to avoid leaking sensitive values (credentials, payloads, paths) into log output. Enable temporarily for debugging. | + +### Message format parameters + +| Parameter | Type | Default | Description | +|-------------------|--------|---------|----------------------------------------------------------------| +| `format` | string | "json" | Message format: json, lineprotocol, or text | +| `timestamp_field` | string | none | Field containing timestamp (format depends on message format) | + +**Format-specific timestamp_field syntax:** + +The timestamp field format differs between JSON and Text formats: + +| Format | Syntax | Split Method | Example (CLI) | Example (TOML) | +|----------|---------------------|--------------------|------------------|--------------------| +| JSON | `field_name:format` | Split by first `:` | `"timestamp:ms"` | `"$.timestamp:ms"` | +| Text | `regex:format` | Split by last `:` | `"ts:(\\d+):ms"` | `"ts:(\\d+):ms"` | + +**Note:** In CLI arguments, JSON paths are specified without `$.` prefix (added automatically). In TOML configuration, use full JSONPath syntax with `$.` prefix. + +**Note:** Text format uses the last colon to split, allowing regex patterns to contain colons (for example, time patterns). + +**Supported timestamp formats:** +- `ns` - nanoseconds (Unix timestamp) +- `ms` - milliseconds (Unix timestamp) +- `s` - seconds (Unix timestamp) +- `datetime` - ISO 8601 string (for example, "2021-12-01T12:00:00Z") + +### JSON format parameters + +| Parameter | Type | Default | Description | +|-----------|--------|----------|---------------------------------------------------------------------------| +| `tags` | string | none | Space-separated tag names. Example: "room sensor location" | +| `fields` | string | required | Space-separated field mappings. Format: "name:type=jsonpath" without `$.` | + +**Field specification format:** `"temp:float=temperature hum:int=humidity status:bool=online"` + +**Supported field types:** `int`, `uint`, `float`, `string`, `bool` + +### Text format parameters + +| Parameter | Type | Default | Description | +|-----------|--------|-----------|-------------------------------------------------------------------| +| `tags` | string | none | Space-separated tag patterns. Format: "name=regex_pattern" | +| `fields` | string | required | Space-separated field patterns. Format: "name:type=regex_pattern" | + +**Tag specification format:** `"room=room:([^,\\s]+) sensor=sensor:(\\w+)"` + +**Field specification format:** `"temp:float=temp:([\\d.]+) status:bool=(true|false)"` + +### TOML configuration + +| Parameter | Type | Default | Description | +|--------------------|--------|---------|-------------------------------------------------| +| `config_file_path` | string | none | Path to TOML config file (absolute or relative) | + +*To use a TOML configuration file, specify the `config_file_path` in the trigger arguments.* + +### File path resolution + +All file paths in the plugin (configuration file, TLS certificates) follow the same resolution logic: + +- **Absolute paths** (for example, `/etc/amqp/config.toml`) are used as-is +- **Relative paths** (for example, `config.toml`, `certs/ca.crt`) are resolved from `PLUGIN_DIR` environment variable + +If a relative path is specified and `PLUGIN_DIR` is not set, the plugin will return an error. + +#### Example TOML configuration + +[amqp_config_example.toml](https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/amqp_subscriber/amqp_config_example.toml) - comprehensive configuration example with all three message formats + +## Data requirements + +The plugin automatically creates the target measurement table on first write. Field mappings are required for JSON and Text formats to specify which fields to extract and their data types. + +### Message encoding requirements + +- **UTF-8 text only**: The plugin only processes UTF-8 encoded text messages. Binary messages are automatically skipped with a warning logged. +- **Non-empty payloads**: Empty or whitespace-only messages are automatically skipped. + +## Software Requirements + +- **{{% product-name %}}**: with the Processing Engine enabled +- **Python packages**: + - `pika` (AMQP client library) + - `jsonpath-ng` (JSON path parsing for JSON format) + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install required Python packages: + + ```bash + influxdb3 install package pika + influxdb3 install package jsonpath-ng + ``` +## Trigger setup + +### Scheduled ingestion with TOML configuration + +Recommended for production use with complex mappings: + +```bash +# 1. Set PLUGIN_DIR environment variable +export PLUGIN_DIR=~/.plugins + +# 2. Copy and edit configuration file +cp amqp_config_example.toml $PLUGIN_DIR/my_amqp_config.toml +# Edit my_amqp_config.toml with your broker and mapping settings + +# 3. Create the trigger +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/amqp_subscriber/amqp_subscriber.py \ + --trigger-spec "every:10s" \ + --trigger-arguments config_file_path=my_amqp_config.toml \ + amqp_ingestion +``` +### Scheduled ingestion with command-line arguments + +For simple JSON message ingestion: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/amqp_subscriber/amqp_subscriber.py \ + --trigger-spec "every:5s" \ + --trigger-arguments 'host=localhost,queues=sensor_data alerts,format=json,table_name=sensor_data,fields=temp:float=temperature hum:int=humidity,tags=location sensor_id' \ + amqp_sensors +``` +### Secure AMQP connection with TLS + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/amqp_subscriber/amqp_subscriber.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'host=secure-broker.example.com,port=5671,queues=secure_data,format=json,table_name=secure_data,ssl_ca_cert=certs/ca.crt,username=myuser,password=mypass,fields=value:float=value' \ + secure_amqp +``` +## Message Formats + +### JSON Format + +The primary use case for structured IoT data. Supports nested fields using JSONPath expressions. + +#### TOML Configuration + +```toml +[amqp] +host = "localhost" +port = 5672 +queues = ["sensor_data", "alerts"] +format = "json" + +[mapping.json] +table_name = "sensor_data" +timestamp_field = "$.timestamp:ms" + +[mapping.json.tags] +location = "$.location" +sensor_id = "$.sensor.id" + +[mapping.json.fields] +temperature = ["$.temp", "float"] +humidity = ["$.humidity", "int"] +status = ["$.online", "bool"] +``` +#### Example Message + +```json +{ + "timestamp": 1638360000000, + "location": "warehouse_a", + "sensor": { + "id": "sensor_001" + }, + "temp": 22.5, + "humidity": 65, + "online": true +} +``` +#### Resulting Data + +``` +sensor_data,location=warehouse_a,sensor_id=sensor_001 temperature=22.5,humidity=65i,status=true 1638360000000000000 +``` +#### JSON Array Support + +Process batch messages containing arrays of JSON objects: + +```json +[ + {"timestamp": 1638360000000, "sensor_id": "001", "temperature": 22.5}, + {"timestamp": 1638360001000, "sensor_id": "002", "temperature": 23.1}, + {"timestamp": 1638360002000, "sensor_id": "003", "temperature": 21.8} +] +``` +**Array processing behavior:** +- Each array element is processed independently as a separate data point +- If one element fails to parse, the others continue processing (partial success) +- Parse errors for individual elements are logged to `amqp_exceptions` table +- Statistics count each AMQP message as one unit (messages_received+=1, messages_processed+=1), regardless of array size + +### Line Protocol Format + +For messages already in InfluxDB line protocol format, use passthrough mode. No mapping configuration needed - messages are validated and written directly. + +#### TOML Configuration + +```toml +[amqp] +host = "rabbitmq.example.com" +queues = ["influxdb_metrics"] +format = "lineprotocol" +``` +#### CLI Configuration + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename amqp_subscriber.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'host=rabbitmq.example.com,queues=influxdb_metrics,format=lineprotocol' \ + amqp_lineprotocol +``` +#### Example Message + +``` +sensor_data,location=warehouse_a,sensor_id=001 temperature=22.5,humidity=65i 1638360000000000000 +``` +#### Supported Line Protocol Types + +| Type | Suffix | Example | +|------------------|----------------|--------------------| +| Float | none | `temperature=22.5` | +| Integer | `i` | `count=100i` | +| Unsigned Integer | `u` | `bytes=1024u` | +| String | `"..."` | `status="running"` | +| Boolean | `true`/`false` | `active=true` | + +### Text Format + +Parse plain text messages using regular expressions: + +#### TOML Configuration + +```toml +[amqp] +format = "text" + +[mapping.text] +table_name = "sensor_logs" +timestamp_field = "ts:(\\d+):ms" + +[mapping.text.tags] +location = "location=([^,\\s]+)" + +[mapping.text.fields] +temperature = ["temp:([\\d.]+)", "float"] +humidity = ["hum:(\\d+)", "int"] +status = ["status:(true|false)", "bool"] +``` +#### Example Message + +``` +location=warehouse_a,temp:22.5,hum:65,status:true,ts:1638360000000 +``` +## Statistics and Monitoring + +The plugin tracks comprehensive statistics and writes them to the `amqp_stats` table on every plugin invocation. + +**Important notes:** +- Statistics are written **on every plugin invocation** +- Each queue is tracked separately with independent statistics +- Statistics persist across plugin restarts using the InfluxDB cache + +### amqp_stats Table + +| Field | Type | Description | +|-------------------------|-------|----------------------------------------------------------| +| `queue` (tag) | tag | AMQP queue name | +| `host` (tag) | tag | AMQP broker address | +| `virtual_host` (tag) | tag | AMQP virtual host | +| `messages_received` | int | Total messages received (all time) | +| `messages_processed` | int | Total messages processed (all time) | +| `messages_failed` | int | Total messages failed (all time) | +| `success_rate` | float | Total success rate (all time, %) | +| `period_received` | int | Messages received in current period | +| `period_processed` | int | Messages processed in current period | +| `period_failed` | int | Messages failed in current period | +| `period_success_rate` | float | Success rate for current period (%) | + +### Querying Statistics + +```bash +# Get latest statistics +influxdb3 query --database mydb \ + "SELECT * FROM amqp_stats ORDER BY time DESC LIMIT 10" + +# Success rate over time +influxdb3 query --database mydb \ + "SELECT queue, success_rate, messages_processed, messages_failed + FROM amqp_stats + WHERE time > now() - INTERVAL '1 hour' + ORDER BY time DESC" +``` +## Error Handling + +Parse errors and message processing failures are logged to the `amqp_exceptions` table: + +### amqp_exceptions Table + +| Field | Type | Description | +|-------------------|--------|--------------------------------------------| +| `queue` (tag) | tag | AMQP queue where error occurred | +| `error_type` (tag)| tag | Type of error (for example, JSONDecodeError) | +| `error_message` | string | Detailed error message | +| `raw_message` | string | Original AMQP message (truncated to 1KB) | + +### Checking for Errors + +```bash +influxdb3 query --database mydb \ + "SELECT * FROM amqp_exceptions ORDER BY time DESC LIMIT 10" +``` +## Troubleshooting + +### Check Plugin Logs + +```bash +influxdb3 query --database _internal \ + "SELECT * FROM system.processing_engine_logs + WHERE trigger_name = 'amqp_ingestion' + ORDER BY time DESC LIMIT 20" +``` +### Common Issues + +#### "pika library not installed" + +```bash +influxdb3 install package pika +``` +#### "Configuration file not found" + +- For relative paths, ensure `PLUGIN_DIR` environment variable is set +- For absolute paths, verify the file exists at the specified location + +```bash +# For relative paths +export PLUGIN_DIR=~/.plugins +ls $PLUGIN_DIR/my_amqp_config.toml + +# Or use absolute path +ls /etc/amqp/my_amqp_config.toml +``` +#### "Failed to connect to AMQP broker" + +- Verify broker address and port +- Check network connectivity +- For TLS connections, verify certificate paths +- Verify authentication credentials +- Verify `auth_mechanism` matches the broker configuration (`plain` requires `username`/`password`; `external` requires client certificates) + +#### "Both ssl_client_cert and ssl_client_key must be provided for mutual TLS" + +For mutual TLS authentication, both client certificate and key are required. + +#### "No fields were mapped from JSON data" + +- Verify JSONPath expressions in field mappings (use `$.` prefix) +- Check that JSON structure matches your paths +- Review `amqp_exceptions` table for detailed errors + +#### Messages not being processed + +- Check trigger status: `influxdb3 show summary --database mydb` +- Verify AMQP connection in plugin logs +- Ensure the queue exists and has messages +- Check that the queue name is correct + +#### Failed messages are being discarded + +By default, failed messages are rejected without requeue (`requeue_on_failure=false`): +- Set `requeue_on_failure=true` to return failed messages to the queue for retry +- Use `ack_policy=always` to acknowledge all messages regardless of errors +- Check `amqp_exceptions` table to diagnose parsing issues +- Configure a dead-letter queue in RabbitMQ to capture rejected messages for analysis + +## Architecture + +### How It Works + +1. **Scheduled Trigger**: Plugin runs on schedule (for example, `every:10s`) +2. **Configuration Caching**: Plugin configuration is parsed once and cached between executions +3. **Message Retrieval**: Plugin connects, retrieves up to `max_messages` from queue using `basic_get` +4. **Parse & Write**: Messages parsed according to format and written to InfluxDB +5. **Acknowledgement**: Messages acknowledged based on `ack_policy` setting +6. **Error Tracking**: Parse errors logged to `amqp_exceptions` table +7. **Statistics**: Written to `amqp_stats` table on every plugin invocation +8. **Disconnect**: Connection closed after processing + +### Performance Optimization + +The plugin includes several optimizations for high-throughput scenarios: + +- **Configuration Caching**: Plugin configuration is parsed once and reused across all trigger executions +- **Pre-compiled Patterns**: JSONPath expressions and regex patterns are compiled once during parser initialization, not per-message +- **Batch Retrieval**: Multiple messages retrieved in a single connection session +- **QoS Control**: Prefetch count limits memory usage on consumer side + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/basic-transformation.md b/content/shared/influxdb3-plugins/plugins-library/official/basic-transformation.md index 4d825bbe4b..538e9c84a4 100644 --- a/content/shared/influxdb3-plugins/plugins-library/official/basic-transformation.md +++ b/content/shared/influxdb3-plugins/plugins-library/official/basic-transformation.md @@ -1,5 +1,5 @@ - + The Basic Transformation Plugin enables real-time and scheduled transformation of time series data in {{% product-name %}}. Transform field and tag names, convert values between units, and apply custom string replacements to standardize or clean your data. The plugin supports both scheduled batch processing of historical data and real-time transformation as data is written. ## Configuration @@ -14,12 +14,13 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor ### Required parameters -| Parameter | Type | Default | Description | -|----------------------|--------|------------------|---------------------------------------------------| -| `measurement` | string | required | Source measurement containing data to transform | -| `target_measurement` | string | required | Destination measurement for transformed data | -| `target_database` | string | current database | Database for storing transformed data | -| `dry_run` | string | "false" | When "true", logs transformations without writing | +| Parameter | Type | Default | Description | +|-----------------------|---------|------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `measurement` | string | required | Source measurement containing data to transform | +| `target_measurement` | string | required | Destination measurement for transformed data | +| `target_database` | string | current database | Database for storing transformed data | +| `dry_run` | string | "false" | When "true", logs transformations without writing | +| `enable_full_logging` | boolean | false | When `true`, full exception messages are written to logs. When `false` (default), only the exception type is logged, to avoid leaking sensitive values. Enable temporarily for debugging. | ### Transformation parameters @@ -62,7 +63,7 @@ The plugin assumes that the table schema is already defined in the database, as - **{{% product-name %}}**: with the Processing Engine enabled - **Python packages**: - - `pint` (for unit conversions) + - `pint` (for unit conversions) ## Installation steps @@ -403,6 +404,7 @@ For plugin issues, see the Plugins repository [issues page](https://github.com/i The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + ## Schema requirements diff --git a/content/shared/influxdb3-plugins/plugins-library/official/bird-data-simulator.md b/content/shared/influxdb3-plugins/plugins-library/official/bird-data-simulator.md new file mode 100644 index 0000000000..900d98bc4a --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/bird-data-simulator.md @@ -0,0 +1,171 @@ + + +The Bird Tracking Simulator Plugin generates a stream of synthetic bird telemetry for demos, testing, and sample-data workflows. +On its first run, it creates a persistent flock of named birds, assigns each bird a species, natural range, starting location, heading, and healthy body temperature, then stores that flock in the Processing Engine cache. +Each scheduled call advances the flock with sinusoidal flight speed, gentle heading changes, latitude and longitude updates, and temperature jitter. + +The plugin intentionally exposes only volume controls. + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. +The trigger interval plus the options below control how much data the plugin writes. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. +This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Optional parameters + +There are no required parameters. +All configuration parameters control data volume only. + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `bird_count` | integer | `25` | Number of persistent simulated birds to track | +| `points_per_bird` | integer | `1` | Number of movement points to emit for each bird on each scheduled call. When greater than `1`, timestamps are evenly spaced across the elapsed time since the previous call. | + +### TOML configuration + +This plugin does not expose TOML configuration. +Use the two inline volume options above and the trigger interval to control output volume. + +## Requirements + +### Data requirements + +This plugin does not require incoming writes or source measurements. +It generates data directly from a scheduled trigger. +The generated flock state is stored in the Processing Engine's trigger-specific cache. +Changing `bird_count` creates a new cached flock with the requested size. + +### Schema requirements + +The plugin writes to the `bird_tracking` measurement. + +Tags: + +- `species`: common species name, such as `American Robin` +- `name`: generated name for the individual bird + +Fields: + +- `body_temp`: body temperature in degrees Celsius +- `longitude`: current longitude in decimal degrees +- `latitude`: current latitude in decimal degrees +- `speed`: current speed in miles per hour +- `heading`: current heading in degrees, where `0` is north and `90` is east + +### Species metadata + +The plugin embeds 20 United States bird species with simplified ranges, weight ranges, healthy body temperature ranges, and approximate top flight speeds directly in `bird_data_simulator.py`. + +The species catalog uses [Cornell Lab All About Birds](https://www.allaboutbirds.org/guide/) species accounts and range maps as the primary reference for species presence, range, habitat, and measurements. +General healthy body temperature ranges are based on published avian veterinary reference values such as the [Merck Veterinary Manual normal temperature table](https://www.merckvetmanual.com/reference-values-and-conversion-tables/reference-guides/normal-rectal-temperature-ranges). + +### Software requirements + +- **{{% product-name %}}**: with the Processing Engine enabled. +- **Python packages**: + - `Faker` + +## Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install `Faker` into the Processing Engine Python environment: + + ```bash + influxdb3 install package Faker + ``` +## Trigger setup + +### Basic scheduled trigger + +```bash +influxdb3 create trigger \ + --database sample_data \ + --path "gh:influxdata/bird_data_simulator/bird_data_simulator.py" \ + --trigger-spec "every:1s" \ + bird_tracking +``` +### Larger flock + +```bash +influxdb3 create trigger \ + --database sample_data \ + --path "gh:influxdata/bird_data_simulator/bird_data_simulator.py" \ + --trigger-spec "every:1s" \ + --trigger-arguments bird_count=100 \ + bird_tracking_large +``` +### More points per bird + +```bash +influxdb3 create trigger \ + --database sample_data \ + --path "gh:influxdata/bird_data_simulator/bird_data_simulator.py" \ + --trigger-spec "every:10s" \ + --trigger-arguments bird_count=50,points_per_bird=10 \ + bird_tracking_dense +``` +## Example usage + +### Generate bird telemetry + +```bash +# Create a small flock that writes once per second. +influxdb3 create trigger \ + --database sample_data \ + --path "gh:influxdata/bird_data_simulator/bird_data_simulator.py" \ + --trigger-spec "every:1s" \ + --trigger-arguments bird_count=10 \ + bird_tracking_demo + +# Query generated points after the trigger runs. +influxdb3 query \ + --database sample_data \ + "SELECT * FROM bird_tracking ORDER BY time DESC LIMIT 5" +``` +### Expected output + +```text +species | name | body_temp | longitude | latitude | speed | heading | time +-----------------|-------|-----------|-------------|-----------|-------|---------|--------------------- +American Robin | Willa | 41.822 | -83.182337 | 39.912884 | 21.4 | 83.2 | 2026-04-29T12:00:04Z +Cactus Wren | Felix | 42.117 | -111.913552 | 33.382018 | 8.7 | 244.9 | 2026-04-29T12:00:04Z +Florida Scrub-Jay| Pearl | 41.603 | -81.224901 | 28.399102 | 12.1 | 11.6 | 2026-04-29T12:00:04Z +``` + +## Logging + +Logs are stored in the `_internal` database (or the database where the trigger is created) in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" +``` + +Log columns: +- **event_time**: Timestamp of the log event +- **trigger_name**: Name of the trigger that generated the log +- **log_level**: Severity level (INFO, WARN, ERROR) +- **log_text**: Message describing the action or error + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/chronos-forecasting.md b/content/shared/influxdb3-plugins/plugins-library/official/chronos-forecasting.md new file mode 100644 index 0000000000..42909475b5 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/chronos-forecasting.md @@ -0,0 +1,299 @@ + + +⚡ scheduled, http +🏷️ forecasting, machine-learning, time-series, deep-learning +🔧 {{% product-name %}} + +> **Note:** This plugin requires {{% product-name %}}.8.2 or later. + + +The Chronos Forecasting Plugin enables zero-shot time-series forecasting for data in {{% product-name %}} using Amazon's Chronos model family from HuggingFace. +Generate predictions for future data points without model training, using pre-trained transformer models. +Supports both scheduled batch forecasting and on-demand HTTP-triggered forecasts. + +- **Zero-shot inference**: No training required — pre-trained models generalize to any time series +- **Model flexibility**: Supports Chronos-2 (group attention multivariate), Chronos-Bolt (fast), and original Chronos-T5 +- **Multivariate support**: Chronos-2 models accept covariate fields for multivariate forecasting; `covariate_mode` selects whether they are used as auxiliary `past_covariates` or as jointly forecast target series +- **Prediction intervals**: Returns 50% and 80% prediction intervals alongside median forecasts +- **Model caching**: Downloaded models are cached on disk (HuggingFace cache) for fast reloads + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. +Some plugins support TOML configuration files, which can be specified using the plugin's `config_file_path` parameter. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. +This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Scheduled trigger parameters + +| Parameter | Type | Default | Description | +|----------------------|--------|------------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| `measurement` | string | required | Source table containing historical time-series data | +| `field` | string | required | Numeric field name to forecast | +| `window` | string | required | Historical lookback window. Format: `` (s, min, h, d) | +| `horizon` | int | required | Number of forecast steps to generate | +| `target_measurement` | string | `_forecasts.{measurement}` | Destination table for forecast results | +| `model_id` | string | `amazon/chronos-bolt-tiny` | HuggingFace model ID | +| `context_limit` | int | `512` | Maximum data points fed to the model | +| `agg_interval` | string | `30s` | Aggregation interval for `date_bin` query | +| `tag_values` | string | none | Dot-separated tag filters, values joined by `@` (e.g. `tag:v1@v2.tag2:v3`) | +| `covariate_fields` | string | none | Space-separated covariate field names. Setting this enables Chronos-2 multivariate forecasting (requires a Chronos-2 model) | +| `covariate_mode` | string | `covariate` | How covariates are used (Chronos-2): `covariate` (auxiliary past covariates) or `target` (jointly forecast all series) | +| `target_database` | string | current | Database for forecast storage | + +### HTTP trigger parameters + +HTTP parameters are sent in the JSON request body. Any value also set as a trigger argument is used as a default and overridden by the request body. The `covariate_fields` value may be a space-separated string or a JSON array. + +| Parameter | Type | Default | Description | +|----------------------|--------|------------------------------|-----------------------------------------------------------------------------------------------------------------------------| +| `table` | string | required | Source table name containing historical data | +| `field` | string | required | Numeric field name to forecast | +| `horizon` | int | `64` | Number of forecast steps to generate | +| `context_limit` | int | `512` | Maximum context window size (data points) | +| `model_id` | string | `amazon/chronos-bolt-tiny` | HuggingFace model ID | +| `covariate_fields` | string | none | Space-separated covariate field names. Setting this enables Chronos-2 multivariate forecasting (requires a Chronos-2 model) | +| `covariate_mode` | string | `covariate` | How covariates are used (Chronos-2): `covariate` (auxiliary past covariates) or `target` (jointly forecast all series) | +| `write_results` | string | `false` | Write forecast results to the database | +| `target_measurement` | string | none | Destination table for results (required if `write_results` is true) | +| `target_database` | string | current | Database for forecast storage | + +> **`where_clause`**: An optional SQL `WHERE` clause for filtering source data, passed in the request body like any other parameter. Example: `{"where_clause": "host = 'server1'"}`. + +### TOML configuration + +| Parameter | Type | Default | Description | +|--------------------|--------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------| +| `config_file_path` | string | none | Path to a TOML config file: absolute, or relative to the plugin directory (`INFLUXDB3_PLUGIN_DIR` or `PLUGIN_DIR`). Required for TOML configuration | + +*To use a TOML configuration file, specify the `config_file_path` in the trigger arguments. Relative paths are resolved from the plugin directory (`INFLUXDB3_PLUGIN_DIR` or `PLUGIN_DIR`), with a fallback to the processing engine's virtual environment; absolute paths are used as-is.* + +#### Example TOML configuration + +[chronos_forecasting_scheduler.toml](https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/chronos_forecasting/chronos_forecasting_scheduler.toml) + +## Software requirements + +- **{{% product-name %}}**: with the Processing Engine enabled. +- **Python packages**: + - `chronos-forecasting` (Amazon Chronos model pipeline) + - `torch` (PyTorch for model inference) + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install required Python packages: + + ```bash + influxdb3 install package chronos-forecasting + influxdb3 install package torch + ``` +## Trigger setup + +### HTTP trigger + +Create a trigger for on-demand forecasting: + +```bash +influxdb3 create trigger \ + --database mydb \ + --path chronos_forecasting.py \ + --trigger-spec "request:forecast_series" \ + chronos_forecast_http +``` +### Scheduled trigger + +Create a trigger for periodic forecasting: + +```bash +influxdb3 create trigger \ + --database mydb \ + --path chronos_forecasting.py \ + --trigger-spec "every:5m" \ + --trigger-arguments "measurement=sensor_data,field=temperature,window=6h,horizon=64" \ + chronos_forecast_scheduled +``` +### Enable triggers + +```bash +influxdb3 enable trigger --database mydb chronos_forecast_http +influxdb3 enable trigger --database mydb chronos_forecast_scheduled +``` +## Example usage + +### On-demand HTTP forecast + +Parameters are sent in the JSON request body: + +```bash +curl -X POST "http://localhost:8181/api/v3/engine/forecast_series" \ + -H "Content-Type: application/json" \ + -d '{"table": "sensor_data", "field": "temperature", "horizon": 28, "context_limit": 128}' +``` +### Expected output + +```json +{ + "status": "ok", + "model_id": "amazon/chronos-bolt-tiny", + "table": "sensor_data", + "field": "temperature", + "context_length": 128, + "horizon": 28, + "load_seconds": 0.03, + "inference_ms": 42.1, + "step_ms": 30000, + "historical": [ + {"timestamp": 1776300000000, "value": 21.34} + ], + "forecast": [ + {"step": 1, "timestamp": 1776300030000, "median": 21.5, "lower_80": 19.8, "upper_80": 23.1, "lower_50": 20.3, "upper_50": 22.7} + ] +} +``` +### Error responses + +On failure, the HTTP endpoint returns a JSON object with `status` set to `error` and a human-readable `error` message (HTTP path only; the scheduled trigger logs errors instead): + +```json +{"status": "error", "error": "Only 4 data points — need at least 10"} +``` +### Filtering source data with a WHERE clause + +Add `where_clause` to the request body: + +```bash +curl -X POST "http://localhost:8181/api/v3/engine/forecast_series" \ + -H "Content-Type: application/json" \ + -d '{"table": "sensor_data", "field": "temperature", "where_clause": "host = '"'"'server1'"'"'"}' +``` +### Multivariate forecast with Chronos-2 + +```bash +curl -X POST "http://localhost:8181/api/v3/engine/forecast_series" \ + -H "Content-Type: application/json" \ + -d '{"table": "sensor_data", "field": "temperature", "model_id": "amazon/chronos-2", "covariate_fields": ["humidity", "pressure"], "horizon": 64}' +``` +## Code overview + +### Key functions + +- **`process_request(influxdb3_local, query_parameters, request_headers, request_body, args=None)`**: HTTP entry point for on-demand forecasting. + Queries historical data, runs inference, and returns JSON with historical context and forecast points including confidence intervals. +- **`process_scheduled_call(influxdb3_local, call_time: datetime, args: dict | None = None)`**: Scheduled entry point for recurring forecasts. + Reads config from trigger args or TOML, queries with `date_bin` aggregation, runs inference, and writes results back via `LineBuilder`. +- **`_load_model(model_id, influxdb3_local, task_id)`**: Loads Chronos models from HuggingFace. + The plugin module is reloaded on every trigger run, so there is no in-process model cache; reloads are served from the on-disk HuggingFace cache (`HF_HOME`). +- **`_build_query_simple(table, fields, where_clause, context_limit)` / `_build_query_aggregated(measurement, fields, window, agg_interval, tag_values, context_limit)`**: Build the SQL that selects the target and any covariate fields in a single query, so every row stays time-aligned (covariates are aligned to the target by bin/row, not by length). +- **`_query_aligned_series(influxdb3_local, sql, aliases, time_col, reverse=False)` / `_fill_nulls(seq)`**: Execute the query, drop rows with a null target, keep covariate columns row-aligned, and forward/back-fill covariate gaps. +- **`_build_model_input(target_vals, covariates, model_id, covariate_mode)`**: Builds the model input appropriate to the model type. + For Chronos-2 covariates: `covariate` mode passes them as `past_covariates`; `target` mode stacks them as extra target variates (`[1, channels, length]`) for group attention. Bolt and univariate cases use a list of 1D tensors. +- **`_run_inference(pipeline, tensor, horizon, target_channel=0)`**: Runs `predict_quantiles` for the quantile levels in `[0.1, 0.25, 0.5, 0.75, 0.9]` (within the trained `[0.1, 0.9]` range; `0.25`/`0.75` are interpolated) and returns the matrix transposed to `[quantiles, horizon]`. +- **`_write_lines(influxdb3_local, lines, target_database, task_id)`**: Writes `LineBuilder` objects synchronously via `write_sync` / `write_sync_to_db`. + +### Docker environment notes + +The plugin sets environment variables at import time for Docker compatibility: + +- `USER=influxdb3` — HuggingFace Hub requires a username +- `HF_HOME=/tmp/hf_cache` — writable cache directory for model downloads +- `HOME=/tmp` — fallback home directory for unmapped UIDs +- `TORCHDYNAMO_DISABLE=1` — disables `torch.compile` to avoid conflicts in embedded Python +- `HF_HUB_DISABLE_PROGRESS_BARS=1`, `TQDM_DISABLE=1` — disable tqdm/HuggingFace progress bars + +These are no-ops when running outside Docker with a normal user environment. + +## Output data structure + +Forecast results (scheduled mode) are written to the target measurement with the following structure: + +### Tags + +- `field`: source field name +- `model`: model label (for example, `chronos-bolt-tiny`) +- `source_table`: source measurement name +- `mode`: `univariate` or `multivariate` +- Additional tags from `tag_values` configuration + +### Fields + +- `forecast`: median predicted value +- `lower_80`, `upper_80`: 80% prediction interval bounds (quantiles 0.1 / 0.9) +- `lower_50`, `upper_50`: 50% prediction interval bounds (quantiles 0.25 / 0.75) +- `step`: forecast step number (1-indexed) + +### Timestamp + +- `time`: future timestamp in nanoseconds, spaced by `agg_interval` starting from the last observed data point + +## Troubleshooting + +### Common issues + +**Model download failures** + +- Ensure the InfluxDB host has internet access for the first model download from HuggingFace. +- Verify `HF_HOME` is writable. + In Docker, the plugin automatically sets `HF_HOME=/tmp/hf_cache`. +- For air-gapped environments, pre-download models and set `model_id` to the local path. + +**Insufficient data** + +- The plugin requires at least 10 data points. + Ensure the `window` parameter covers enough data at the specified `agg_interval`. +- For a `30s` interval and `6h` window, expect up to 720 points (capped at `context_limit`). + +**Slow inference** + +- `chronos-bolt-tiny` (~8M params) provides near-instant CPU inference. +- `amazon/chronos-2` is significantly slower on CPU (1-3 seconds per forecast). +- The first invocation downloads the model; subsequent calls reload it from the on-disk HuggingFace cache (`HF_HOME`). There is no in-process model cache, since the plugin module is reloaded on every trigger run. + +**Multivariate mode not engaging** + +- Set `covariate_fields` with at least one field name (this alone enables multivariate mode). +- Use a Chronos-2 model (`model_id` containing "chronos-2" or "chronos_2"). + Bolt and original Chronos models do not support group attention. +- Choose how covariates are treated with `covariate_mode`: `covariate` (default) passes them as auxiliary `past_covariates`; `target` jointly forecasts the target and all covariate series via group attention. + +**Timestamp issues in HTTP response** + +- If timestamps return as `null`, verify that `influxdb3_local.query()` returns parseable timestamp values. +- The plugin handles nanosecond integers, datetime objects, and ISO strings. + + +## Logging + +Logs are stored in the `_internal` database (or the database where the trigger is created) in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" +``` + +Log columns: +- **event_time**: Timestamp of the log event +- **trigger_name**: Name of the trigger that generated the log +- **log_level**: Severity level (INFO, WARN, ERROR) +- **log_text**: Message describing the action or error + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/downsampler.md b/content/shared/influxdb3-plugins/plugins-library/official/downsampler.md index 2213ff96c0..56489d460a 100644 --- a/content/shared/influxdb3-plugins/plugins-library/official/downsampler.md +++ b/content/shared/influxdb3-plugins/plugins-library/official/downsampler.md @@ -1,4 +1,5 @@ - + + The Downsampler Plugin enables time-based data aggregation and downsampling in {{% product-name %}}. Reduce data volume by aggregating measurements over specified time intervals using functions like avg, sum, min, max, median, count, stddev, first_value, last_value, var, or approx_median. The plugin supports both scheduled batch processing of historical data and on-demand downsampling through HTTP requests. Each downsampled record includes metadata about the original data points compressed. ## Configuration @@ -11,37 +12,42 @@ If a plugin supports multiple trigger specifications, some parameters may depend This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. -### Required parameters - -| Parameter | Type | Default | Description | -|----------------------|--------|---------------------------|------------------------------------------------------------------------------------| -| `source_measurement` | string | required | Source measurement containing data to downsample | -| `target_measurement` | string | required | Destination measurement for downsampled data | -| `window` | string | required (scheduled only) | Time window for each downsampling job. Format: `` (for example, "1h", "1d") | - -### Aggregation parameters - -| Parameter | Type | Default | Description | -|-------------------|--------|------------|--------------------------------------------------------------------------------------| -| `interval` | string | "10min" | Time interval for downsampling. Format: `` (for example, "10min", "2h", "1d") | -| `calculations` | string | "avg" | Aggregation functions. Single function or dot-separated field:aggregation pairs | -| `specific_fields` | string | all fields | Dot-separated list of fields to downsample (for example, "co.temperature") | -| `excluded_fields` | string | none | Dot-separated list of fields and tags to exclude from downsampling results | - -### Filtering parameters - -| Parameter | Type | Default | Description | -|--------------|--------|---------|---------------------------------------------------------------------| -| `tag_values` | string | none | Tag filters. Format: `tag:value1@value2@value3` for multiple values | -| `offset` | string | "0" | Time offset to apply to the window | - -### Advanced parameters - -| Parameter | Type | Default | Description | -|-------------------|---------|-----------|-----------------------------------------------------| -| `target_database` | string | "default" | Database for storing downsampled data | -| `max_retries` | integer | 5 | Maximum number of retries for write operations | -| `batch_size` | string | "30d" | Time interval for batch processing (HTTP mode only) | +### Scheduled trigger parameters + +| Parameter | Type | Default | Description | +|----------------------|---------|------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `source_measurement` | string | required | Source measurement containing data to downsample | +| `target_measurement` | string | required | Destination measurement for downsampled data | +| `window` | string | required | Time window for each downsampling job. Format: `` (for example, "1h", "1d") | +| `interval` | string | "10min" | Time interval for downsampling. Format: `` (for example, "10min", "2h", "1d") | +| `calculations` | string | "avg" | Aggregation functions. Single function or dot-separated field:aggregation pairs | +| `specific_fields` | string | all fields | Dot-separated list of fields to downsample (for example, "co.temperature") | +| `excluded_fields` | string | none | Dot-separated list of fields and tags to exclude from downsampling results | +| `tag_values` | string | none | Tag filters. Format: `tag:value1@value2@value3` for multiple values | +| `offset` | string | "0" | Time offset to apply to the window | +| `target_database` | string | "default" | Database for storing downsampled data | +| `max_retries` | integer | 5 | Maximum number of retries for write operations | +| `enable_full_logging` | boolean | false | When `true`, full exception messages are written to logs. When `false` (default), only the exception type is logged, to avoid leaking sensitive values. Enable temporarily for debugging. | + +### HTTP request parameters + +Send these parameters as JSON in the HTTP POST request body: + +| Parameter | Type | Default | Description | +|-----------------------|---------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `source_measurement` | string | required | Source measurement containing data to downsample | +| `target_measurement` | string | required | Destination measurement for downsampled data | +| `interval` | string | "10min" | Time interval for downsampling. Format: `` (for example, "10min", "2h", "1d") | +| `batch_size` | string | "30d" | Time interval for each HTTP backfill batch | +| `calculations` | string/array | "avg" | Aggregation functions. Use `"avg"` for all fields, or an array of `[field, aggregation]` pairs | +| `specific_fields` | array | all fields | List of fields to downsample | +| `excluded_fields` | array | none | List of fields and tags to exclude from downsampling results | +| `tag_values` | object | none | Tag filters as an object mapping tag names to lists of values | +| `target_database` | string | "default" | Database for storing downsampled data | +| `max_retries` | integer | 5 | Maximum number of retries for write operations | +| `backfill_start` | string | oldest point | ISO 8601 datetime with timezone for the start of the HTTP backfill window | +| `backfill_end` | string | current time | ISO 8601 datetime with timezone for the end of the HTTP backfill window | +| `enable_full_logging` | boolean | false | When `true`, full exception messages are written to logs. When `false` (default), only the exception type is logged, to avoid leaking sensitive values. Enable temporarily for debugging. | ### TOML configuration @@ -224,7 +230,7 @@ Key operations: 3. Applies time-based aggregation with specified functions 4. Writes downsampled data with metadata columns -#### `process_http_request(influxdb3_local, request_body, args)` +#### `process_request(influxdb3_local, query_parameters, request_headers, request_body, args)` Handles HTTP-triggered on-demand downsampling. Processes batch downsampling with configurable time ranges for backfill scenarios. @@ -235,9 +241,9 @@ Key operations: 3. Applies aggregation functions to historical data 4. Returns processing statistics and results -#### `aggregate_data(data, interval, calculations)` +#### `build_downsample_query(fields_list, measurement, tags_list, interval, tag_values, start_time, end_time)` -Core aggregation engine that applies statistical functions to time-series data. +Builds the SQL query that applies time bucketing, tag filters, grouping, and aggregation functions to source data. Supported aggregation functions: @@ -361,4 +367,6 @@ For plugin issues, see the Plugins repository [issues page](https://github.com/i ## Find support for {{% product-name %}} The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. -For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. \ No newline at end of file +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/earthquake-sampler.md b/content/shared/influxdb3-plugins/plugins-library/official/earthquake-sampler.md new file mode 100644 index 0000000000..4e9eec3160 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/earthquake-sampler.md @@ -0,0 +1,285 @@ + + +The Earthquake Sampler Plugin ingests earthquake events from the USGS GeoJSON feeds on a schedule and writes normalized points for dashboards and alerting. It can also read from a custom JSON endpoint or from an existing InfluxDB table, and optionally writes directly into an existing canonical `quake` table using that table's column names (`write_quake_schema=true`). Events are deduplicated with a per-event update-marker cache, so reruns only write new or updated earthquakes. + +- **Zero authentication**: USGS feeds require no API keys or signup +- **Two source modes**: HTTP (USGS or custom JSON) and `influxdb_table` (transform rows from an existing table) +- **Per-event deduplication**: update markers are cached per event id; updated events are re-written, unchanged events are skipped +- **Canonical quake schema**: optional direct writes into an existing `quake` table + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. All parameters are optional; invalid values abort the run with an error log rather than silently falling back to defaults. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Data selection parameters + +| Parameter | Type | Default | Description | +|--------------------|---------|----------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `feed` | string | all_hour | USGS GeoJSON feed key: `all`, `significant`, `4.5`, `2.5`, or `1.0` combined with `_hour`, `_day`, `_week`, or `_month` (for example `significant_day`) | +| `source_type` | string | http | Data source type: `http` fetches JSON from `source_url` or `feed`; `influxdb_table` queries an existing table in the trigger database | +| `source_url` | string | none | Custom source URL (`http` or `https` only). When provided, overrides `feed` and uses `source_format` parsing | +| `source_format` | string | usgs_geojson | Source parser for HTTP mode: `usgs_geojson` or `flat_json` (for records like `{id, latitude, longitude, mag, time, ...}`) | +| `source_table` | string | quake | Source table name when `source_type=influxdb_table` | +| `source_query` | string | none | Optional SQL override for `influxdb_table` mode; disables watermark paging and the source-table existence check. A query containing commas must come from a TOML file — see [TOML configuration](#toml-configuration) | +| `lookback_minutes` | integer | 15 | Initial lookback window for `influxdb_table` mode. Later runs page forward from the cached fetch watermark while `skip_unchanged=true` | + +### Optional parameters + +| Parameter | Type | Default | Description | +|-----------------------|---------|---------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `measurement` | string | earthquakes | Destination measurement name | +| `write_quake_schema` | boolean | false | Write events using the canonical `quake` table's column names; no tags or normalized-only columns are written. Use with `measurement=quake` | +| `min_magnitude` | float | none | Optional minimum magnitude. When omitted, nothing is filtered (USGS feeds include negative-magnitude microseisms and events without a magnitude) | +| `max_events` | integer | 250 | Maximum events written per run, applied after filtering. Events deferred by the cap remain uncached and are written on a later run | +| `use_event_timestamp` | boolean | true | Use the event time as the point timestamp; `false` uses the trigger execution time. Ignored (forced `true`) when `write_quake_schema=true` | +| `skip_unchanged` | boolean | true | Skip events whose update marker is not newer than the last written copy of the same event. Events without an id are always written | +| `user_agent` | string | InfluxDB3-Earthquake-Plugin/1.0 | Custom User-Agent header for API requests | +| `enable_full_logging` | boolean | false | When `true`, full exception messages are logged. When `false` (default), only exception types are logged | +| `config_file_path` | string | none | Path to a TOML configuration file, relative to the plugin directory. See [TOML configuration](#toml-configuration) | + +### TOML configuration + +Trigger arguments are a comma-separated `key=value` list, so a value that itself contains a comma — most notably `source_query` — cannot be passed inline. Put those parameters in a TOML file instead and point `config_file_path` at it. The path is resolved relative to the plugin directory (`PLUGIN_DIR`, `INFLUXDB3_PLUGIN_DIR`, or the processing-engine virtualenv). + +Values in the file override the inline trigger arguments. A file that is missing, malformed, or fails validation is reported in the logs and skipped, and the run continues with the inline arguments; a path that does not end in `.toml` is rejected the same way. + +`earthquake_sampler_config_scheduler.toml` ships alongside the plugin with every +parameter documented and commented out. Copy it into your plugin directory and +uncomment what you need: + +```toml +measurement = "earthquakes" +feed = "all_day" +min_magnitude = 2.5 +max_events = 500 +skip_unchanged = true + +# A query containing commas is only expressible here, not in --trigger-arguments. +# Use a single-quoted TOML string when column names need double quotes. +source_type = "influxdb_table" +source_table = "quake" +source_query = 'SELECT time, id, mag, depth, "magType", net, updated FROM quake ORDER BY time DESC LIMIT 500' +``` +```bash +influxdb3 create trigger \ + --database quakes \ + --path "gh:influxdata/earthquake_sampler/earthquake_sampler.py" \ + --trigger-spec "every:5m" \ + --trigger-arguments "config_file_path=earthquake_sampler_config_scheduler.toml" \ + earthquakes_from_toml +``` +Types are native in TOML: `min_magnitude = 2.5` and `skip_unchanged = true` need no quoting, unlike the string values that inline arguments always deliver. + +## Schema requirements + +`write_quake_schema=true` targets an existing `quake` table with the USGS CSV column layout (`depth`, `dmin`, `gap`, `id`, `latitude`, `longitude`, `mag`, `magType`, `net`, `nst`, `place`, `rms`, `status`, `time`, `type`, and the CSV-only columns below). + +- All numeric columns (including `nst` and `magNst`) are written as float64. This matches quake tables created by CSV import, where numeric columns containing blanks are inferred as doubles. If your existing table stores these columns as int64, the writes are rejected with a type conflict. +- `depthError`, `horizontalError`, `magError`, `magNst`, `locationSource`, and `magSource` exist only in USGS CSV feeds, not GeoJSON, so they are written only when a `flat_json` or `influxdb_table` source supplies them. +- Quake-schema rows carry no tags, so point identity rests entirely on the timestamp. USGS supplies millisecond-precision times; the plugin fills the unused sub-millisecond bits with a stable per-event offset so two earthquakes in the same millisecond do not overwrite each other. Millisecond-level time is unchanged. + +## Software requirements + +- **{{% product-name %}}**: 3.8.2 or later (the plugin writes with `write_sync`), with the Processing Engine enabled +- **Python packages**: `influxdata-plugin-utils>=0.3.0` +- **Network access**: Outbound HTTPS access to `earthquake.usgs.gov` (HTTP mode) + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install the required Python package: + + ```bash + influxdb3 install package influxdata-plugin-utils + ``` +## Trigger setup + +### Scheduled trigger + +Create a trigger that ingests the USGS `all_hour` feed every two minutes: + +```bash +influxdb3 create trigger \ + --database quakes \ + --path "gh:influxdata/earthquake_sampler/earthquake_sampler.py" \ + --trigger-spec "every:2m" \ + earthquake_sampler_trigger + +# Enable the trigger +influxdb3 enable trigger --database quakes earthquake_sampler_trigger +``` +## Example usage + +### Example 1: Normalized earthquake ingestion + +Ingest the daily feed into the normalized `earthquakes` measurement: + +```bash +# Create the trigger +influxdb3 create trigger \ + --database quakes \ + --path "gh:influxdata/earthquake_sampler/earthquake_sampler.py" \ + --trigger-spec "every:5m" \ + --trigger-arguments "feed=all_day" \ + earthquakes_all_day + +# Enable the trigger +influxdb3 enable trigger --database quakes earthquakes_all_day + +# Query events (after a few minutes) +influxdb3 query \ + --database quakes \ + "SELECT event_id, magnitude, place, latitude, longitude, depth_km, time FROM earthquakes ORDER BY time DESC LIMIT 5" +``` +### Expected output + + event_id | magnitude | place | latitude | longitude | depth_km | time + -----------|-----------|------------------------------|----------|-----------|----------|----- + us7000abcd | 4.6 | 100 km SSW of Sand Point, AK | 54.5 | -161.2 | 32.4 | 2026-08-21T17:58:12.421Z + ak0261abcd | 1.4 | 12 km NNE of Palmer, AK | 61.7 | -148.9 | 27.9 | 2026-08-21T17:55:03.118Z + nc75abcdef | 0.9 | 8 km WNW of Cobb, CA | 38.8 | -122.8 | 1.6 | 2026-08-21T17:51:47.902Z + +### Example 2: Write directly into an existing quake table + +Write USGS events into an existing `quake` table using its original column names (see [Schema requirements](#schema-requirements)): + +```bash +influxdb3 create trigger \ + --database usgs \ + --path "gh:influxdata/earthquake_sampler/earthquake_sampler.py" \ + --trigger-spec "every:2m" \ + --trigger-arguments "feed=all_hour,measurement=quake,write_quake_schema=true" \ + usgs_to_quake +``` +### Example 3: Transform rows from an existing table + +Read rows from an existing `quake` table and write them as normalized events. The plugin pages forward from a cached fetch watermark, so each run picks up where the previous one stopped: + +```bash +influxdb3 create trigger \ + --database usgs \ + --path "gh:influxdata/earthquake_sampler/earthquake_sampler.py" \ + --trigger-spec "every:1m" \ + --trigger-arguments "source_type=influxdb_table,source_table=quake,measurement=earthquakes,lookback_minutes=60" \ + quake_to_earthquakes +``` +### Example 4: Magnitude filtering + +Only ingest events at or above magnitude 2.5: + +```bash +influxdb3 create trigger \ + --database quakes \ + --path "gh:influxdata/earthquake_sampler/earthquake_sampler.py" \ + --trigger-spec "every:5m" \ + --trigger-arguments "feed=all_day,min_magnitude=2.5" \ + earthquakes_m25 +``` +## Code overview + +### Files + +- `earthquake_sampler.py`: The main plugin code containing the scheduled handler for earthquake ingestion +- `requirements.txt`: Python dependencies (`influxdata-plugin-utils`, used for configuration loading, validation, and writes) +- `earthquake_sampler_config_scheduler.toml`: Example TOML configuration with every parameter documented + +### Logging + +Logs are stored in the trigger's database in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database YOUR_DATABASE "SELECT event_time, log_level, log_text FROM system.processing_engine_logs WHERE trigger_name = 'earthquake_sampler_trigger' ORDER BY event_time DESC LIMIT 20" +``` +Log lines are prefixed with a per-run task id. Logged source names strip URL credentials/query strings and truncate custom SQL. + +### Main functions + +#### `process_scheduled_call(influxdb3_local, call_time, args)` + +Handles a scheduled run end to end: validates configuration (aborting with an error on invalid values), fetches events from the configured source, normalizes and filters them, deduplicates against the per-event marker cache, and writes points with `write_sync` so write errors surface during the run. + +#### `_fetch_payload(url, user_agent)` / `_fetch_table_rows(...)` + +HTTP fetching (scheme-validated, `http`/`https` only) and table reading. Table reads page oldest-first from the cached fetch watermark (`WHERE time > watermark ORDER BY time ASC LIMIT max_events`), falling back to the `lookback_minutes` window on the first run. + +#### `_normalize_usgs_feature(feature)` / `_normalize_flat_event(item)` + +Normalize a USGS GeoJSON feature or a flat JSON/table record into the common internal event shape. Timestamps of unknown shape (ISO strings, epoch seconds/ms/us/ns) are coerced by magnitude. + +#### `_write_event(...)` / `_write_quake_event(...)` + +Build and write a point in the normalized schema (tags: `event_type`, `status`, `alert`, `net`, `mag_type`; `event_id` is a string field to avoid unbounded series cardinality) or in the canonical quake schema (no tags). Fields the source does not supply are left out of the point rather than written as empty values, and an event with no usable field at all is skipped and logged. Millisecond-aligned timestamps get a stable per-event sub-millisecond offset so same-millisecond events stay distinct. + +## Troubleshooting + +### Common issues + +#### Issue: No data appearing + +**Solution**: Check trigger status, review plugin logs, and verify network connectivity: + +```bash +# Check trigger status +influxdb3 show summary --database quakes --token YOUR_TOKEN + +# Check plugin logs +influxdb3 query --database quakes "SELECT event_time, log_level, log_text FROM system.processing_engine_logs WHERE log_text LIKE '%Earthquake%' ORDER BY event_time DESC LIMIT 10" + +# Verify the feed is reachable +curl -H "User-Agent: test" https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.geojson +``` +#### Issue: `Invalid configuration` error in logs + +**Solution**: A trigger argument has an invalid value (unknown `feed` key, non-numeric `min_magnitude`, unrecognized boolean, `max_events` below 1, and so on). The message quotes the offending value; the run aborts rather than proceeding with a silent fallback. Booleans accept `true/false`, `yes/no`, `on/off`, and `1/0`. + +#### Issue: `Source table ... not found in the trigger database` + +**Solution**: `source_type=influxdb_table` checks that `source_table` exists before querying it. Verify the name with `influxdb3 query --database YOUR_DATABASE "SHOW TABLES"`, or supply an explicit `source_query`, which bypasses the check. + +#### Issue: Log summary shows `written=0` with a nonzero `skipped` count + +**Solution**: This is normal steady-state behavior: `skip_unchanged=true` skips events already written at their current update marker. New and updated events are still written. Set `skip_unchanged=false` to re-write everything the source returns. + +#### Issue: Field type conflict when writing with `write_quake_schema=true` + +**Solution**: The plugin writes all numeric quake columns as float64 (see [Schema requirements](#schema-requirements)). If your existing table stores `nst` or `magNst` as int64, rename the destination via `measurement` or recreate the table with float64 columns. + +#### Issue: Table mode does not re-read older rows + +**Solution**: With `skip_unchanged=true`, table reads page forward from the cached fetch watermark and do not revisit rows behind it. Set `skip_unchanged=false` to re-read the full `lookback_minutes` window, or provide an explicit `source_query`. + +### Debugging tips + +1. **Check trigger status**: + ```bash + influxdb3 show summary --database quakes --token YOUR_TOKEN + ``` +2. **Enable/Disable trigger**: + ```bash + influxdb3 disable trigger earthquake_sampler_trigger --database quakes --token YOUR_TOKEN + influxdb3 enable trigger earthquake_sampler_trigger --database quakes --token YOUR_TOKEN + ``` +3. **Enable full exception logging temporarily**: add `enable_full_logging=true` to the trigger arguments. + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/forecast-error-evaluator.md b/content/shared/influxdb3-plugins/plugins-library/official/forecast-error-evaluator.md index b9feffe358..f3fbca54b7 100644 --- a/content/shared/influxdb3-plugins/plugins-library/official/forecast-error-evaluator.md +++ b/content/shared/influxdb3-plugins/plugins-library/official/forecast-error-evaluator.md @@ -1,5 +1,8 @@ + + +The Forecast Error Evaluator Plugin validates forecast model accuracy for time series data in {{% product-name %}} by comparing predicted values with actual observations. On every scheduled run the plugin matches the two measurements over a time window, computes an error metric (MSE, MAE, RMSE, MAPE, or SMAPE) for each matched timestamp, and notifies for the points that reach a configured threshold. It includes debounce logic to suppress transient anomalies and supports multi-channel notifications via the Notification Sender Plugin. -The Forecast Error Evaluator Plugin validates forecast model accuracy for time series data in {{% product-name %}} by comparing predicted values with actual observations. The plugin periodically computes error metrics (MSE, MAE, RMSE, MAPE, or SMAPE), detects anomalies based on error thresholds, and sends notifications when forecast accuracy degrades. It includes debounce logic to suppress transient anomalies and supports multi-channel notifications via the Notification Sender Plugin. +The metric is computed per timestamp rather than aggregated over the window, which is what lets `min_condition_duration` measure how long an elevated error persists. As a consequence `rmse` yields the same value as `mae`: the root of a single squared difference is its absolute value. ## Configuration @@ -13,31 +16,40 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor ### Required parameters -| Parameter | Type | Default | Description | -|------------------------|--------|----------|------------------------------------------------------------------------------| -| `forecast_measurement` | string | required | Measurement containing forecasted values | -| `actual_measurement` | string | required | Measurement containing actual (ground truth) values | -| `forecast_field` | string | required | Field name for forecasted values | -| `actual_field` | string | required | Field name for actual values | -| `error_metric` | string | required | Error metric to compute: "mse", "mae", "rmse", "mape", or "smape" | -| `error_thresholds` | string | required | Threshold levels. Format: `INFO-"0.5":WARN-"0.9":ERROR-"1.2":CRITICAL-"1.5"` | -| `window` | string | required | Time window for data analysis. Format: `` (for example, "1h") | -| `senders` | string | required | Dot-separated list of notification channels (for example, "slack.discord") | +| Parameter | Type | Default | Description | +|------------------------|--------|----------|-------------------------------------------------------------------------------------------------------------------------------------| +| `forecast_measurement` | string | required | Measurement containing forecasted values | +| `actual_measurement` | string | required | Measurement containing actual (ground truth) values | +| `forecast_field` | string | required | Field name for forecasted values | +| `actual_field` | string | required | Field name for actual values | +| `error_metric` | string | required | Error metric to compute: `mse`, `mae`, `rmse`, `mape`, or `smape` | +| `error_thresholds` | string | required | Colon-separated `-` pairs, e.g. `INFO-"0.5":WARN-"0.9":ERROR-"1.2":CRITICAL-"1.5"`. See [Thresholds](#thresholds) | +| `window` | string | required | Time window for data analysis. Must be a positive duration. Units: `us`, `ms`, `s`, `min`, `h`, `d`, `w` | +| `senders` | string | required | Dot-separated list of notification channels (for example, "slack.discord") | + +### Thresholds + +Levels are `INFO`, `WARN`, `ERROR` and `CRITICAL`, and each threshold must be above `0`. Every supported metric is non-negative, so a threshold of `0` or below would flag every point of the window; such a level is skipped with a warning. + +Levels are evaluated independently: a point whose error reaches several thresholds produces one notification per level, each with its own debounce state. Configure only the levels you want to be paged about. Levels are processed from the highest threshold down, so `max_notifications_per_run` is spent on the most severe alerts first. + +Malformed segments, and a level given twice, are skipped with a warning and the remaining levels still apply. If no level survives parsing, the run logs an error and stops without notifying. ### Notification parameters -| Parameter | Type | Default | Description | -|---------------------|---------|------------------|-------------------------------------------------------------------------------------------------------------------| -| `notification_text` | string | default template | Template for notification message with variables `$measurement`, `$level`, `$field`, `$error`, `$metric`, `$tags` | -| `notification_path` | string | "notify" | URL path for the notification sending plugin | -| `port_override` | integer | 8181 | Port number where InfluxDB accepts requests | +| Parameter | Type | Default | Description | +|-----------------------------|---------|------------------|---------------------------------------------------------------------------------------------------------------------------------| +| `notification_text` | string | default template | Template for notification message with variables `$measurement`, `$level`, `$field`, `$error`, `$metric`, `$tags`, `$timestamp` | +| `notification_path` | string | "notify" | URL path for the notification sending plugin | +| `port_override` | integer | 8181 | Port number where InfluxDB accepts requests | +| `max_notifications_per_run` | integer | 20 | Maximum notifications sent by a single run. Alerts beyond the limit are counted in a warning and not resent later | ### Timing parameters -| Parameter | Type | Default | Description | -|--------------------------|--------|---------|----------------------------------------------------------------------------------| -| `min_condition_duration` | string | none | Minimum duration for anomaly condition to persist before triggering notification | -| `rounding_freq` | string | "1s" | Frequency to round timestamps for alignment | +| Parameter | Type | Default | Description | +|--------------------------|--------|-------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `min_condition_duration` | string | `0s` | Time an error must stay above a threshold before alerting. Units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`. With the default the first point above the threshold alerts | +| `rounding_freq` | string | no rounding | Fixed pandas frequency used to round timestamps before matching the two measurements, e.g. `1s`, `500ms`, `5min`, `1h` | ### Authentication parameters @@ -77,13 +89,32 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor | `twilio_from_number` | string | required | Twilio sender number (for example, "+1234567890") | | `twilio_to_number` | string | required | Recipient number (for example, "+0987654321") | +### Matching forecast to actual values + +Forecast and actual rows are matched with an inner join on the timestamp plus the tags that both measurements share. Tags present in only one of them are ignored with a warning, so a forecast table without the tag columns of the actual table still works. + +Set `rounding_freq` when the two series are written with slightly different timestamps. Rounding coarser than the sampling interval puts several points into the same slot; only the earliest point of each slot is kept, and the number of collapsed rows is logged. Without that, matching would join every forecast of the slot against every actual value in it. + +Rows where either value is missing are dropped before the metric is computed. For `mape`, rows with `actual = 0` are skipped; for `smape`, rows where both values are `0` are skipped. + +### Alert state + +Debounce and alert state live in the trigger-local cache, keyed by measurement, field, level and tag values: + +- While an error stays above a threshold for less than `min_condition_duration`, the plugin logs the pending state and waits. The duration is measured in data time, and a pending start that has scrolled out of the window is discarded, so a gap in the data cannot stand in for a persistent error. +- After an alert is delivered, its timestamp is recorded and earlier or equal timestamps are never alerted again. Overlapping windows on successive runs therefore do not resend the same point. +- If delivery fails after all retries, the state is left untouched so the next run alerts on that point again. +- The cache is in-memory and trigger-local: restarting the server clears the debounce and last-alert state. + ### TOML configuration | Parameter | Type | Default | Description | |--------------------|--------|---------|----------------------------------------------------------------------------------| | `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (required for TOML configuration) | -*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. Relative paths are resolved against the first directory that is set: `PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent of `VIRTUAL_ENV`. Only that directory is used — the file is not looked up in the remaining ones. + +When `config_file_path` is set, the TOML file provides the whole configuration and inline trigger arguments are ignored. `INFLUXDB3_AUTH_TOKEN` from the environment still applies when `influxdb3_auth_token` is not set in the file. In TOML, `senders` and `error_thresholds` can use native structures (a list and a table) instead of the inline string formats, though the inline strings are also accepted. #### Example TOML configuration @@ -96,8 +127,9 @@ For more information on using TOML configuration files, see the Using TOML Confi - **{{% product-name %}}**: with the Processing Engine enabled. - **Notification Sender Plugin for {{% product-name %}}**: Required for sending notifications. See the [influxdata/notifier plugin](/influxdb3/version/plugins/library/official/notifier/). - **Python packages**: - - `pandas` (for data processing) - - `requests` (for HTTP notifications) + - `influxdata-plugin-utils>=0.3.0` (configuration loading, parsing, and schema introspection) + - `pandas` (for data processing) + - `requests` (for HTTP notifications) ### Installation steps @@ -113,6 +145,7 @@ For more information on using TOML configuration files, see the Using TOML Confi 2. Install required Python packages: ```bash + influxdb3 install package influxdata-plugin-utils influxdb3 install package pandas influxdb3 install package requests ``` @@ -146,7 +179,7 @@ influxdb3 create trigger \ --database weather_db \ --path "gh:influxdata/forecast_error_evaluator/forecast_error_evaluator.py" \ --trigger-spec "every:15m" \ - --trigger-arguments 'forecast_measurement=temp_forecast,actual_measurement=temp_actual,forecast_field=predicted,actual_field=temperature,error_metric=rmse,error_thresholds=INFO-"0.5":WARN-"1.0":ERROR-"2.0":CRITICAL-"3.0",window=30m,senders=slack,slack_webhook_url="$SLACK_WEBHOOK_URL",min_condition_duration=10m' \ + --trigger-arguments 'forecast_measurement=temp_forecast,actual_measurement=temp_actual,forecast_field=predicted,actual_field=temperature,error_metric=rmse,error_thresholds=INFO-"0.5":WARN-"1.0":ERROR-"2.0":CRITICAL-"3.0",window=30min,senders=slack,slack_webhook_url="$SLACK_WEBHOOK_URL",min_condition_duration=10min' \ temp_forecast_check # Write forecast data @@ -166,16 +199,17 @@ influxdb3 query \ ``` **Expected output** -- Plugin computes RMSE between forecast and actual values -- If RMSE > 0.5, sends INFO-level notification -- If RMSE > 1.0, sends WARN-level notification -- Only triggers if condition persists for 10+ minutes (debounce) +- Plugin computes the error between forecast and actual values for every matched timestamp +- Points with an error of 0.5 or more send an INFO notification, 1.0 or more a WARN notification, and so on for each configured level +- A point is only alerted after its error has stayed above the level for 10 minutes (debounce) Set `SLACK_WEBHOOK_URL` to your Slack incoming webhook URL. **Notification example:** -[WARN] Forecast error alert in temp_forecast.predicted: rmse=1.2. Tags: location=station1 +[WARN] Forecast error alert in temp_actual.temperature: rmse=1.2. Tags: location=station1 + +The message names the actual measurement and field, because that is where the observed value comes from. ### Example 2: Multi-metric validation with multiple channels @@ -206,7 +240,7 @@ influxdb3 create trigger \ --database production_forecasts \ --path "gh:influxdata/forecast_error_evaluator/forecast_error_evaluator.py" \ --trigger-spec "every:5m" \ - --trigger-arguments 'forecast_measurement=demand_forecast,actual_measurement=demand_actual,forecast_field=predicted_demand,actual_field=actual_demand,error_metric=mse,error_thresholds=CRITICAL-"100000",window=15m,senders=sms,twilio_from_number="+1234567890",twilio_to_number="+0987654321",notification_text="CRITICAL: Production demand forecast error exceeded threshold. MSE: $$error",min_condition_duration=2m' \ + --trigger-arguments 'forecast_measurement=demand_forecast,actual_measurement=demand_actual,forecast_field=predicted_demand,actual_field=actual_demand,error_metric=mse,error_thresholds=CRITICAL-"100000",window=15min,senders=sms,twilio_from_number="+1234567890",twilio_to_number="+0987654321",notification_text="CRITICAL: Production demand forecast error exceeded threshold. MSE: $$error",min_condition_duration=2min' \ critical_forecast_alert ``` ## Using TOML Configuration Files @@ -237,7 +271,7 @@ error_thresholds = 'INFO-"0.5":WARN-"1.0":ERROR-"2.0":CRITICAL-"3.0"' window = "1h" senders = "slack" slack_webhook_url = "$SLACK_WEBHOOK_URL" -min_condition_duration = "10m" +min_condition_duration = "10min" rounding_freq = "1min" notification_text = "[$$level] Forecast validation alert: $$metric=$$error in $$measurement.$$field" @@ -262,6 +296,9 @@ influxdb3 create trigger \ - `forecast_error_evaluator.py`: The main plugin code containing scheduler handler for forecast validation - `forecast_error_config_scheduler.toml`: Example TOML configuration file +- `test_forecast_error_evaluator.py`: Pytest suite, runs without a live {{% product-name %}} server +- `requirements.txt`: Runtime dependencies (`influxdata-plugin-utils>=0.3.0`, `pandas`, `requests`) +- `requirements-dev.txt`: Development dependencies (`pytest`) ### Logging @@ -286,34 +323,32 @@ Handles scheduled forecast validation tasks. Queries forecast and actual measure Key operations: 1. Parses configuration from arguments or TOML file -2. Queries forecast and actual measurements within time window -3. Aligns timestamps using rounding frequency -4. Computes specified error metric (MSE, MAE, RMSE, MAPE, or SMAPE) -5. Evaluates thresholds and applies debounce logic -6. Sends notifications via configured channels - -#### `compute_error_metric(forecast_values, actual_values, metric_type)` +2. Verifies that both measurements exist and resolves the tags they share +3. Queries forecast and actual measurements within the time window +4. Rounds timestamps, collapses duplicate keys and matches the two series +5. Computes the error metric for every matched timestamp +6. Evaluates each threshold level, applies debounce logic and skips already-alerted points +7. Sends notifications via configured channels, up to `max_notifications_per_run` -Core error computation engine that calculates forecast accuracy metrics. +#### `compute_error(influxdb3_local, merged, error_metric, task_id)` -Supported error metrics: +Adds a per-timestamp `error` column to the matched frame. -- `mse`: Mean Squared Error - measures average squared differences -- `mae`: Mean Absolute Error - measures average absolute differences -- `rmse`: Root Mean Squared Error - square root of MSE, same units as original data -- `mape`: Mean Absolute Percentage Error - percentage-based error -- `smape`: Symmetric Mean Absolute Percentage Error - bounded 0-200%, handles over/under-estimation symmetrically +| Metric | Formula per timestamp | Notes | +|---------|-------------------------------------------------------------|------------------------------------------------------------| +| `mse` | `(forecast - actual)²` | Thresholds are in squared units | +| `mae` | `\|forecast - actual\|` | | +| `rmse` | `((forecast - actual)²)^0.5` | Equals `mae` for a single point | +| `mape` | `\|forecast - actual\| / \|actual\| * 100` | Rows with `actual = 0` are skipped | +| `smape` | `200 * \|forecast - actual\| / (\|forecast\| + \|actual\|)` | Bounded 0-200%; rows where both values are `0` are skipped | -#### `evaluate_thresholds(error_value, threshold_config)` +#### `align_frames(influxdb3_local, df_forecast, df_actual, tags, rounding_freq, task_id)` -Evaluates computed error against configured thresholds to determine alert level. +Rounds timestamps, keeps the earliest row per key and inner-joins the two frames on the timestamp and the shared tags. -Returns alert level based on threshold ranges: +#### `parse_error_thresholds(influxdb3_local, config, task_id)` -- `INFO`: Informational threshold exceeded -- `WARN`: Warning threshold exceeded -- `ERROR`: Error threshold exceeded -- `CRITICAL`: Critical threshold exceeded +Parses the inline `-` string or the TOML table into a `{level: threshold}` mapping, skipping unknown levels, non-numeric values and thresholds at or below zero. ## Troubleshooting @@ -349,6 +384,20 @@ curl -X POST "your_webhook_url" -d '{"text": "test message"}' # For percentage metrics (MAPE, SMAPE) --trigger-arguments 'error_thresholds=INFO-"5.0":WARN-"10.0":ERROR-"20.0":CRITICAL-"30.0"' ``` +A `Skipping threshold` warning names the level that was dropped and why. `No valid error thresholds configured` means every level was rejected, so nothing was evaluated. + +#### Issue: Trigger fails with "Failed to load configuration" + +**Solution**: The message names the offending parameter. Common causes are a duration without a supported unit (use `min`, not `m`), a `window` of `0s`, a `port_override` outside 1-65535 and an `error_metric` outside `mse`, `mae`, `rmse`, `mape`, `smape`. + +#### Issue: Many rows collapse into one timestamp + +**Solution**: A `Collapsed N forecast and M actual rows sharing a rounded timestamp` line means `rounding_freq` is coarser than the sampling interval, so only the earliest point of each slot is compared. Lower `rounding_freq` to match how far apart the two series are actually written. + +#### Issue: Notifications stop mid-run + +**Solution**: `Suppressed N notifications after reaching max_notifications_per_run` means the per-run cap was hit. Raise `max_notifications_per_run`, raise the thresholds, or set `min_condition_duration` so short spikes are not alerted. + #### Issue: MAPE/SMAPE calculation errors with zero values **Solution**: MAPE cannot be calculated when actual values are zero, and SMAPE cannot be calculated when both forecast and actual are zero. The plugin automatically skips such rows and logs warnings. For datasets with frequent zero values, consider using MAE or RMSE instead. @@ -378,7 +427,7 @@ influxdb3 serve --plugin-dir ~/.plugins 3. **Test with shorter windows** for faster debugging: ```bash - --trigger-arguments 'window=10m,min_condition_duration=1m' + --trigger-arguments 'window=10min,min_condition_duration=1min' ``` 4. **Monitor notification delivery** in logs: @@ -389,9 +438,10 @@ influxdb3 serve --plugin-dir ~/.plugins ### Performance considerations - **Data alignment**: Use appropriate `rounding_freq` to balance accuracy and performance -- **Window size**: Larger windows increase computation time but provide more robust error estimates +- **Window size**: Larger windows evaluate more points per run, and every point is checked against every configured level - **Debounce duration**: Balance between noise suppression and alert responsiveness -- **Notification throttling**: Built-in retry logic prevents notification spam +- **Notification throttling**: Deliveries are sequential with up to three retries each, so `max_notifications_per_run` bounds how long a run can take +- **Trigger interval**: An interval shorter than `window` re-reads the overlap on every run; the last-alert state keeps it from resending, but the data is queried again - **Memory usage**: Plugin processes data in pandas DataFrames - consider memory for large datasets ## Report an issue @@ -401,4 +451,6 @@ For plugin issues, see the Plugins repository [issues page](https://github.com/i ## Find support for {{% product-name %}} The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. -For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. \ No newline at end of file +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/gapfill.md b/content/shared/influxdb3-plugins/plugins-library/official/gapfill.md new file mode 100644 index 0000000000..5674be222c --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/gapfill.md @@ -0,0 +1,375 @@ + + +> **Note:** This plugin requires {{% product-name %}}.8.2 or later (uses the synchronous write API). + + +The Gapfill Plugin detects gaps in time series (spacing between consecutive +points wider than a threshold) and fills them with imputed values using +standard missing-data imputation methods. Fill points land on an epoch-anchored +time grid strictly inside each gap, so overlapping runs are idempotent. It +works both on uniform-grid data (for example, the output of the +[`resampler`](https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/resampler) +plugin) and directly on irregular raw series, and supports two modes of +operation: continuous scheduled filling and one-off historical repair over +HTTP. + +- **Detects and fills in one pass**: a gap is any inter-point spacing greater + than `gap_threshold`; detection runs per numeric field, so a field that goes + missing while others keep reporting is still filled — non-numeric fields are + carried onto fill points by last known value +- **Standard imputation methods**: linear, previous (LOCF), next, nearest, + cubic spline, PCHIP (monotone cubic, no overshoot at gap edges), constant +- **Two output modes**: materialize the full series into a target measurement + (pipeline mode), or append only fill points into the source (in-place repair) +- **Bounded imputation**: gaps longer than `max_fill_gap` are never invented, + only reported + +Unlike SQL +[`date_bin_gapfill`](https://docs.influxdata.com/influxdb3/enterprise/reference/sql/functions/time-and-date/#date_bin_gapfill), +which fills gaps at query time, this plugin **materializes** a continuous series +that downstream consumers — dashboards, other plugins such as +[`signal_filter`](https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/signal_filter) +— read as physical points, with more fill methods, optional marking of imputed +points, and a per-gap quality report. + +## Configuration + +Plugin parameters may be specified as key-value pairs in the +`--trigger-arguments` flag (`influxdb3 create trigger`) or in the +`trigger_arguments` field of the API. Values are strings; the plugin coerces +them. Alternatively, supply every parameter from a TOML file via +`config_file_path` — see [TOML configuration](#toml-configuration). For the +HTTP trigger, parameters are passed as a JSON request body. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines +supported trigger types and configuration parameters, enabling the +[InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to +display and configure the plugin. + +### Shared parameters (scheduled args and HTTP body) + +| Parameter | Type | Default | Description | +|----------------------|---------|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `source_measurement` | string | required | Source measurement to scan for gaps. | +| `target_measurement` | string | *(in-place)* | Target measurement: the full series (existing points plus fills) is written there. Omit to append only the fill points into the source measurement. Must differ from the source. | +| `interval` | string | `1s` | Expected series cadence and fill grid step, at least 1ms. Units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`. | +| `method` | string | `linear` | Fill method: `linear`, `previous` (LOCF), `next`, `nearest`, `cubic`, `pchip`, or `constant`. | +| `fill_value` | float | — | Constant used when `method=constant`; required for it, invalid otherwise. | +| `gap_threshold` | string | 1.5× interval | Spacing between consecutive points above which a gap is detected, at least one interval. | +| `max_fill_gap` | string | *(unlimited)* | Gaps longer than this are not filled, only reported. At least one `gap_threshold`. | +| `fields` | string | all numeric | Space-separated numeric fields to fill with the configured method; numeric fields not listed are dropped. Non-numeric fields are carried into fill points by last known value unless excluded. | +| `excluded_fields` | string | none | Space-separated fields of any type to exclude from the output. | +| `mark_filled` | boolean | `false` | When `true`, points that received filled values carry a boolean marker field (`filled=true` by default); such a point may also hold real values of other fields. | +| `filled_field_name` | string | `filled` | Name of the marker field written when `mark_filled` is true; must not collide with an existing source column. | +| `report_measurement` | string | *(off)* | Optional measurement receiving one row per detected gap; written to `target_database` when set. | +| `target_database` | string | *(trigger db)* | Database for writing output. Requires `target_measurement` (chain mode). | +| `max_retries` | integer | `5` | Maximum number of write attempts. | +| `config_file_path` | string | — | Path to a TOML file supplying all parameters — replaces the trigger arguments or HTTP body entirely. Relative paths resolve against `PLUGIN_DIR`. | + +### Scheduled-only parameters + +| Parameter | Type | Default | Description | +|------------|--------|-----------|-------------------------------------------------------------| +| `window` | string | `10min` | How much history each run processes, at least one interval. | +| `offset` | string | `0s` | Processing delay for late-arriving data. | + +### HTTP-only parameters + +| Parameter | Type | Default | Description | +|------------------|--------|-----------------|-----------------------------------------------------------------------------------------------------| +| `backfill_start` | string | *(oldest data)* | ISO 8601 start of the repair range (for example, `2026-07-18T14:00:00Z`). Naive values are treated as UTC. | +| `backfill_end` | string | *(now)* | ISO 8601 end of the repair range. | +| `batch_size` | string | `30d` | Time span processed per batch, at least one `interval`. | + +### TOML configuration + +Set the `PLUGIN_DIR` environment variable and reference the file with the +`config_file_path` trigger argument (relative paths resolve against +`PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent of `VIRTUAL_ENV`). +The TOML file then supplies **all** parameters — it is mutually exclusive with +inline trigger arguments. See +[`gapfill_config_scheduler.toml`](gapfill_config_scheduler.toml) for an +annotated template. + +## Output modes + +- **Chain mode** (`target_measurement` set): the full series — existing points + plus fills — is written to the target measurement each run, so downstream + consumers read one continuous measurement. Raw data is never modified. +- **In-place repair** (`target_measurement` omitted): fill values are appended + into the source measurement. Filling is per field, so a fill may land on a + timestamp where other fields already hold real values — the missing field's + value (and the marker, when enabled) is added to that point; existing values + are never modified. + +Every value is written with the source column's stored type: integer fills are +rounded and UInt64 stays `uint`, so output never conflicts with the schema. +Blending methods (`linear`, `cubic`, `pchip`, `constant`) produce float64, so +in chain mode those columns — fills and copied values alike — become float in +the target; in in-place mode such fills are cast back to the column's type. + +## Gap report + +With `report_measurement` set, each detected gap produces one row: + +- **Tags**: the series tags plus `gap_field` (the field the gap was detected + in). +- **Fields**: `gap_start_ns`, `gap_end_ns` (boundary timestamps), `duration_s` + (float seconds), `points` (fill points actually written for the gap; `0` + when skipped), `status` (`filled`, or `skipped` when the gap is longer than + `max_fill_gap` or no point could be written). +- **Timestamp**: the gap end, so re-reporting the same gap overwrites in place. + +Report rows are written next to the output: with `target_database` set they +land in that database, not in the trigger's. + +Alerting on this measurement (for example, with the +[`notifier`](https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/notifier) +plugin) turns gapfill into a sensor-health monitor. + +## Data requirements + +- Existing timestamps are never changed: fills are added on the epoch-anchored + grid, existing points keep their original time. The output is continuous but + not uniform — run + [`resampler`](https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/resampler) + first when downstream consumers need an exact grid. +- On jittery raw data, raise `gap_threshold` above the expected jitter (the + default is 1.5× `interval`), otherwise normal spacing is detected as gaps and + imputed points appear between healthy readings. +- Detecting a gap requires both its boundary points inside one run's query + window (quantified below), so a longer outage is neither filled nor reported. + Repair those with the HTTP trigger over an explicit range, and alert on them + with a deadman check (e.g. + [`threshold_deadman_checks`](https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/threshold_deadman_checks)). +- A gap is filled only after it closes (a point arrives on its right side); an + ongoing outage is not extrapolated. +- A series/field needs at least 2 points to be processed; `cubic` needs at + least 4 (falls back to `linear` with a warning). +- Visibility costs reads: each run queries three times the range it processes + (the range plus lookback padding on both sides). A scheduled run processes + `window + max(window, max_fill_gap)`, so `window=10min` without + `max_fill_gap` reads about 60 minutes of history per run. HTTP backfill + batches are padded the same way. + +## Software Requirements + +- **{{% product-name %}}**: with the Processing Engine enabled + (`--plugin-dir` configured). +- **Python packages**: `scipy` and `influxdata-plugin-utils` (numpy is + installed with scipy). + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled + (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install the Python dependencies into the plugin environment: + + ```bash + influxdb3 install package "influxdata-plugin-utils>=0.3.0" + influxdb3 install package scipy + ``` +## Trigger setup + +### Scheduled trigger (continuous pipeline) + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/gapfill/gapfill.py \ + --trigger-spec "every:1m" \ + --trigger-arguments 'source_measurement=signal_resampled,target_measurement=signal_filled,interval=1s,method=linear,max_fill_gap=1min' \ + gapfill_signal +influxdb3 enable trigger --database mydb gapfill_signal +``` +### HTTP trigger (one-off historical repair) + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/gapfill/gapfill.py \ + --trigger-spec "request:gapfill" \ + gapfill_backfill +influxdb3 enable trigger --database mydb gapfill_backfill + +curl -X POST "http://localhost:8181/api/v3/engine/gapfill" \ + -H "Authorization: Bearer $INFLUXDB3_AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "source_measurement": "sensors", + "interval": "15s", + "method": "pchip", + "backfill_start": "2026-07-18T00:00:00Z", + "backfill_end": "2026-07-19T00:00:00Z" + }' +``` +**Expected response** (fills appended in-place, since `target_measurement` is +omitted): + +```json +{ + "status": "ok", + "task_id": "…", + "batches": 1, + "rows_scanned": 5730, + "gaps_filled": 12, + "gaps_skipped": 0, + "fills_written": 96, + "rows_copied": 0 +} +``` +## Example usage + +### Example 1: Fill outage gaps after resampling (pipeline mode) + +With `signal_resampled` on a 1s grid containing 10–30s outage holes: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/gapfill/gapfill.py \ + --trigger-spec "every:1m" \ + --trigger-arguments 'source_measurement=signal_resampled,target_measurement=signal_filled,interval=1s,max_fill_gap=1min,mark_filled=true' \ + gapfill_signal +influxdb3 enable trigger --database mydb gapfill_signal + +influxdb3 query --database mydb \ + "SELECT time, value, filled FROM signal_filled ORDER BY time DESC LIMIT 10" +``` +**Expected output**: a continuous 1s series in `signal_filled`; timestamps that +were missing in `signal_resampled` carry interpolated `value`s and +`filled=true`, copied real points have `filled` null. + +### Example 2: Constant fill for an event counter + +A gap in an event-rate series means "no events", not "interpolate": + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/gapfill/gapfill.py \ + --trigger-spec "every:5m" \ + --trigger-arguments 'source_measurement=events_per_min,interval=1min,method=constant,fill_value=0' \ + gapfill_events +influxdb3 enable trigger --database mydb gapfill_events +``` +**Expected output**: zero-valued points appended in-place at missing minutes. + +### Example 3: Gap report for sensor-health monitoring + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/gapfill/gapfill.py \ + --trigger-spec "every:1m" \ + --trigger-arguments 'source_measurement=sensors,interval=15s,max_fill_gap=2min,report_measurement=gapfill_report' \ + gapfill_sensors + +influxdb3 query --database mydb \ + "SELECT * FROM gapfill_report WHERE status = 'skipped' ORDER BY time DESC" +``` +**Expected output**: one row per gap with boundaries, duration, and status; +`skipped` rows mark outages too long to impute — alert on those. + +## Code overview + +### Files + +- `gapfill.py`: The main plugin code containing the handlers for scheduled + filling and HTTP backfill +- `gapfill_config_scheduler.toml`: Example TOML configuration file + +### Logging + +Logs are stored in the `_internal` database in the +`system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal \ + "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'gapfill_signal' ORDER BY event_time DESC LIMIT 20" +``` +### Main functions + +#### `process_scheduled_call(influxdb3_local, call_time, args)` + +Entry point for the scheduled trigger. Loads and validates the configuration, +then fills a sliding window of history ending at `call_time - offset`. + +#### `process_request(influxdb3_local, query_parameters, request_headers, request_body, args)` + +Entry point for the HTTP trigger. Repairs `[backfill_start, backfill_end)` in +`batch_size` chunks, resolving the schema once, and returns the run totals. + +#### `run_gapfill(...)` + +One pass over a time range: resolves the schema, queries the padded window, +groups rows by tag set, fills each series (and copies existing points in chain +mode), and writes the result with retries. + +#### `fill_series(...)` + +Detects gaps of one series per numeric field, computes fill values on the +epoch grid, attaches carried fields by last known value, and builds the gap +report rows. + +## Troubleshooting + +### Common issues + +#### Issue: "required packages are not installed" + +**Cause**: The plugin environment lacks `scipy` or `influxdata-plugin-utils`. + +**Solution**: Run `influxdb3 install package "influxdata-plugin-utils>=0.3.0"` +and `influxdb3 install package scipy`, then re-enable the trigger. + +#### Issue: Field type conflict on write + +**Cause**: The chain-mode target measurement already contains a field whose +stored type conflicts with the plugin's output (blending methods write +float64; another producer may have created the column as integer). + +**Solution**: Use a fresh target measurement, or a selection method +(`previous`, `next`, `nearest`), which preserves source types. In-place mode +preserves integer column types by rounding fills. + +#### Issue: No fills appear + +**Cause**: The gap has not closed yet (no point on its right side), the gap is +longer than `max_fill_gap` or the lookback, or the series/field has fewer than +2 points. + +**Solution**: Check the info logs for skip counts; increase `max_fill_gap` or +repair the range explicitly via the HTTP trigger. + +#### Issue: "source_measurement already has a column named 'filled'" + +**Cause**: `mark_filled` is true and the marker name (`filled_field_name`, +default `filled`) is taken by a source tag or by a non-boolean field; the +plugin refuses to overwrite it. In in-place mode, a boolean column of that +name is treated as the plugin's own marker from earlier runs and reused. + +**Solution**: Set `filled_field_name` to a name not used by the source, or +disable `mark_filled`. With `mark_filled` off, a source field named `filled` +is processed normally. + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/geo-enrichment.md b/content/shared/influxdb3-plugins/plugins-library/official/geo-enrichment.md new file mode 100644 index 0000000000..743c9bf1e8 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/geo-enrichment.md @@ -0,0 +1,789 @@ + + +> **Note:** This plugin requires {{% product-name %}}.8.2 or later (uses the synchronous write API). + + +The Geo Enrichment Plugin turns coordinates carried by incoming points into +location attributes you can group by — a country and city, a zone or site you +defined yourself, or a cell of a global grid. By default they merge back into +the **same rows the coordinates came from**, so no join is needed to use them; +naming a target table or database collects them separately instead. + +- **Four resolution strategies**: offline place lookup, point-in-polygon against + your own zones, nearest site from your own list, and grid cells (H3, geohash + or S2). Zones and sites come from GeoJSON or CSV, whichever you have +- **Reads coordinates in many shapes**: separate `lat`/`lon` columns as numbers + or strings, integer-scaled tracker output, a single combined column + (`"55.75,37.61"`, WKT, GeoJSON), or an existing geohash/H3 index +- **In-place or into a target table**: attributes written as fields merge into + the existing row; written as tags they need a separate table +- **HTTP backfill**: apply enrichment to history, retry only the rows that + failed, or re-resolve everything after correcting a reference file +- **Caches what it resolves**: a coordinate is resolved once and reused for + every later point that rounds to the same key + +## How enrichment lands on the row + +A row's identity in {{% product-name %}} is its **full tag set plus its timestamp**. That +single fact decides where the plugin can write. + +Attributes written as **fields** leave the tag set untouched, so the write has +the same identity as the original point and the columns merge into it: + +``` +gps,device=A lat=55.7558,lon=37.6173,speed=60 T ← your client +gps,device=A geo_country="RU",geo_city="Moscow" T ← the plugin + +SELECT * FROM gps +→ device=A lat=55.7558 lon=37.6173 speed=60 geo_country=RU geo_city=Moscow +``` +Attributes written as **tags** change the identity, producing a *second* row, so +`output_mode=tag` requires `target_measurement` and the plugin rejects a +configuration that would write tags back into the source table. + +| `output_mode` | destination | result | +|---------------|---------------|--------------------------------------------| +| `field` | source table | merges into the existing row | +| `field` | other table | full row copied, geo added as fields | +| `tag` | other table | full row copied, geo added as tags | +| `tag` | source table | rejected — would duplicate every row | + +Merging holds regardless of write order, survives compaction, and is not undone +if your client re-sends the raw point later: each column keeps its last non-null +value. + +### The echo batch + +In-place mode writes into the very table the trigger watches, so the plugin's +own write comes back as a second invocation. Rows that already carry every +configured output column are skipped, which ends the loop after that one extra +pass. Expect roughly **twice the trigger invocations and twice the write volume** +on an enriched table; where ingest headroom is tight, use `target_measurement` +instead. + +## Configuration + +### Plugin metadata + +The plugin includes a JSON metadata schema in its docstring for +[InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) +integration, defining `onwrite_args_config` and `http_body_config`. + +### Core parameters + +| Parameter | Type | Default | Description | +|-----------------------|--------|--------------|------------------------------------------------------------------------------------------| +| `source_measurements` | string | *required* | Space-separated tables to enrich. Other tables in the batch are ignored. | +| `output_columns` | string | *required* | Space-separated `attribute:column` pairs, e.g. `country_code:geo_country city:geo_city`. | +| `output_mode` | string | `field` | `field` or `tag`. `tag` requires `target_measurement`. | +| `target_measurement` | string | *(empty)* | Destination table. Empty enriches the source table in place. | +| `target_database` | string | *(empty)* | Destination database. Defaults to the trigger's database. | +| `unknown_value` | string | `UNKNOWN` | Written when a coordinate cannot be resolved. | + +In [TOML](#toml-configuration), `source_measurements` also takes a native array +and `output_columns` a native table. + +Every configured column is always written; unresolved attributes get +`unknown_value`. + +#### Column types + +Attributes are written as **strings**, including numbers such as `population` +and any numeric GeoJSON property. Booleans are rendered as JSON, `true` / +`false`. The one exception is `distance_m`, at every rank a float, which reports +`-1` when nothing resolved. + +A numeric attribute therefore cannot be compared or aggregated as a number: +`geo_pop > 1000000` compares text, and `'9' > '10381222'`. Cast it in the query, +excluding the unresolved rows: + +```sql +SELECT * FROM gps WHERE CAST(geo_pop AS BIGINT) > 1000000 + AND geo_pop != 'UNKNOWN' +``` +### Coordinate input + +Configure exactly one input mode. Columns may be tags or fields. + +| Parameter | Type | Default | Description | +|-----------------|--------|-----------|----------------------------------------------------------------------| +| `lat_field` | string | `lat` | Latitude column, number or string. | +| `lon_field` | string | `lon` | Longitude column, number or string. | +| `coord_scale` | number | `1` | Positive divisor turning scaled integers back into degrees. | +| `point_field` | string | *(empty)* | Single column holding both coordinates, instead of the pair above. | +| `point_format` | string | `lat_lon` | `lat_lon`, `lon_lat`, `wkt` (`POINT(lon lat)`) or `geojson`. | +| `geohash_field` | string | *(empty)* | Geohash column, decoded to the cell center. | +| `h3_field` | string | *(empty)* | H3 index column, decoded to the cell center. | + +Rows without usable coordinates are skipped. Coordinates outside +[-90, 90] / [-180, 180] are counted and written as `unknown_value`. + +#### Scaled integer coordinates + +Many trackers report degrees multiplied by a fixed power of ten to avoid +sending decimals, so `55.7558` arrives as `557558000`. `coord_scale` is the +number both coordinates are divided by: + +| `lat` | `lon` | `coord_scale` | Resolved as | +|-------------|-------------|----------------|------------------| +| `557558000` | `376184000` | `1e7` | 55.7558, 37.6184 | +| `55755800` | `37618400` | `1e6` | 55.7558, 37.6184 | +| `55.7558` | `37.6184` | `1` (default) | 55.7558, 37.6184 | + +Pick the divisor that turns your raw number back into degrees: count the digits +the device shifted. Write it plainly (`10000000`) or in scientific notation +(`1e7`) — both are read as the same number. Any positive value is accepted, +including fractions; zero and negatives are rejected when the trigger loads. + +The division happens after the coordinates are read, so it applies to every +input mode, including `point_field`, `geohash_field` and `h3_field`. Those +already decode to degrees, so scaling them is almost always a mistake — leave +`coord_scale` at `1` unless the column truly holds scaled integers. + +### Caching + +| Parameter | Type | Default | Description | +|---------------------|------|----------|-----------------------------------------------------------------------------| +| `quantize_decimals` | int | `4` | Decimal places a coordinate is rounded to before it becomes a key. `0`–`9`. | +| `cache_size` | int | `100000` | Distinct rounded coordinates kept before LRU eviction. `1` or more. | + +Resolving a coordinate costs far more than a dictionary lookup, and most +coordinates repeat: a fixed sensor reports the same position thousands of times +a day. Raw floats never repeat exactly — GPS noise moves the last digits — so +the coordinate is rounded first and everything that rounds alike shares one +result. + +| `quantize_decimals` | grid | effect | +|---------------------|----------|-----------------------------------------------------------| +| 3 | ≈ 111 m | very high hit rate, unusable near zone boundaries | +| **4** | ≈ 11 m | default — GPS-noise sized, a stationary asset always hits | +| 5 | ≈ 1.1 m | near-exact, few hits from a moving asset | +| 6 | ≈ 0.11 m | effectively no rounding | + +The cost is a boundary error: a point within roughly that distance of a polygon +edge may take a neighbor's answer. At the default the window is smaller than +consumer GPS error. Raise it where exact edge behavior matters. + +### HTTP body parameters + +The endpoint takes its **entire configuration from the request body** and reads +no trigger arguments, so the same request behaves identically whichever trigger +serves it. Every parameter above may be given in the body under the same name, +plus the backfill-only fields here. + +`source_measurements` keeps its name but backfills one table per call: give +several and the first is used, the rest are ignored with a warning. A field set +to `null` counts as absent. If the trigger was created with arguments, they are +ignored and a warning is logged. + +| Parameter | Type | Default | Description | +|-----------------|--------|-----------|-----------------------------------------------------------------------------------------------------| +| `start` / `end` | string | *(empty)* | RFC 3339 bounds, given together. `start` inclusive, `end` exclusive. Omit both for the whole table. | +| `batch_size` | int | `1000` | Rows read per page. Values below `1` are raised to `1`. | +| `retry_unknown` | bool | `false` | Re-resolve rows whose geo column equals `unknown_value`. | +| `force` | bool | `false` | Re-resolve every row regardless of its current values. | + +`start` and `end` keep nanosecond precision. `retry_unknown` and `force` take a +JSON boolean or any of `true`/`false`, `yes`/`no`, `on`/`off`, `1`/`0` as a +string; anything else is a 400. All five may also be set in a +[TOML file](#toml-configuration), where they act as defaults the body overrides. + +Use `retry_unknown` after widening `max_radius_m`, and `force` after redrawing a +zone — those rows already hold a resolved value, so `retry_unknown` would pass +over them. The reference file is re-read on every HTTP call. + +### TOML configuration + +| Parameter | Type | Default | Description | +|--------------------|--------|-----------|-----------------------------------------------------| +| `config_file_path` | string | *(empty)* | `.toml` file, relative to `PLUGIN_DIR` or absolute. | + +On a write trigger its values override the trigger arguments. + +In an HTTP request body it goes further: the configuration is then read from +**that file alone**, and every body field naming a plugin parameter is ignored, +so a long setup is named once instead of repeated in every backfill request. + +The five backfill fields are the exception. They may be set in the file too, but +the body always wins, so the file holds the defaults and each call overrides only +what it needs — usually the window: + +```json +{ + "config_file_path": "geo_enrichment_config_data_writes.toml", + "start": "2026-08-01T00:00:00Z", + "end": "2026-08-29T00:00:00Z", + "force": true +} +``` +```bash +--trigger-arguments 'config_file_path=geo_enrichment_config_data_writes.toml' +``` +## Resolution strategies + +| Parameter | Type | Default | Description | +|------------|--------|-----------|-------------------------------------------------| +| `strategy` | string | `builtin` | `builtin`, `polygon`, `nearest` or `grid`. | + +One strategy is active per trigger. Each answers a different question and offers +its own attributes for `output_columns`; asking for an attribute the active +strategy cannot produce is a configuration error, reported at load time with the +list of what is available. + +| Strategy | Answers | Reference data | Geometry it needs | +|-----------|-------------------------------|----------------------|------------------------| +| `builtin` | what settlement is this near? | bundled, none to set | — | +| `polygon` | which of my zones is this in? | `reference_file` | Polygon, MultiPolygon | +| `nearest` | which of my sites is this at? | `reference_file` | Point | +| `grid` | which grid cell is this in? | none | — | + +Each strategy's own parameters are listed with it below. Parameters belonging to +an inactive strategy are ignored, and no package is imported for a strategy or a +file format you do not use. + +### Reference file + +`polygon` and `nearest` read their zones or sites from one file. The **format** +decides how it is read, the **strategy** decides how a point is matched against +it, and the two are independent: either strategy takes either format. + +| Parameter | Type | Default | Description | +|-----------------------------|--------|--------------|----------------------------------------------------------------------| +| `reference_file` | string | *required* | `.geojson`, `.json` or `.csv`, relative to `PLUGIN_DIR` or absolute. | +| `reference_encoding` | string | `utf-8-sig` | Python codec name for a CSV file. GeoJSON is always UTF-8. | +| `reference_lat_column` | string | *(detected)* | Latitude column of a CSV file. | +| `reference_lon_column` | string | *(detected)* | Longitude column of a CSV file. | +| `reference_geometry_column` | string | *(detected)* | WKT column of a CSV file. | + +#### Where the geometry comes from + +**GeoJSON** — a `FeatureCollection`, or a single `Feature`. Each feature's +`geometry` is the shape and its `properties` are the attributes. + +**CSV** — either a column of WKT, or a pair of coordinate columns: + +```csv +zone,geometry +plant-A,"POLYGON((37.5 55.7, 37.8 55.7, 37.8 55.9, 37.5 55.9, 37.5 55.7))" +``` +```csv +code,lat,lon,region +KONA-01,19.64,-155.99,HI-WEST +``` +Detection runs in this order, and an explicit setting always wins: + +1. `reference_geometry_column`, read as WKT; +2. `reference_lat_column` / `reference_lon_column`; +3. a column named `geometry` or `wkt`, read as WKT; +4. a latitude column (`lat`, `latitude`) and a longitude column (`lon`, `lng`, + `long`, `longitude`). + +Names are matched ignoring case. Setting a geometry column *and* a coordinate +column is a configuration error — they are alternatives. When nothing matches, +the error lists the columns actually parsed, along with the delimiter used to +parse them. + +Every column that did not become geometry is an attribute. + +#### Reading a CSV + +The delimiter is detected from the header — comma, semicolon, tab and pipe are +recognized, and a comma inside a quoted value does not confuse it. When the +delimiter is *not* a comma, a comma inside a number is unambiguous and is read +as a decimal mark, so European exports work as they are: + +```csv +code;lat;lon +center;"55,7512";"37,6184" +``` +The default encoding accepts UTF-8 with or without a byte-order mark, which +covers files saved by Excel. Set `reference_encoding` for anything else, such as +`cp1251`. GeoJSON is always UTF-8 by RFC 7946, so the setting does not apply to +it. + +#### How attributes are named + +| Format | An attribute in `output_columns` is | +|---------|---------------------------------------| +| GeoJSON | a **JSONPath** into `properties` | +| CSV | a **column name** | + +GeoJSON properties are arbitrary JSON, so a path reaches nested values; +a CSV row is flat, so the name is the column. + +``` +GeoJSON: output_columns='zone:geo_zone owner.name:geo_owner owner.contact.email:geo_email' +CSV: output_columns='zone:geo_zone owner:geo_owner' +``` +In a path, `codes[0]` indexes an array, and a property whose own name contains a +dot is reached by quoting it: `"odd.name"`. + +A path missing from *some* features is normal — those get `unknown_value`: + +``` +zones: {"zone": "plant-A", "owner": {"name": "ACME"}} + {"zone": "plant-B"} + +output_columns='zone:geo_zone owner.name:geo_owner' +→ point in A: geo_zone=plant-A geo_owner=ACME +→ point in B: geo_zone=plant-B geo_owner=UNKNOWN +``` +A path missing from *every* feature is a configuration error instead, because it +is almost always a typo, and writing `UNKNOWN` forever would hide it behind a +column that merely looks empty: + +``` +output_columns='zone:geo_zone owner.phone:geo_phone' +→ Configuration error: output_columns attribute 'owner.phone' matches no feature. +``` +Paths are resolved once when the file is indexed, so they cost nothing per row +and every mistake surfaces there rather than in your data: + +``` +'owner' → resolves to a dict; only single values can be written. + Point the path at a leaf, e.g. 'owner.'. +'codes[*]' → matches 2 values in one feature; a column holds a single value. +'owner[' → is not a valid JSONPath: Parse error near the end of string! +``` +#### Geometry the strategy cannot use + +`polygon` needs areal geometry, `nearest` needs points. An entry of the wrong +kind is skipped with a warning, so a stray label point among fifty zones does +not stop the trigger; a file with **nothing** usable fails at startup rather +than writing `UNKNOWN` into every row. + +A point is never reduced to a polygon's centroid: the centroid of a region is +not where anything is. + +Coordinates are read as WGS84 degrees, longitude first in WKT and GeoJSON. Data +in a projected system is rejected as out of range rather than misread. + +### `builtin` — nearest populated place, offline + +Answers "what settlement is this point in?" from the GeoNames snapshot bundled +with `reverse_geocode`. No reference file to prepare — this is the zero-setup +path. Attributes: `country_code`, `country`, `state`, `city`, `population`, +`distance_m`. + +| Parameter | Type | Default | Description | +|------------------|--------|------------|-------------------------------------------------------------------------| +| `min_population` | int | `0` | Consider only places at least this populous. `0` or more. | +| `max_radius_m` | number | *no limit* | Meters. Points farther than this from the place are unknown. Above `0`. | + +`min_population` is a zoom control — the dataset goes down to hamlets, so a +point on a city's edge resolves to a suburb: + +``` +(55.5800, 37.5000) outskirts of Moscow + min_population=0 → Kommunarka 4,684 + min_population=10000 → Yasenevo 180,000 + min_population=1000000 → Moscow 10,381,222 +``` +The match is the nearest **city center**, not a boundary the point falls inside, +so distances of a few kilometers are normal and mean nothing is wrong. There is +no radius by default — a 1 km limit would discard roughly 8 of every 10 points +in a city and virtually everything outside one. + +Set `max_radius_m` when a plausible-looking wrong answer is worse than a blank: +a point in Antarctica resolves to South Africa, 5,000 km away, and a mid-ocean +point to French Polynesia. Map `distance_m` to a column first and query it — +that shows the real spread of your own data before you pick a number. + +> Raising `min_population` moves the answer farther away, so pair the two: +> `(41.6, -93.9)` in rural Iowa resolves to Waukee at `min_population=0`, but to +> **Chicago, some 500 km away**, at `min_population=1000000`. + +### `polygon` — point inside a zone you drew + +Answers "which of my zones is this point in?" Attributes: whatever the +[reference file](#reference-file) supplies. + +| Parameter | Type | Default | Description | +|----------------------|--------|------------|--------------------------------------------------------------------------------------------| +| `overlap_policy` | string | `smallest` | `smallest`, `largest`, `first` or `priority`. Winner when a point is inside several zones. | +| `priority_attribute` | string | *(empty)* | Attribute ranked when `overlap_policy=priority`. Read only by that policy. | + +Zones usually overlap because they nest — a building inside a plant inside a +region — so a point matches all three: + +- `smallest` — smallest area, the most specific zone. The default, because + nesting is the common case. +- `largest` — largest area, the most general zone. The region rather than the + building, for rolling a detailed file up to a coarse column. +- `first` — file order. Cheapest and fully predictable when zones never overlap. +- `priority` — highest value of `priority_attribute`. + +One reference file can therefore feed two triggers at different levels of detail: + +```bash +--trigger-arguments 'output_columns=zone:geo_building,overlap_policy=smallest' +--trigger-arguments 'output_columns=zone:geo_region,overlap_policy=largest' +``` +```json +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { "facility": "KONA-01", "region": "HI-WEST" }, + "geometry": { + "type": "Polygon", + "coordinates": [[[-156.0,19.6],[-155.9,19.6],[-155.9,19.7],[-156.0,19.7],[-156.0,19.6]]] + } + } + ] +} +``` +A zone crossing the antimeridian must be **split at it**, as RFC 7946 requires. +An unsplit ring runs the wrong way around the globe: it stops matching its own +interior and starts matching the opposite side of the planet. The plugin warns +at startup about any geometry spanning more than 180° of longitude. + +### `nearest` — closest site from a list + +Answers "which of my sites is this point at?" for anyone who has site +coordinates but no drawn boundaries. Attributes: whatever the +[reference file](#reference-file) supplies, plus `distance_m`. + +| Parameter | Type | Default | Description | +|-----------------|--------|---------|--------------------------------------------------------------------------| +| `nearest_count` | int | `1` | How many closest sites to describe. `1` or more. | +| `max_radius_m` | number | `1000` | Meters. Points farther than this from every site are unknown. Above `0`. | + +Matching runs on the unit sphere, so the nearest site is the true great-circle +nearest, not an artifact of longitude compression. + +`max_radius_m` is what keeps "nearest" meaningful — without it every point on +Earth belongs to some site. Map `distance_m` to see how good each match was: a +truck 40 m from the depot is *at* the depot, one 900 m away merely happens to be +closest. Because it is a float column it cannot hold `unknown_value`; +unresolved rows get **`-1`**. + +Raise it well past the default for assets that spend most of their time in +transit. A fleet on the highway sits tens of kilometers from every depot, so at +`1000` every row reads `UNKNOWN` and `-1` until the vehicle pulls into a yard. +There is no "no limit" keyword: pass a value larger than half the Earth's +circumference, such as `max_radius_m=20000000`. + +#### More than one site per point + +`nearest_count` describes the closest *N* sites instead of just the closest one. +Ranks after the first repeat every output column with a `_2`, `_3` suffix: + +```bash +--trigger-arguments 'output_columns=code:geo_site distance_m:geo_dist,nearest_count=3' +``` +| geo_site | geo_dist | geo_site_2 | geo_dist_2 | geo_site_3 | geo_dist_3 | +|-----------|----------|------------|------------|------------|------------| +| `KONA-01` | `189.7` | `WAIM-03` | `53731.9` | `HILO-02` | `95515.9` | + +A suffix group describes one site completely, so map the site's name alongside +its distance — `geo_dist_2` on its own does not say what it measured. Ranks are +filtered by `max_radius_m` individually: a second site outside the radius leaves +that group unresolved while the first stays populated. Asking for more sites +than the file holds is not an error; the surplus ranks are always unresolved and +the plugin warns once at startup. + +Ranks come from distance alone, so mid-route the neighbors are simply whatever +the vehicle is driving past — read them as "what is nearby", not as the route. + +With `output_mode=tag` every rank becomes a tag, and the plugin warns about it. +The series key then covers the whole combination of ranks, and two nearly +equidistant sites trade places as the point moves, opening a new series on each +swap. Distances are floats and stay fields either way. + +### `grid` — cell of a global grid + +Answers "which cell of a fixed worldwide grid is this point in?" No reference +data. All points in a cell collapse to one value you can `GROUP BY`, which is +what makes heatmaps possible when the query engine has no geo functions of its +own. Attribute: `cell`. + +| Parameter | Type | Default | Description | +|------------------|--------|-----------------|-----------------------------------------------------| +| `grid_type` | string | `h3` | `h3`, `geohash` or `s2`. | +| `grid_precision` | int | `7` / `6` / `9` | Cell size. Range depends on `grid_type`, see below. | + +- **`h3`** — hexagons. Every neighbor is equidistant from the center, so + neighborhood and distance analysis behaves well. Resolutions 0–15. +- **`geohash`** — lat/lon rectangles. The identifier's prefix is a coarser cell, + so truncating it zooms out. Cells stretch away from the equator. Lengths 1–12. +- **`s2`** — spherical quadrilaterals with even areas worldwide. Levels 0–30. + +| `grid_type` | `grid_precision` | cell size | +|-------------|------------------|--------------------------| +| `h3` | 6 | 36.1 km², edge 3 724 m | +| `h3` | **7** | 5.16 km², edge 1 406 m | +| `h3` | 8 | 0.74 km², edge 531 m | +| `h3` | 9 | 0.11 km², edge 201 m | +| `geohash` | 5 | 4 892 × 4 892 m | +| `geohash` | **6** | 611 × 1 223 m | +| `geohash` | 7 | 153 × 153 m | +| `s2` | **9** | 324 km², side ≈ 18 km | +| `s2` | 11 | 20.3 km², side ≈ 4.5 km | +| `s2` | 13 | 1.27 km², side ≈ 1.1 km | + +> **Precision is a cardinality control.** Each finer step multiplies the distinct +> values the column can take — the very thing this plugin otherwise exists to +> contain. Stay at or above the defaults unless you have measured the effect. + +## Software Requirements + +- **{{% product-name %}}**: 3.8.2 or later, with the Processing Engine enabled +- **Python packages**: `influxdata-plugin-utils>=0.4.0`, plus the packages for + the strategies you use + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled: + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install the packages: + + ```bash + influxdb3 install package influxdata-plugin-utils + influxdb3 install package reverse_geocode # strategy=builtin + influxdb3 install package shapely # strategy=polygon + influxdb3 install package scipy # strategy=nearest + influxdb3 install package jsonpath-ng # a GeoJSON reference_file + influxdb3 install package h3 # grid_type=h3, h3_field + influxdb3 install package pygeohash # grid_type=geohash, geohash_field + influxdb3 install package s2sphere # grid_type=s2 + ``` + Packages are imported lazily, so only install what your configuration uses. + +## Trigger setup + +### Write trigger (live enrichment) + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/geo_enrichment/geo_enrichment.py \ + --trigger-spec "table:gps" \ + --trigger-arguments 'source_measurements=gps,output_columns=country_code:geo_country city:geo_city,strategy=builtin' \ + geo_enrich_gps +influxdb3 enable trigger --database mydb geo_enrich_gps +``` +### HTTP trigger (backfill) + +The trigger needs no arguments — the request body carries the configuration. + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/geo_enrichment/geo_enrichment.py \ + --trigger-spec "request:geo_backfill" \ + geo_backfill +influxdb3 enable trigger --database mydb geo_backfill + +curl -X POST "http://localhost:8181/api/v3/engine/geo_backfill" \ + -H "Authorization: Bearer $INFLUXDB3_AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "source_measurements": "gps", + "output_columns": "country_code:geo_country city:geo_city", + "strategy": "builtin", + "start": "2026-08-01T00:00:00Z", + "end": "2026-08-29T00:00:00Z", + "retry_unknown": true + }' +``` +**Expected response:** + +```json +{ + "status": "ok", + "measurement": "gps", + "stats": { + "rows": 5730, "resolved": 5719, "unresolved": 11, + "no_coordinates": 0, "invalid_coordinates": 0, "skipped_enriched": 0, + "cache_hits": 5602, "cache_misses": 128, "errors": 0, "written": 5730 + } +} +``` +## Example usage + +### Example 1: Country and city on fleet positions (in place) + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/geo_enrichment/geo_enrichment.py \ + --trigger-spec "table:gps" \ + --trigger-arguments 'source_measurements=gps,output_columns=country_code:geo_country city:geo_city,strategy=builtin,min_population=10000' \ + geo_country +``` +```bash +influxdb3 write --database mydb 'gps,device=truck7 lat=55.7558,lon=37.6173,speed=54' +influxdb3 query --database mydb "SELECT device, speed, geo_country, geo_city FROM gps" +``` +``` +device speed geo_country geo_city +truck7 54 RU Moscow +``` +### Example 2: Your own plant zones, as real tags + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/geo_enrichment/geo_enrichment.py \ + --trigger-spec "table:gps" \ + --trigger-arguments 'source_measurements=gps,target_measurement=gps_zoned,output_mode=tag,output_columns=facility:facility region:region,strategy=polygon,reference_file=/plugins/data/zones.geojson,overlap_policy=smallest' \ + geo_zones +``` +```bash +influxdb3 query --database mydb \ + "SELECT facility, avg(speed) FROM gps_zoned GROUP BY facility" +``` +### Example 3: Nearest depot with match quality + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/geo_enrichment/geo_enrichment.py \ + --trigger-spec "table:gps" \ + --trigger-arguments 'source_measurements=gps,output_columns=code:geo_site distance_m:geo_distance,strategy=nearest,reference_file=/plugins/data/sites.csv,max_radius_m=500' \ + geo_sites +``` +```bash +influxdb3 query --database mydb \ + "SELECT geo_site, count(*) FROM gps WHERE geo_distance >= 0 GROUP BY geo_site" +``` +### Example 4: H3 cells for a heatmap + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/geo_enrichment/geo_enrichment.py \ + --trigger-spec "table:gps" \ + --trigger-arguments 'source_measurements=gps,output_columns=cell:geo_cell,strategy=grid,grid_type=h3,grid_precision=8' \ + geo_cells +``` +```bash +influxdb3 query --database mydb \ + "SELECT geo_cell, count(*) AS hits FROM gps GROUP BY geo_cell ORDER BY hits DESC" +``` +### Example 5: Integer-encoded tracker with a combined column + +```bash +--trigger-arguments 'source_measurements=tracker,output_columns=country_code:geo_country,point_field=pos,point_format=lon_lat,coord_scale=1e7' +``` +## Code overview + +### Files + +- `geo_enrichment.py` — the plugin +- `geo_enrichment_config_data_writes.toml` — annotated configuration template +- `manifest.toml` — plugin manifest +- `requirements.txt` — runtime dependencies +- `test_geo_enrichment.py` — pytest suite, runs without a live {{% product-name %}} server +- `README.md` — this documentation + +### Logging + +Logs go to the `system.processing_engine_logs` table. Each run logs a summary: + +``` +rows=1200 resolved=1180 unresolved=20 no_coordinates=0 invalid=0 +already_enriched=600 cache_hits=1150 cache_misses=50 errors=0 written=1200 +``` +`already_enriched` counts echo-batch rows, so on a healthy in-place trigger it +is roughly equal to the rows written on the previous pass. + +### Main functions + +#### `process_writes(influxdb3_local, table_batches, args)` + +Enriches each WAL flush. Skips rows that already carry the output columns, then +extracts, validates, resolves and writes. Writes do not retry — a WAL-flush +trigger runs inline with ingestion, so a backoff sleep would throttle it. + +#### `process_request(influxdb3_local, query_parameters, request_headers, request_body, args)` + +Pages through a time range with the same pipeline and retries failed writes. +Re-reads the reference data on every call. + +#### `read_reference(cfg, requested_attributes)` + +Reads the reference file by extension into geometry/attribute records, which both +`polygon` and `nearest` build their index from. + +#### `resolve_attributes(cfg, resolver, memo, lat, lon)` + +Rounds the coordinate, reuses a memoized result, otherwise calls the resolver. + +#### `build_enrichment_line(row, table, values, cfg, schema)` + +Builds the output line: in place only the source tags are reproduced, so the +write merges; to a target table the whole row is copied. + +## Troubleshooting + +### Issue: "output_mode='tag' needs 'target_measurement'" + +A tag is part of a row's identity, so writing one into the source table creates a +second row and doubles every aggregate. Either set `target_measurement`, or use +`output_mode=field` to enrich in place. + +### Issue: rows have `UNKNOWN` everywhere + +- `strategy=nearest`: every site is farther than `max_radius_m`. Map + `distance_m` to a column and query it to see the real distances. When only the + `_2` and later groups are unresolved, either the radius excludes them or + `nearest_count` exceeds the number of sites in the file. +- `strategy=polygon`: the point falls outside every zone. Check that the file + uses `[longitude, latitude]` order, which both GeoJSON and WKT require and + which is the reverse of how coordinates are usually spoken. + +### Issue: no geo columns appear at all + +- The trigger table must be listed in `source_measurements`. +- Rows without usable coordinates are skipped silently; check `no_coordinates` + in the summary log. +- If coordinates arrive as scaled integers, set `coord_scale`. + +### Issue: "'attribute' cannot be produced by strategy" + +`output_columns` names an attribute the strategy does not have. The error lists +what is available; for `polygon` and `nearest` that comes from your own file, so +a typo in a CSV header shows up here. A typo in a GeoJSON path is reported +separately, as a path matching no feature. + +### Issue: field type conflict on write + +Geo attributes are written as strings and `distance_m` as a float. If a column +of that name already exists with another type, the whole write batch is +rejected. Choose different column names, or drop the old column. + +### Issue: the trigger fires twice per write + +Expected in place — see [The echo batch](#the-echo-batch). The second pass writes +nothing. + +### Issue: a package is missing + +The error names the package and what needs it — a strategy, a coordinate input, +or the reference file's format: + +``` +'strategy=polygon' needs the 'shapely' package. Install it with +'influxdb3 install package shapely'. +``` + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/import.md b/content/shared/influxdb3-plugins/plugins-library/official/import.md new file mode 100644 index 0000000000..5f7667ca3e --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/import.md @@ -0,0 +1,840 @@ + + +> **Note:** This plugin requires {{% product-name %}}.8.2 or later. + + +The InfluxDB Import Plugin enables seamless data import from InfluxDB v1, v2, or v3 instances to {{% product-name %}}. It provides comprehensive import capabilities with pause/resume functionality, progress tracking, conflict detection, and robust error handling. The plugin operates via HTTP endpoints, allowing you to start, pause, resume, cancel, and monitor imports through simple HTTP requests. + +Key features: +- Import data from InfluxDB v1, v2, or v3 to {{% product-name %}} +- Automatic data sampling for optimal batch sizing +- Resume interrupted imports from the last checkpoint +- Pause and cancel running imports +- Crash recovery with stale import detection +- Progress tracking and statistics +- Tag/field conflict detection and resolution +- Data type mismatch handling +- Configurable time ranges and table filtering +- Dry run mode for import planning (estimates, schema conflicts, configuration preview) +- Support for both token and username/password authentication + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI), in the `trigger_arguments` field (API) when creating a trigger or via body of HTTP request. This plugin supports TOML configuration files, which can be specified using the `config_file_path` parameter. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Required parameters + +| Parameter | Type | Default | Description | +|---------------------|---------|----------|--------------------------------------------------------------------| +| `source_url` | string | required | Source InfluxDB URL (with optional port, for example, `http://localhost:8086`) | +| `influxdb_version` | integer | required | Source InfluxDB version: 1, 2, or 3 | +| `source_database` | string | required | Source database name to import from | + +### Authentication + +Credentials are passed via HTTP headers on each request. + +| Header | Purpose | +|--------|---------| +| `Source-Token` | Bearer/API token authentication (InfluxDB v1/v2/v3) | +| `Source-Username` | Basic auth username (InfluxDB v1) | +| `Source-Password` | Basic auth password (InfluxDB v1) | + +**Method 1: Token-based authentication** + +Use the `Source-Token` header for token-based authentication: + +```bash +curl -X POST http://localhost:8181/api/v3/engine/import?action=start \ + -H "Content-Type: application/json" \ + -H "Source-Token: my-secret-token" \ + -d '{ + "source_url": "http://localhost:8086", + "influxdb_version": 2, + "source_database": "telegraf" + }' +``` +**Method 2: Username/Password authentication** + +Use the `Source-Username` and `Source-Password` headers for basic authentication: + +```bash +curl -X POST http://localhost:8181/api/v3/engine/import?action=start \ + -H "Content-Type: application/json" \ + -H "Source-Username: admin" \ + -H "Source-Password: my-password" \ + -d '{ + "source_url": "http://localhost:8086", + "influxdb_version": 1, + "source_database": "telegraf" + }' +``` +> **Note**: Authentication errors from the source InfluxDB are returned directly. The plugin does not validate credentials upfront. + +### Optional parameters + +| Parameter | Type | Default | Description | +|----------------------|---------|----------------|---------------------------------------------------------------------------------------------------------------| +| `dest_database` | string | none | Destination database name in {{% product-name %}} (if not specified, uses database where trigger was created) | +| `start_timestamp` | string | none | Import start time (datetime format). If not specified, starts from oldest data | +| `end_timestamp` | string | none | Import end time (datetime format). If not specified, imports to newest data | +| `query_interval_ms` | integer | 100 | Delay between queries in milliseconds to avoid overloading source database | +| `import_direction` | string | "oldest_first" | Import direction: "oldest_first" or "newest_first" | +| `target_batch_size` | integer | 2000 | Target number of rows per query batch | +| `table_filter` | string | none | Dot-separated list of tables to import (for example, "cpu.mem.disk"). If not specified, imports all tables | +| `dry_run` | boolean | false | If true, generates import plan without processing data (shows estimates, schema conflicts, and configuration) | + +### TOML configuration + +| Parameter | Type | Default | Description | +|--------------------|--------|---------|----------------------------------------------------------------------------------| +| `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (required for TOML configuration) | + +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. + +#### Example TOML configuration + +[import_config.toml](https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/import/import_config.toml) + +For more information on using TOML configuration files, see the Using TOML Configuration Files section in the [influxdb3_plugins/README.md](https://github.com/influxdata/influxdb3_plugins/blob/master/README.md). + +## Software Requirements + +- **{{% product-name %}}**: with the Processing Engine enabled. +- **Source InfluxDB instance**: InfluxDB v1.x or v2.x instance accessible via HTTP/HTTPS. +- **Python packages**: + - `requests` (for HTTP communication with source InfluxDB) + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine and `PLUGIN_DIR` environment variable: + + ```bash + PLUGIN_DIR=~/.plugins influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install required Python packages: + + ```bash + influxdb3 install package requests + ``` +## Trigger setup + +### HTTP trigger setup + +Create an HTTP trigger to handle import requests: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/import/import.py \ + --trigger-spec "request:import" \ + import_trigger +``` +Enable the trigger: + +```bash +influxdb3 enable trigger --database mydb import_trigger +``` +The endpoint is registered at `/api/v3/engine/import`. + +## HTTP Endpoint + +The import plugin provides the following type of requests: + +### Start Import + +Start a new import from source InfluxDB to {{% product-name %}}. + +**Request**: `POST /api/v3/engine/import?action=start` + +**Headers**: +- `Source-Token: my-token` (or `Source-Username` + `Source-Password`) +- `Content-Type: application/json` + +**Request body** (JSON): +```json +{ + "source_url": "http://localhost:8086", + "influxdb_version": 1, + "source_database": "telegraf", + "dest_database": "imported_data", + "start_timestamp": "2024-01-01T00:00:00Z", + "end_timestamp": "2024-12-31T23:59:59Z", + "table_filter": "cpu.mem.disk" +} +``` +### Get Import Status + +Check the status and progress of a import. + +**Request**: `GET /api/v3/engine/import?action=status&import_id=` + + +### Pause Import + +Pause a running import to resume later. + +**Request**: `POST /api/v3/engine/import?action=pause&import_id=` + +> **Note**: Returns error if import is not found, already paused, already cancelled, or already completed. + +### Resume Import + +Resume a paused or interrupted import. + +**Request**: `POST /api/v3/engine/import?action=resume&import_id=` + +**Headers**: +- `Source-Token: my-token` (or `Source-Username` + `Source-Password`) + +> **Note**: Credentials must be provided via headers when resuming. Authentication credentials are not stored for security reasons and must be provided when resuming. Returns error if import is not found, already cancelled, already completed, or actively running. + +#### Crash recovery + +If the plugin or the server crashes during an import, the import state may be left as "running" even though nothing is actually running. The resume action handles this with **stale import detection**: + +- When the import state is "running", the plugin checks the timestamp of the last `import_state` record. +- If the last update is older than **5 minutes**, the import is considered stale (crashed) and resume is allowed. +- If no `import_state` records exist at all (the import crashed before processing any tables), the import is restarted from the beginning. + +If the plugin itself crashes (for example, source database becomes unavailable), it writes a paused state before exiting, so the import can be resumed with a regular resume call after fixing the issue. + +### Cancel Import + +Cancel a running import. Cancelled imports cannot be resumed. + +**Request**: `POST /api/v3/engine/import?action=cancel&import_id=` + +> **Note**: Returns error if import is not found, already cancelled, or already completed. + +### Test Connection + +Test connectivity to a URL and identify if it's an InfluxDB instance. Uses a 5-second timeout for fast feedback. + +**Request**: `POST /api/v3/engine/import?action=test_connection` + +**Request body** (JSON): +```json +{ + "source_url": "http://localhost:8086" +} +``` +> **Note**: If port is omitted, it is inferred from the scheme (`http` → 80, `https` → 443). + +**Success response** (InfluxDB v1/v2 detected): +```json +{ + "success": true, + "version": "2.7.0", + "build": "OSS" +} +``` +**Success response** (InfluxDB v3 detected via `cluster-uuid` header): +```json +{ + "success": true, + "version": "3.x.x", + "build": "" +} +``` +> **Note**: InfluxDB v3 does not expose version headers without authentication. Detection uses the `cluster-uuid` header instead. + +**Failure response** (not InfluxDB or unreachable): +```json +{ + "success": false, + "message": "Not an InfluxDB instance" +} +``` +**Failure response** (InfluxDB requires authentication, version unknown): +```json +{ + "success": false, + "message": "Unable to determine InfluxDB version" +} +``` +> **Note**: When InfluxDB returns 401/403 without version headers, the connection test cannot determine the version. This typically means authentication is required. The instance is likely InfluxDB, but version detection requires valid credentials. + +### List Databases + +Get list of databases from source InfluxDB instance. + +**Request**: `POST /api/v3/engine/import?action=databases` + +**Headers**: +- `Source-Token: my-token` (or `Source-Username` + `Source-Password`) +- `Content-Type: application/json` + +**Request body** (JSON): +```json +{ + "source_url": "http://localhost:8086", + "influxdb_version": 1 +} +``` +### List Tables + +Get list of tables/measurements from a source database. + +**Request**: `POST /api/v3/engine/import?action=tables` + +**Headers**: +- `Source-Token: my-token` (or `Source-Username` + `Source-Password`) +- `Content-Type: application/json` + +**Request body** (JSON): +```json +{ + "source_url": "http://localhost:8086", + "influxdb_version": 1, + "source_database": "telegraf" +} +``` +> **Note**: For InfluxDB v2, include `source_org` in the request body. + +## Example usage + +### Example 1: Basic import with token authentication + +Import all data from an InfluxDB v1 instance: + +```bash +# Create and enable HTTP trigger +influxdb3 create trigger \ + --database mydb \ + --plugin-filename import.py \ + --trigger-spec "request:import" \ + import_trigger + +influxdb3 enable trigger --database mydb import_trigger + +# Start import via HTTP +curl -X POST http://localhost:8181/api/v3/engine/import?action=start \ + -H "Source-Token: my-super-secret-token" \ + -H "Content-Type: application/json" \ + -d '{ + "source_url": "http://localhost:8086", + "influxdb_version": 1, + "source_database": "telegraf", + "dest_database": "imported_data" + }' +``` +### Expected results + +- Plugin connects to source InfluxDB at `http://localhost:8086` (port from URL) +- Discovers all measurements in the `telegraf` database +- Estimates import time based on data sampling +- Imports all data to {{% product-name %}} in the `imported_data` database +- Logs import_id for tracking statistics + +### Example 2: Time-range import with table filtering + +Import specific tables within a date range: + +```bash +# Start import with time range and table filter +curl -X POST http://localhost:8181/api/v3/engine/import?action=start \ + -H "Source-Username: admin" \ + -H "Source-Password: my-password" \ + -H "Content-Type: application/json" \ + -d '{ + "source_url": "http://influxdb-source.example.com:8086", + "influxdb_version": 1, + "source_database": "telegraf", + "dest_database": "production_metrics", + "start_timestamp": "2024-01-01T00:00:00Z", + "end_timestamp": "2024-12-31T23:59:59Z", + "table_filter": "cpu.mem.disk.network", + "import_direction": "newest_first", + "target_batch_size": 5000 + }' +``` +### Expected results + +- Imports only `cpu`, `mem`, `disk`, and `network` measurements +- Processes data from January 1, 2024 to December 31, 2024 +- Imports newest data first +- Uses larger batch size (5000 rows) for better performance + +### Example 3: Pause, check status, and resume import + +Monitor and control a long-running import: + +```bash +# Start import (logs import_id, does not return it immediately) +curl -X POST http://localhost:8181/api/v3/engine/import?action=start \ + -H "Source-Token: my-token" \ + -H "Content-Type: application/json" \ + -d '{ + "source_url": "http://localhost:8086", + "influxdb_version": 2, + "source_database": "large_database", + "dest_database": "imported" + }' + +# Find import_id from logs: +influxdb3 query --database _internal "SELECT log_text FROM system.processing_engine_logs WHERE trigger_name = 'import_trigger' AND log_text LIKE '%Starting import%' ORDER BY event_time DESC LIMIT 1" + +# Set the import_id from logs +IMPORT_ID="" + +# Pause import (for example, during high-traffic hours) +curl -X POST "http://localhost:8181/api/v3/engine/import?action=pause&import_id=$IMPORT_ID" + +# Check status after import completion (paused, cancelled, or completed) +curl "http://localhost:8181/api/v3/engine/import?action=status&import_id=$IMPORT_ID" + +# Resume later +curl -X POST "http://localhost:8181/api/v3/engine/import?action=resume&import_id=$IMPORT_ID" \ + -H "Source-Token: my-token" +``` +### Expected results + +- Import starts and logs a unique import_id (check logs to obtain it) +- Import continues running in the background, logging progress +- Pause command stops import gracefully at current position +- Status endpoint returns comprehensive statistics **only after import completion** (paused, cancelled, or finished) +- Resume command continues from the exact point where it was paused and returns final results upon completion + +### Example 4: Dry run for import plan + +```bash +curl -X POST http://localhost:8181/api/v3/engine/import?action=start \ + -H "Source-Token: my-token" \ + -H "Content-Type: application/json" \ + -d '{ + "source_url": "http://localhost:8086", + "influxdb_version": 1, + "source_database": "telegraf", + "dry_run": true + }' +``` +### Expected results + +With `dry_run: true`, the plugin generates a comprehensive import plan **without processing any data**. It only performs: +- Schema inspection (tags and fields) +- Data sampling for time estimation +- Conflict detection + +The response returns immediately with a detailed import plan: + +```json {lint="false"} +{ + "import_id": "abc123...", + "status": "dry_run_plan", + "source": { + "url": "http://localhost:8086", + "database": "telegraf", + "influxdb_version": 1 + }, + "destination": { + "database": "imported_data" + }, + "time_range": { + "start": "all data", + "end": "all data" + }, + "import_settings": { + "direction": "oldest_first", + "target_batch_size": 2000, + "query_interval_ms": 100 + }, + "tables": { + "total": 5, + "list": ["cpu", "mem", "disk", "network", "processes"], + "filtered": "all tables" + }, + "estimated_import": { + "total_rows": 5000000, + "estimated_duration": "1 hour 15 minutes", + "estimated_duration_seconds": 4500, + "per_table_estimates": [ + { + "measurement": "cpu", + "estimated_rows": 1000000, + "estimated_seconds": 900 + }, + { + "measurement": "mem", + "estimated_rows": 800000, + "estimated_seconds": 720 + } + ] + }, + "schema_conflicts": { + "total": 2, + "details": [ + { + "measurement": "cpu", + "type": "tag_field_conflict", + "conflicts": ["host", "region"], + "resolution": "Tags will be renamed with '_tag' suffix: host -> host_tag, region -> region_tag" + } + ] + } +} +``` +**Note**: Dry run mode is fast and lightweight - it does not query or process any actual data points, only metadata. Use it to: +- Preview import scope and estimates +- Identify schema conflicts before import +- Validate configuration and connectivity +- Plan import time windows + +## Using TOML Configuration Files + +This plugin supports using TOML configuration files to specify all plugin arguments. + +### Important Requirements + +**To use TOML configuration files, you must set the `PLUGIN_DIR` environment variable in the {{% product-name %}} host environment.** + +### Setting Up TOML Configuration + +1. **Start {{% product-name %}} with the PLUGIN_DIR environment variable set**: + + ```bash + PLUGIN_DIR=~/.plugins influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. **Copy the example TOML configuration file to your plugin directory**: + + ```bash + cp import_config.toml ~/.plugins/ + ``` +3. **Edit the TOML file** to match your requirements: + + ```toml + # Required parameters + source_url = "http://localhost:8086" + influxdb_version = 1 + source_database = "telegraf" + + # Optional parameters + dest_database = "imported_data" + start_timestamp = "2024-01-01T00:00:00Z" + end_timestamp = "2024-12-31T23:59:59Z" + table_filter = "cpu.mem.disk" + ``` +> **Note**: Credentials are NOT stored in TOML files. Provide them via HTTP headers on each request. + +4. **Create a trigger using the `config_file_path` argument**: + + ```bash + influxdb3 create trigger \ + --database mydb \ + --plugin-filename import.py \ + --trigger-spec "request:import" \ + --trigger-arguments config_file_path=import_config.toml \ + import_trigger + ``` +5. **Start import via HTTP** (config from TOML file will be used as defaults, can be overridden in request body): + + ```bash + curl -X POST http://localhost:8181/api/v3/engine/import?action=start + ``` +## Configuration Priority and Loading + +The import plugin loads configuration from multiple sources with the following priority order (highest to lowest): + +1. **HTTP Request Body** (highest priority) - JSON parameters in POST request body +2. **TOML Configuration File** - Parameters from file specified in `config_file_path` +3. **Trigger Arguments** - Parameters from `--trigger-arguments` when creating trigger +4. **Environment Variables** (lowest priority) - System environment variables + +### Configuration Loading Process + +When a import starts, the plugin loads configuration in this order: + +```python +# 1. Start with environment variables (lowest priority) +IMPORT_SOURCE_URL, IMPORT_SOURCE_DATABASE, etc. + +# 2. Override with trigger arguments (--trigger-arguments) +config_file_path=import_config.toml, source_url=http://localhost:8086, etc. + +# 3. Override with TOML file contents (if config_file_path specified) +[from import_config.toml file] + +# 4. Override with HTTP request body (highest priority) +{ + "source_url": "http://localhost:8086", + ... +} +``` +### Environment Variables Supported + +The following environment variables can be used: + +- `IMPORT_SOURCE_URL` → `source_url` +- `IMPORT_SOURCE_DATABASE` → `source_database` +- `IMPORT_DEST_DATABASE` → `dest_database` +- `IMPORT_START_TIMESTAMP` → `start_timestamp` +- `IMPORT_END_TIMESTAMP` → `end_timestamp` + +## Data Type Mismatch Handling + +The plugin automatically handles data type mismatches that can occur in older InfluxDB versions where different nodes might have different field types for the same field name. + +### How It Works + +1. **Schema Detection**: At import start, plugin queries source database for field types using `SHOW FIELD KEYS` +2. **Runtime Type Checking**: For each data point, plugin checks if the actual value type matches the expected field type +3. **Automatic Field Creation**: If type mismatch is detected, plugin creates a new field with a type suffix + +### Supported Type Suffixes + +When type mismatches occur, the plugin appends these suffixes: + +- `_string` - for string values +- `_integer` - for integer values +- `_float` - for float values +- `_boolean` - for boolean values + +## Code overview + +### Files + +- `import.py`: The main plugin code containing HTTP request handler and import logic +- `import_config.toml`: Example TOML configuration file + +### Logging + +Logs are stored in the `_internal` database in the `system.processing_engine_logs` table: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'import_trigger'" +``` +Log columns: + +- **event_time**: Timestamp of the log event +- **trigger_name**: Name of the trigger that generated the log +- **log_level**: Severity level (INFO, WARN, ERROR) +- **log_text**: Message describing the action or error + +### Import state tracking + +The plugin creates several measurements to track import state: + +#### `import_config` +Stores import configuration (credentials excluded for security). + +```bash +influxdb3 query --database mydb "SELECT * FROM import_config WHERE import_id = 'your-import-id'" +``` +#### `import_state` +Tracks per-table import progress. + +```bash +influxdb3 query --database mydb "SELECT * FROM import_state WHERE import_id = 'your-import-id' ORDER BY time DESC" +``` +#### `import_pause_state` +Stores pause/cancel/completed state for controlling running imports. + +```bash +influxdb3 query --database mydb "SELECT * FROM import_pause_state WHERE import_id = 'your-import-id' ORDER BY time DESC LIMIT 1" +``` +### Main functions + +#### `process_request(influxdb3_local, query_parameters, request_headers, request_body, args)` + +HTTP request handler that routes to appropriate import actions based on the `action` query parameter. Extracts credentials from `request_headers` using `extract_credentials()` and passes them to action handlers. + +#### `extract_credentials(request_headers)` + +Extracts authentication credentials from HTTP headers. Returns a dict with keys `source_token`, `source_username`, `source_password` (values are `None` if header not present). + +#### `start_import(influxdb3_local, config, credentials, task_id)` + +Starts a new import process: +1. Performs pre-flight checks (connectivity, measurements discovery) +2. Estimates import time based on data sampling +3. Creates import configuration and state records +4. Initiates table-by-table import +5. On any unhandled error, writes paused state so the import can be resumed after fixing the issue + +#### `import_table(influxdb3_local, config, credentials, import_id, measurement, start_time, end_time, task_id, ...)` + +Imports a single table: +1. Finds actual data boundaries within specified range +2. Samples data to determine optimal batch window size +3. Detects and resolves tag/field conflicts +4. Queries data in batches and converts to line protocol +5. Writes to destination database +6. Tracks progress and checks for pause/cancel signals + +#### `resume_import(influxdb3_local, import_id, credentials, task_id)` + +Resumes an interrupted import: +1. Detects stale imports — if the import state is "running" but the last update is older than 5 minutes, treats it as crashed and allows resume +2. If no `import_state` records exist (crashed before processing any tables), restarts from the beginning +3. Loads saved import configuration +4. Identifies incomplete tables and their last checkpoint +5. Continues import from checkpoint positions +6. On any unhandled error, writes paused state so the import can be resumed again + +#### `get_import_stats(influxdb3_local, import_id, task_id)` + +Returns comprehensive statistics for a import including overall status, per-table progress, timing information, and configuration. + +#### `check_source_connection(body_data, session)` + +Tests connectivity to a URL and identifies if it's an InfluxDB instance (5-second timeout): +1. Validates that `source_url` is provided +2. Infers port from scheme if not specified (http→80, https→443) +3. Sends request to `/ping` endpoint +4. Returns success with version/build from `X-Influxdb-*` headers (v1/v2) +5. Falls back to `cluster-uuid` header detection for v3 (returns `version: "3.x.x"`) +6. Returns failure with message if not InfluxDB or unreachable + +#### `get_source_databases_list(body_data, credentials, session)` + +Lists databases from source InfluxDB instance: +1. Validates required parameters +2. For v1: Executes `SHOW DATABASES` query, filters out `_internal` +3. For v2: Queries `/api/v2/buckets` API, filters out system buckets (prefixed with `_`) +4. Returns sorted list of database names + +#### `get_source_tables_list(body_data, credentials, session)` + +Lists tables/measurements from a source database: +1. Validates required parameters including `source_database` +2. For v1: Executes `SHOW MEASUREMENTS` query +3. For v2: Executes Flux schema.measurements() query (requires `source_org`) +4. Returns sorted list of table names + +### Key algorithms + +#### Automatic batch sizing + +The plugin samples data at different time intervals to determine optimal window size: + +```python +# Test intervals: 1 second, 1 minute, 1 hour, 1 day +# Calculate rows per second from samples +# Determine window size to achieve target_batch_size +optimal_window = target_batch_size / avg_rows_per_second +``` +#### Tag/field conflict resolution + +When a column name exists as both tag and field in source data: + +```python +# Original data has conflict: +# tag: room +# field: room + +# Plugin renames conflicting tag: +# tag: room_tag +# field: room (unchanged) +``` +#### Resume checkpoint tracking + +During import, the plugin saves checkpoints: + +```python +# Save paused_at_time (data timestamp, not record timestamp) +# On resume: +# 1. Load last paused_at_time +# 2. Add 1 microsecond offset to avoid duplicates +# 3. Continue import from (paused_at_time + 1µs) +``` +## Troubleshooting + +### Common issues + +#### Issue: "Failed to connect to source database" error + +**Solution**: + +1. Verify source InfluxDB is running and accessible: + ```bash + curl http:///ping + ``` +2. Check network connectivity and firewall rules +3. Verify credentials are correct +4. For InfluxDB v2/v3, ensure you're using token authentication + +#### Issue: Authentication errors from source InfluxDB + +**Solution**: Pass credentials via HTTP headers: +1. For token auth: Add header `-H "Source-Token: your-token"` +2. For username/password: Add headers `-H "Source-Username: user" -H "Source-Password: pass"` +3. Verify credentials work directly against source InfluxDB +4. Check that headers are not being stripped by proxies + +#### Issue: "Import is already running" after server crash + +**Solution**: + +1. Wait at least 5 minutes after the crash — the plugin uses a 5-minute stale import threshold +2. Call the resume endpoint with credentials: + ```bash + curl -X POST "http://localhost:8181/api/v3/engine/import?action=resume&import_id=$IMPORT_ID" \ + -H "Content-Type: application/json" \ + -d '{"source_token": "my-token"}' + ``` +3. The plugin detects the stale state and resumes from the last checkpoint (or restarts from the beginning if no checkpoint exists) + +#### Issue: "Import already completed" when trying to resume + +**Solution**: + +1. Check import status: + ```bash + curl "http://localhost:8181/api/v3/import?action=status&import_id=" + ``` +2. If truly incomplete, check for status discrepancies in `import_state` table +3. Start a new import if needed + +#### Issue: Tag/field conflicts causing warnings + +**Solution**: This is informational only. The plugin automatically renames conflicting tags with a `_tag` suffix: +- Original tag `temperature` → `temperature_tag` +- Field `temperature` remains unchanged + +#### Issue: Slow import performance + +**Solution**: + +1. Increase `target_batch_size` (for example, from 2000 to 5000) +2. Decrease `query_interval_ms` if source can handle higher load +3. Use table filtering to import tables in parallel using multiple triggers +4. Check network latency between source and destination + + +### Performance considerations + +- **Network bandwidth**: Main bottleneck for large imports. Use local network when possible. +- **Source database load**: The plugin includes rate limiting (`query_interval_ms`) to avoid overwhelming source. +- **Batch size optimization**: Plugin automatically samples data to determine optimal batch size, but you can override with `target_batch_size`. +- **Connection pooling**: Plugin uses HTTP session with connection pooling for better performance. +- **Retry logic**: Built-in exponential backoff (1s → 2s → 4s → 8s → 16s) for transient errors. + +## Import best practices + +1. **Use table filtering**: Import critical tables first, then others in batches +2. **Plan for pauses**: Pause during high-traffic hours if sharing infrastructure +3. **Verify data**: Compare row counts and sample data after import +4. **Handle conflicts**: Review log warnings about tag/field conflicts + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/influxdb-to-iceberg.md b/content/shared/influxdb3-plugins/plugins-library/official/influxdb-to-iceberg.md index cd7fe51496..9c72f40f73 100644 --- a/content/shared/influxdb3-plugins/plugins-library/official/influxdb-to-iceberg.md +++ b/content/shared/influxdb3-plugins/plugins-library/official/influxdb-to-iceberg.md @@ -1,4 +1,5 @@ - + + The InfluxDB to Iceberg Plugin enables data transfer from {{% product-name %}} to Apache Iceberg tables. Transfer time series data to Iceberg for long-term storage, analytics, or integration with data lake architectures. The plugin supports both scheduled batch transfers of historical data and on-demand transfers via HTTP API. ## Configuration @@ -65,11 +66,11 @@ For more information on using TOML configuration files, see the Using TOML Confi ## Schema management - Automatically creates Iceberg table schema from the first batch of data -- Maps pandas data types to Iceberg types: - - `int64` → `IntegerType` - - `float64` → `FloatType` - - `datetime64[us]` → `TimestampType` - - `object` → `StringType` +- Maps Pandas data types to Iceberg types: + - `int64` → `IntegerType` + - `float64` → `FloatType` + - `datetime64[us]` → `TimestampType` + - `object` → `StringType` - Fields with no null values are marked as `required` - The `time` column is converted to `datetime64[us]` for Iceberg compatibility - Tables are created in format: `.` @@ -87,9 +88,9 @@ When `auto_update_schema=true`: - **{{% product-name %}}**: with the Processing Engine enabled - **Python packages**: - - `pandas` (for data manipulation) - - `pyarrow` (for Parquet support) - - `pyiceberg[catalog-options]` (for Iceberg integration) + - `pandas` (for data manipulation) + - `pyarrow` (for Parquet support) + - `pyiceberg[catalog-options]` (for Iceberg integration) ### Installation steps @@ -284,7 +285,7 @@ Key operations: 3. Creates Iceberg table if needed 4. Appends data to Iceberg table -#### `process_http_request(influxdb3_local, request_body, args)` +#### `process_request(influxdb3_local, query_parameters, request_headers, request_body, args)` Handles on-demand data transfers via HTTP. Supports backfill operations with configurable batch sizes. @@ -373,4 +374,6 @@ For plugin issues, see the Plugins repository [issues page](https://github.com/i ## Find support for {{% product-name %}} The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. -For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. \ No newline at end of file +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/kafka-subscriber.md b/content/shared/influxdb3-plugins/plugins-library/official/kafka-subscriber.md new file mode 100644 index 0000000000..4008515f2e --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/kafka-subscriber.md @@ -0,0 +1,825 @@ + + +> **Note:** This plugin requires {{% product-name %}}.8.2 or later. + + +The Kafka Subscriber Plugin enables real-time ingestion of Kafka messages into {{% product-name %}}. Subscribe to Kafka topics and automatically transform messages into time-series data with support for JSON, Line Protocol, and custom text formats. The plugin uses consumer groups for reliable message delivery and provides flexible offset commit policies for different use cases. The consumer is reused across scheduled invocations to avoid a consumer-group rebalance every cycle, and it optionally supports a dead-letter queue topic for failed messages and id-based deduplication for messages without their own timestamp. + +**Binary formats:** `avro`, `jsonschema`, and `protobuf` (Confluent wire format) are decoded via Confluent Schema Registry. `avro` can also be decoded as raw schemaless Avro with a local `.avsc` schema, and `protobuf` with a local `.proto` schema. See [Binary formats](#binary-formats). + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. This plugin supports TOML configuration files for complex mapping scenarios, which can be specified using the `config_file_path` parameter. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Required parameters + +In TOML configuration, `bootstrap_servers`, `topics`, and `group_id` are placed under the `[kafka]` section; `table_name` and `table_name_field` are placed under `[mapping.json]` or `[mapping.text]` section. + +| Parameter | Type | Default | Description | +|---------------------|--------|---------------------------|-------------------------------------------------------------------------------------------------------------------------------| +| `bootstrap_servers` | string | required | Space-separated list of Kafka broker addresses (for example, "kafka1:9092 kafka2:9092") | +| `topics` | string | required | Space-separated list of topics (for example, "sensor_data metrics") | +| `group_id` | string | required | Kafka consumer group ID (must be unique per consumer group) | +| `table_name` | string | conditional | InfluxDB measurement name for storing data. Required for all formats except `lineprotocol`, unless `table_name_field` is set. | +| `table_name_field` | string | none | JSON field name or regex pattern to extract table name dynamically from each message. Alternative to static `table_name`. | + +### Connection parameters + +In TOML configuration, these parameters are placed under the `[kafka]` section. + +| Parameter | Type | Default | Description | +|----------------------|--------|-------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `auto_offset_reset` | string | "earliest" | Where to start consuming on first connect: "earliest" or "latest" | +| `max_poll_records` | int | 500 | Maximum messages per scheduled call. Set to 0 for unlimited. | +| `max_poll_interval` | int | 300 | Maximum seconds between scheduled poll cycles before the broker considers the consumer dead (maps to `max.poll.interval.ms`). The consumer is reused across invocations, so it must stay in the group between polls — increase this when the trigger interval is longer than ~5 minutes. | + +### Offset Commit Policy + +In TOML configuration, this parameter is placed under the `[kafka]` section. + +| Parameter | Type | Default | Description | +|------------------------|--------|--------------|-------------------------------------------------------------------| +| `offset_commit_policy` | string | "on_success" | When to commit offsets: "on_success" or "always" | + +**Policy behavior:** + +| Policy | Behavior | +|--------------|----------------------------------------------------------------------------------------------| +| `on_success` | Commit offsets only after ALL messages in the batch are successfully processed and written | +| `always` | Commit offsets immediately after receiving messages, regardless of processing success | + +**When to use each policy:** + +- **on_success (recommended)**: Use when data integrity is important. Failed messages will be reprocessed on the next trigger execution. +- **always**: Use in high-throughput scenarios where occasional data loss is acceptable, or when you have external error handling (failed messages are logged to `kafka_exceptions` table). + +**Interaction with `dlq_topic`:** + +Because the consumer is reused across invocations, its fetch position advances as messages are polled. Under `on_success`, when a message still needs reprocessing — a transient InfluxDB write failure, or a parse failure (poison message) that could **not** be offloaded — the plugin **rewinds the consumer** to the lowest such offset on each affected partition, so those messages are re-read on the next cycle. Only the affected partitions are rewound; partitions whose messages all succeeded are not. A parse failure that is **successfully** published to `dlq_topic` is treated as handled: the consumer is not rewound for it, and it is preserved in the DLQ instead of being re-read. + +Without `dlq_topic`, a poison message under `on_success` is rewound and reprocessed on every cycle until it is resolved (its original payload is not stored in `kafka_exceptions`, so configure `dlq_topic` if you need to skip past and preserve poison messages). With `always`, the offset is committed immediately regardless and the consumer is never rewound, so a failed message is **not** reprocessed; `dlq_topic` simply preserves the failed messages that would otherwise be dropped. + +> **Note:** Rewinding re-reads from the failed offset, so messages on the same partition that succeeded *after* the failed one are re-read and re-written. Enable [deduplication](#deduplication-parameters) (or use messages carrying their own timestamp) to avoid duplicate points from this at-least-once retry. + +### Security parameters + +In TOML configuration, this parameter is placed under the `[kafka]` section. + +| Parameter | Type | Default | Description | +|---------------------|--------|-------------|--------------------------------------------------------------------------| +| `security_protocol` | string | "PLAINTEXT" | Security protocol: "PLAINTEXT", "SSL", "SASL_PLAINTEXT", "SASL_SSL" | + +### SASL Authentication parameters + +In TOML configuration, these parameters are placed under the `[kafka.sasl]` section with shortened names. + +| Parameter (CLI) | Parameter (TOML) | Type | Default | Description | +|------------------|------------------------------|--------|---------|----------------------------------------------------------------------| +| `sasl_mechanism` | `[kafka.sasl]` `mechanism` | string | none | SASL mechanism: "PLAIN", "SCRAM-SHA-256", "SCRAM-SHA-512" | +| `sasl_username` | `[kafka.sasl]` `username` | string | none | SASL username (required with sasl_mechanism) | +| `sasl_password` | `[kafka.sasl]` `password` | string | none | SASL password (required with sasl_mechanism) | + +**Note:** All three SASL parameters must be provided together when using SASL authentication. + +### SSL/TLS parameters + +In TOML configuration, these parameters are placed under the `[kafka.ssl]` section with shortened names. + +| Parameter (CLI) | Parameter (TOML) | Type | Default | Description | +|--------------------|--------------------------------|--------|---------|------------------------------------------------| +| `ssl_ca_cert` | `[kafka.ssl]` `ca_cert` | string | none | Path to CA certificate file | +| `ssl_cert` | `[kafka.ssl]` `client_cert` | string | none | Path to client certificate for mutual TLS | +| `ssl_key` | `[kafka.ssl]` `client_key` | string | none | Path to client private key for mutual TLS | +| `ssl_key_password` | `[kafka.ssl]` `key_password` | string | none | Password for encrypted client private key | + +**Note:** For mutual TLS, both `ssl_cert` and `ssl_key` must be provided together. + +### Logging parameters + +In TOML configuration, `enable_full_logging` is placed directly under the `[kafka]` section. + +| Parameter | Type | Default | Description | +|-----------------------|---------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `enable_full_logging` | boolean | false | When `true`, full exception messages are written to logs. When `false` (default), only the exception type is logged, to avoid leaking sensitive values (credentials, payloads, paths) into log output. Enable temporarily for debugging. | + +### Dead-letter queue parameters + +In TOML configuration, `dlq_topic` is placed under the `[kafka]` section. + +| Parameter | Type | Default | Description | +|-------------|--------|---------|--------------------------------------------------------------------------------------------------------------------------------------------------------| +| `dlq_topic` | string | none | Kafka topic to publish messages that fail to parse. The original payload and key are produced unchanged, with `source_topic`, `error_type` and `error_message` set as Kafka headers. When unset, no DLQ topic is used. | + +**Behavior:** + +- Only **parse failures** (poison messages) are routed to the DLQ topic — including messages missing a required `dedup_id_field`. Transient write failures to InfluxDB are **not** sent to the DLQ; they keep their offsets uncommitted (with `on_success`) so they are retried on the next cycle. +- Failures continue to be recorded in the `kafka_exceptions` table regardless of `dlq_topic`. +- The DLQ producer reuses the same `bootstrap_servers` and security settings as the consumer. +- **The DLQ topic must already exist on the broker.** The plugin does not create it. If your broker has `auto.create.topics.enable=false` (typical for production and managed services such as Confluent Cloud or MSK), create the topic beforehand. When the topic is missing, delivery fails and — under `on_success` — the offset is not committed, so the batch is reprocessed until the topic exists. The real broker reason (for example `Unknown topic or partition`) is logged so you can fix it quickly. + +### Deduplication parameters + +In TOML configuration, `dedup_window` is placed under the `[kafka]` section; `dedup_id_field` and `dedup_id_name` are placed under `[mapping.json]` or `[mapping.text]` section. + +| Parameter | Type | Default | Description | +|------------------|--------|------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `dedup_id_field` | string | none | Extractor for a unique message id (enables deduplication). Holds **only** the extractor: JSON — a field name in CLI args (extracted via `$.`) or a JSONPath in TOML (e.g. `$.event_id`); Text — a regex pattern with a capturing group. | +| `dedup_id_name` | string | "dedup_id" | Field name the extracted id is written under and tracked by. Optional; only used with `dedup_id_field`. | +| `dedup_window` | string | "24h" | Lookback window for the duplicate check (e.g. `30m`, `24h`, `7d`). Only used with `dedup_id_field`. | + +**Behavior:** + +- Deduplication is **only active when no `timestamp_field` is configured**. When messages carry their own timestamp, InfluxDB already overwrites duplicates by `(measurement, tags, time)`, so no extra check is needed. +- When active, the plugin extracts the id, writes it under the `dedup_id_name` field (default `dedup_id`), and — before writing — checks whether that id already exists within `dedup_window`. The check is a **single query per table per cycle**, constrained by both the time window and the batch's ids (`... WHERE time >= AND "" IN (...)`), so it returns few rows. Duplicates within the same batch are also dropped. +- If the dedup query fails, the plugin **fails open** (writes the records) rather than dropping data. +- A message that is missing the configured id is treated as a parse failure (routed to the DLQ / `kafka_exceptions`). For JSON arrays, individual elements missing the id are skipped with a warning. +- **Performance:** for very high-volume topics prefer a shorter `dedup_window` to keep the query light. +- Pick a `dedup_id_name` that does not collide with a field you already map — it would be written twice. + +### Message format parameters + +In TOML configuration, `format` is placed under the `[kafka]` section; `timestamp_field` is placed under `[mapping.json]` or `[mapping.text]` section. + +| Parameter | Type | Default | Description | +|-------------------|--------|---------|----------------------------------------------------------------------| +| `format` | string | "json" | Message format: json, lineprotocol, text, avro, jsonschema, protobuf | +| `timestamp_field` | string | none | Field containing timestamp (format depends on message format) | + +**Supported formats:** + +| Format | Description | +|----------------|------------------------------------------------------------------------| +| `json` | JSON with JSONPath field mapping | +| `lineprotocol` | InfluxDB Line Protocol passthrough | +| `text` | Plain text with regex-based parsing | +| `avro` | Avro via Confluent Schema Registry (JSONPath mapping) | +| `jsonschema` | JSON Schema via Confluent Schema Registry (JSONPath) | +| `protobuf` | Protobuf (Confluent wire format) via Schema Registry or a local .proto | + +**Format-specific timestamp_field syntax:** + +| Format | Syntax | Split Method | Example (CLI) | Example (TOML) | +|----------|---------------------|--------------------|------------------|--------------------| +| JSON | `field_name:format` | Split by first `:` | `"timestamp:ms"` | `"$.timestamp:ms"` | +| Text | `regex:format` | Split by last `:` | `"ts:(\\d+):ms"` | `"ts:(\\d+):ms"` | + +**Supported timestamp formats:** +- `ns` - nanoseconds (Unix timestamp) +- `ms` - milliseconds (Unix timestamp) +- `s` - seconds (Unix timestamp) +- `datetime` - ISO 8601 string (for example, "2021-12-01T12:00:00Z") + +For `avro` and `jsonschema`, a field decoded as a native `datetime`/`date` +(Avro/JSON Schema logical type) is converted to a timestamp automatically, +regardless of the configured format specifier. + +### JSON format parameters + +In TOML configuration, `tags` are placed under `[mapping.json.tags]` section and `fields` under `[mapping.json.fields]` section. + +| Parameter | Type | Default | Description | +|-----------|--------|----------|---------------------------------------------------------------------------| +| `tags` | string | none | Space-separated tag names. Example: "room sensor location" | +| `fields` | string | required | Space-separated field mappings. Format: "name:type=jsonpath" without `$.` | + +**Field specification format:** `"temp:float=temperature hum:int=humidity status:bool=online"` + +**Supported field types:** `int`, `uint`, `float`, `string`, `bool` + +### Text format parameters + +In TOML configuration, `tags` are placed under `[mapping.text.tags]` section and `fields` under `[mapping.text.fields]` section. + +| Parameter | Type | Default | Description | +|-----------|--------|-----------|-------------------------------------------------------------------| +| `tags` | string | none | Space-separated tag patterns. Format: "name=regex_pattern" | +| `fields` | string | required | Space-separated field patterns. Format: "name:type=regex_pattern" | + +### Binary formats + +The decoded record is a dict (or a top-level list, expanded into one record per +element) for all binary formats, so field/tag/timestamp mapping uses the same +JSONPath syntax as the JSON format, under `[mapping.avro]`, +`[mapping.jsonschema]`, or `[mapping.protobuf]`. + +Per-message decode failures (schema id not found, malformed payload) are handled +like parse failures: the message is logged to `kafka_exceptions`, sent to +`dlq_topic` if configured, or replayed otherwise. Binary payloads are forwarded +to the DLQ as the original raw bytes. + +An unreachable or misconfigured Schema Registry is caught when the decoder is +first built: the cycle is aborted before any message is polled or committed, so +nothing is lost, and the run retries on the next schedule. + +#### Avro / JSON Schema (Schema Registry) + +Set `format` to `avro` or `jsonschema` to consume binary messages in the +Confluent wire format (magic byte + 4-byte schema id + payload). The schema is +fetched from the Schema Registry by the id embedded in each message — no local +schema files are needed. + +The parameter names are identical in CLI/API arguments and TOML. In TOML they +are placed under the `[kafka.schema_registry]` section. + +| Parameter | Type | Default | Description | +|----------------------------------------|--------|----------|-------------------------------------------------------------------------------------------------------------| +| `schema_registry_url` | string | required | Schema Registry URL. Required for `avro` / `jsonschema`, and for `protobuf` without `protobuf_schema_file`. | +| `schema_registry_username` | string | none | Basic-auth username (or API key for Confluent Cloud). | +| `schema_registry_password` | string | none | Basic-auth password (or API secret). Used with username. | +| `schema_registry_ca_cert` | string | none | CA certificate for TLS to the registry. | +| `schema_registry_client_cert` | string | none | Client certificate for mutual TLS to the registry. | +| `schema_registry_client_key` | string | none | Client private key for mutual TLS. Requires client cert. | +| `schema_registry_client_key_password` | string | none | Password for an encrypted client private key. | + +> **Note:** `avro` and `jsonschema` require the extra packages `httpx`, +> `fastavro`, and `jsonschema`. + +#### Avro (local .avsc schema, schemaless) + +Set `format` to `avro` and `avro_schema_file` to a local `.avsc` schema to +consume Avro messages produced **without** a Schema Registry — raw schemaless +encoding with no Confluent wire header. The whole message value is the Avro body +and is decoded with the local schema; the Schema Registry is **not** contacted. + +The parameter name is identical in CLI/API arguments and TOML. In TOML it is +placed under the `[kafka.avro]` section. + +| Parameter | Type | Default | Description | +|--------------------|--------|----------|------------------------------------------------------------------------------| +| `avro_schema_file` | string | required | Path to a local `.avsc`/`.json` schema (absolute or relative to PLUGIN_DIR). | + +> **Note:** requires the `fastavro` package. The schema is parsed once and +> cached. Writer and reader schema are assumed identical (no schema evolution). + +#### Protobuf (local .proto or Schema Registry) + +Set `format` to `protobuf` to consume Protobuf messages in the Confluent wire +format. The schema source is chosen implicitly, like `avro`: + +- **Local `.proto`** — set `protobuf_schema_file`. The wire framing is parsed + locally and the schema is read from the file; the Schema Registry is not + contacted. +- **Schema Registry** — omit `protobuf_schema_file` and set `schema_registry_url`. + The `.proto` (and any referenced schemas) is fetched by the wire-frame schema + id, compiled with `protoc`, and cached per schema id. The `[kafka.schema_registry]` + section (url, auth, TLS) is shared with `avro`/`jsonschema`. + +The message index selects which message type to decode, or set +`protobuf_message_type` to choose it explicitly. In registry mode the schema +matches the producer's, so the message index is always correct and +`protobuf_message_type` is usually unnecessary. + +The parameter names are identical in CLI/API arguments and TOML. In TOML they +are placed under the `[kafka.protobuf]` section. + +| Parameter | Type | Default | Description | +|--------------------------|--------|--------------------|----------------------------------------------------------------------------------------| +| `protobuf_schema_file` | string | conditional | Path to a local `.proto` (absolute or relative to PLUGIN_DIR). Omit to use the registry. | +| `protobuf_include_dir` | string | schema file's dir | Include directory for resolving `import` statements (local mode only). | +| `protobuf_message_type` | string | wire message index | Fully-qualified message name to decode (e.g. `sensors.SensorReading`). | + +> **Note:** `protobuf` requires `grpcio-tools` (bundles `protoc` to compile the +> `.proto`); it is included in the plugin's dependencies and installed +> automatically. The `.proto` is +> compiled once and cached. `MessageToDict` semantics apply: proto3 scalar fields +> equal to their default (`0`/`""`/`false`) are omitted from the record; +> `int64`/`uint64` become strings; enums become their names; `bytes` become +> base64 strings. + +See examples 11–14 in [kafka_config_example.toml](https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/kafka_subscriber/kafka_config_example.toml). + +### TOML configuration + +| Parameter | Type | Default | Description | +|--------------------|--------|---------|-------------------------------------------------| +| `config_file_path` | string | none | Path to TOML config file (absolute or relative) | + +### File path resolution + +All file paths in the plugin (configuration file, TLS certificates) follow the same resolution logic: + +- **Absolute paths** (for example, `/etc/kafka/config.toml`) are used as-is +- **Relative paths** (for example, `config.toml`, `certs/ca.crt`) are resolved against the plugin directory, taken from `PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent of `VIRTUAL_ENV` + +If a relative path is specified and none of these can be resolved, the plugin will return an error. + +#### Example TOML configuration + +[kafka_config_example.toml](https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/kafka_subscriber/kafka_config_example.toml) - comprehensive configuration example with all formats and security options + +## Data requirements + +The plugin automatically creates the target measurement table on first write. Field mappings are required for JSON and Text formats to specify which fields to extract and their data types. + +### Message encoding requirements + +- **Text formats** (`json`, `text`, `lineprotocol`): payloads must be UTF-8 encoded. +- **Binary formats** (`avro`, `jsonschema`, `protobuf`): payloads are decoded from binary (see [Binary formats](#binary-formats)). +- **Non-empty payloads**: Empty or whitespace-only messages are automatically skipped. + +## Software Requirements + +- **{{% product-name %}}**: with the Processing Engine enabled +- **Python packages** (all bundled in `manifest.toml` and installed automatically with the plugin): + - `confluent-kafka>=2.15.0` (Kafka client based on librdkafka) — all formats + - `jsonpath-ng` (JSONPath field mapping) — all formats + - `fastavro` — `avro` format (both Schema Registry and local `.avsc`) + - `httpx`, `authlib`, `cachetools` — any Schema Registry format (`avro` / `jsonschema` / registry-mode `protobuf`) + - `jsonschema` (library) — `jsonschema` message format only + - `protobuf`, `grpcio-tools` — `protobuf` format only + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install required Python packages: + + ```bash + influxdb3 install package confluent-kafka + influxdb3 install package jsonpath-ng + ``` +## Trigger setup + +### Scheduled ingestion with TOML configuration + +Recommended for production use with complex mappings: + +```bash +# 1. Set PLUGIN_DIR environment variable +export PLUGIN_DIR=~/.plugins + +# 2. Copy and edit configuration file +cp kafka_config_example.toml $PLUGIN_DIR/my_kafka_config.toml +# Edit my_kafka_config.toml with your Kafka cluster and mapping settings + +# 3. Create the trigger +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/kafka_subscriber/kafka_subscriber.py \ + --trigger-spec "every:10s" \ + --trigger-arguments config_file_path=my_kafka_config.toml \ + kafka_ingestion +``` +### Scheduled ingestion with command-line arguments + +For simple JSON message ingestion: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/kafka_subscriber/kafka_subscriber.py \ + --trigger-spec "every:5s" \ + --trigger-arguments 'bootstrap_servers=kafka1:9092 kafka2:9092,topics=sensors.temperature sensors.humidity,group_id=influxdb3_consumer,format=json,table_name=sensor_data,fields=temp:float=temperature hum:int=humidity,tags=location sensor_id' \ + kafka_sensors +``` +### Secure Kafka connection with SASL_SSL + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/kafka_subscriber/kafka_subscriber.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'bootstrap_servers=kafka1:9093 kafka2:9093,topics=secure.data,group_id=influxdb3_secure,format=json,table_name=secure_data,security_protocol=SASL_SSL,sasl_mechanism=SCRAM-SHA-512,sasl_username=myuser,sasl_password=mypass,ssl_ca_cert=certs/ca.crt,fields=value:float=value' \ + secure_kafka +``` +## Message Formats + +### JSON Format + +The primary use case for structured data. Supports nested fields using JSONPath expressions. + +#### TOML Configuration + +```toml +[kafka] +bootstrap_servers = ["kafka:9092"] +topics = ["sensors.temperature"] +group_id = "influxdb3_json" +format = "json" + +[mapping.json] +table_name = "sensor_data" +timestamp_field = "$.timestamp:ms" + +[mapping.json.tags] +location = "$.location" +sensor_id = "$.sensor.id" + +[mapping.json.fields] +temperature = ["$.temp", "float"] +humidity = ["$.humidity", "int"] +status = ["$.online", "bool"] +``` +#### Example Message + +```json +{ + "timestamp": 1638360000000, + "location": "warehouse_a", + "sensor": { + "id": "sensor_001" + }, + "temp": 22.5, + "humidity": 65, + "online": true +} +``` +#### Resulting Data + +``` +sensor_data,location=warehouse_a,sensor_id=sensor_001 temperature=22.5,humidity=65i,status=true 1638360000000000000 +``` +#### JSON Array Support + +Process batch messages containing arrays of JSON objects: + +```json +[ + {"timestamp": 1638360000000, "sensor_id": "001", "temperature": 22.5}, + {"timestamp": 1638360001000, "sensor_id": "002", "temperature": 23.1}, + {"timestamp": 1638360002000, "sensor_id": "003", "temperature": 21.8} +] +``` +**Array processing behavior:** +- Each array element is processed independently as a separate data point +- If one element fails to parse, the others continue processing (partial success) +- Parse errors for individual elements are logged as warnings and the element is skipped (not written to `kafka_exceptions`) +- Statistics count 1 Kafka message = 1 unit (regardless of array size) + +### Line Protocol Format + +For messages already in InfluxDB line protocol format, use passthrough mode. No mapping configuration needed. + +#### TOML Configuration + +```toml +[kafka] +bootstrap_servers = ["kafka:9092"] +topics = ["influxdb.metrics"] +group_id = "influxdb3_lineprotocol" +format = "lineprotocol" +``` +#### Example Message + +``` +sensor_data,location=warehouse_a,sensor_id=001 temperature=22.5,humidity=65i 1638360000000000000 +``` +### Text Format + +Parse plain text messages using regular expressions: + +#### TOML Configuration + +```toml +[kafka] +bootstrap_servers = ["kafka:9092"] +topics = ["legacy.logs"] +group_id = "influxdb3_text" +format = "text" + +[mapping.text] +table_name = "sensor_logs" +timestamp_field = "ts:(\\d+):ms" + +[mapping.text.tags] +location = "location=([^,\\s]+)" + +[mapping.text.fields] +temperature = ["temp:([\\d.]+)", "float"] +humidity = ["hum:(\\d+)", "int"] +``` +#### Example Message + +``` +location=warehouse_a,temp:22.5,hum:65,ts:1638360000000 +``` +### Avro Format (Schema Registry) + +Decode Avro messages in the Confluent wire format. The schema is resolved from +the Schema Registry by the id embedded in each message; mapping uses JSONPath +on the decoded record. + +#### TOML Configuration + +```toml +[kafka] +bootstrap_servers = ["kafka:9092"] +topics = ["sensors.avro"] +group_id = "influxdb3_avro" +format = "avro" + +[kafka.schema_registry] +schema_registry_url = "http://schema-registry:8081" +# schema_registry_username = "sr_api_key" # optional basic auth +# schema_registry_password = "sr_api_secret" + +[mapping.avro] +table_name = "sensor_data" +timestamp_field = "$.timestamp:ms" + +[mapping.avro.tags] +device_id = "$.device_id" + +[mapping.avro.fields] +temperature = ["$.temperature", "float"] +humidity = ["$.humidity", "float"] +``` +### Avro Format (local .avsc schema, schemaless) + +Decode raw schemaless Avro (no Schema Registry, no wire header) with a local +`.avsc` schema. Set `avro_schema_file` to switch `format = "avro"` to this mode. + +```toml +[kafka] +bootstrap_servers = ["kafka:9092"] +topics = ["sensors.avro"] +group_id = "influxdb3_avro_file" +format = "avro" + +[kafka.avro] +avro_schema_file = "schemas/sensor.avsc" + +[mapping.avro] +table_name = "sensor_data" +timestamp_field = "$.timestamp:ms" + +[mapping.avro.tags] +device_id = "$.device_id" + +[mapping.avro.fields] +temperature = ["$.temperature", "float"] +humidity = ["$.humidity", "float"] +``` +### JSON Schema Format (Schema Registry) + +Decode JSON-Schema messages in the Confluent wire format. Configuration mirrors +the Avro example, using `format = "jsonschema"` and `[mapping.jsonschema]`. + +```toml +[kafka] +bootstrap_servers = ["kafka:9092"] +topics = ["events.jsonschema"] +group_id = "influxdb3_jsonschema" +format = "jsonschema" + +[kafka.schema_registry] +schema_registry_url = "http://schema-registry:8081" + +[mapping.jsonschema] +table_name = "events" +timestamp_field = "$.ts:ms" + +[mapping.jsonschema.fields] +value = ["$.value", "float"] +``` +### Protobuf Format (local .proto schema) + +Decode Protobuf messages in the Confluent wire format using a local `.proto` +file. The Schema Registry is not contacted; the message type is selected by the +wire message index, or set `protobuf_message_type` to choose it explicitly. + +```toml +[kafka] +bootstrap_servers = ["kafka:9092"] +topics = ["sensors.protobuf"] +group_id = "influxdb3_protobuf" +format = "protobuf" + +[kafka.protobuf] +protobuf_schema_file = "schemas/sensor.proto" +# protobuf_include_dir = "schemas" # optional, for .proto imports +# protobuf_message_type = "sensors.SensorReading" # optional, defaults to wire message index + +[mapping.protobuf] +table_name = "sensor_data" +timestamp_field = "$.timestamp:ms" + +[mapping.protobuf.tags] +device_id = "$.device_id" + +[mapping.protobuf.fields] +temperature = ["$.temperature", "float"] +humidity = ["$.humidity", "float"] +``` +### Protobuf Format (Schema Registry) + +Omit `protobuf_schema_file` and set `schema_registry_url` to fetch and compile +the `.proto` (and any referenced schemas) by the wire-frame schema id. The +`[kafka.protobuf]` section can be omitted entirely. + +```toml +[kafka] +bootstrap_servers = ["kafka:9092"] +topics = ["sensors.protobuf"] +group_id = "influxdb3_protobuf_sr" +format = "protobuf" + +[kafka.schema_registry] +schema_registry_url = "http://schema-registry:8081" + +[mapping.protobuf] +table_name = "sensor_data" +timestamp_field = "$.timestamp:ms" + +[mapping.protobuf.tags] +device_id = "$.device_id" + +[mapping.protobuf.fields] +temperature = ["$.temperature", "float"] +humidity = ["$.humidity", "float"] +``` +## Statistics and Monitoring + +The plugin tracks comprehensive statistics and writes them to the `kafka_stats` table on every plugin invocation. + +### kafka_stats Table + +| Field | Type | Description | +|-----------------------------|-------|-------------------------------------------| +| `topic` (tag) | tag | Kafka topic name | +| `partition` (tag) | tag | Partition number | +| `consumer_group` (tag) | tag | Consumer group ID | +| `bootstrap_servers` (tag) | tag | Kafka cluster address | +| `messages_received` | int | Total messages received | +| `messages_processed` | int | Successfully processed messages | +| `messages_failed` | int | Failed messages | +| `last_offset` | int | Last processed offset for this partition | +| `success_rate` | float | Percentage of successful messages | + +### Querying Statistics + +```bash +# Get latest statistics +influxdb3 query --database mydb \ + "SELECT * FROM kafka_stats ORDER BY time DESC LIMIT 10" + +# Success rate by topic +influxdb3 query --database mydb \ + "SELECT topic, partition, success_rate, messages_processed, messages_failed + FROM kafka_stats + WHERE time > now() - INTERVAL '1 hour' + ORDER BY time DESC" + +# Check consumer lag indicators +influxdb3 query --database mydb \ + "SELECT topic, partition, last_offset + FROM kafka_stats + WHERE consumer_group = 'influxdb3_consumer' + ORDER BY time DESC LIMIT 10" +``` +## Error Handling + +Parse errors and message processing failures are logged to the `kafka_exceptions` table: + +### kafka_exceptions Table + +| Field | Type | Description | +|---------------------|--------|------------------------------------------| +| `topic` (tag) | tag | Kafka topic where error occurred | +| `partition` (tag) | tag | Partition number | +| `error_type` (tag) | tag | Type of error (for example, JSONDecodeError) | +| `offset` | int | Message offset | +| `error_message` | string | Detailed error message | + +### Checking for Errors + +```bash +influxdb3 query --database mydb \ + "SELECT * FROM kafka_exceptions ORDER BY time DESC LIMIT 10" +``` +### Dead-letter queue + +When `dlq_topic` is configured, messages that fail to parse are additionally republished to that Kafka topic with the original payload and key, so they can be inspected or replayed without re-reading the source topic. Error context is attached as Kafka headers: + +| Header | Description | +|-----------------|----------------------------------------------| +| `source_topic` | Topic the message was originally consumed from | +| `error_type` | Exception type (e.g. `JSONDecodeError`) | +| `error_message` | Error detail (truncated to 1KB) | + +Transient InfluxDB write failures are not sent to the DLQ — they are retried on the next cycle (with `offset_commit_policy=on_success`). + +## Troubleshooting + +### Check Plugin Logs + +```bash +influxdb3 query --database _internal \ + "SELECT * FROM system.processing_engine_logs + WHERE trigger_name = 'kafka_ingestion' + ORDER BY time DESC LIMIT 20" +``` +### Common Issues + +#### "confluent-kafka library not installed" or "No module named 'confluent_kafka'" + +```bash +influxdb3 install package confluent-kafka +``` +If you encounter `librdkafka` related errors: +- Ubuntu/Debian: `sudo apt-get install librdkafka-dev` +- macOS: `brew install librdkafka` + +#### "Configuration file not found" + +- For relative paths, ensure `PLUGIN_DIR` environment variable is set +- For absolute paths, verify the file exists at the specified location + +```bash +# For relative paths +export PLUGIN_DIR=~/.plugins +ls $PLUGIN_DIR/my_kafka_config.toml + +# Or use absolute path +ls /etc/kafka/my_kafka_config.toml +``` +#### "Failed to connect to Kafka cluster" + +- Verify bootstrap_servers addresses and ports +- Check network connectivity to Kafka brokers +- For SSL connections, verify certificate paths +- For SASL authentication, verify credentials + +#### "SASL mechanism required when security_protocol includes SASL" + +When using `SASL_PLAINTEXT` or `SASL_SSL`, you must provide: +- `sasl_mechanism` +- `sasl_username` +- `sasl_password` + +#### "No fields were mapped from JSON data" + +- Verify JSONPath expressions in field mappings (use `$.` prefix) +- Check that JSON structure matches your paths +- Review `kafka_exceptions` table for detailed errors + +#### Messages not being processed + +- Check trigger status: `influxdb3 show summary --database mydb` +- Verify Kafka connection in plugin logs +- Check consumer group lag using Kafka tools +- Increase trigger frequency (for example, from `every:10s` to `every:5s`) +- If many messages are queued, increase `max_poll_records` or set to `0` (unlimited) + +#### Offset commit issues with `on_success` policy + +If using `offset_commit_policy=on_success` and seeing repeated message processing: +- Check `kafka_exceptions` table for processing errors +- Fix the root cause of failures +- Consider using `offset_commit_policy=always` if some data loss is acceptable + +## Architecture + +### How It Works + +1. **Scheduled Trigger**: Plugin runs on schedule (for example, `every:10s`) +2. **Configuration Caching**: Plugin configuration is parsed once and cached between executions +3. **Persistent Consumer**: The Kafka consumer is created once and reused across invocations (cached), so it stays in its consumer group and no rebalance happens every cycle +4. **Consumer Poll**: Consumer polls for available messages (drains all available messages) +5. **Offset Commit**: Based on `offset_commit_policy`: + - `always`: Commits immediately after poll + - `on_success`: Commits only after successful processing of all messages +6. **Deduplication** (optional): When `dedup_id_field` is set and no message timestamp is configured, duplicate ids within `dedup_window` are skipped +7. **Parse & Write**: Messages parsed according to format and written to InfluxDB +8. **Error Tracking**: Parse errors logged to `kafka_exceptions` table (and `dlq_topic` if configured) +9. **Statistics**: Written to `kafka_stats` table on every plugin invocation + +### Performance Optimization + +The plugin includes several optimizations for high-throughput scenarios: + +- **Persistent Consumer**: The consumer is reused across trigger executions instead of reconnecting each cycle, eliminating the consumer-group rebalance (and its latency/broker overhead) that a connect/disconnect-per-cycle would cause. It stays in the group while the schedule interval stays below `max_poll_interval`. +- **Configuration Caching**: Plugin configuration is parsed once and reused across all trigger executions +- **Pre-compiled Patterns**: JSONPath expressions and regex patterns are compiled once during parser initialization +- **Batch Polling**: Consumer drains available messages in a single trigger execution (limited by `max_poll_records`, default 500) +- **Manual Offset Commit**: Provides control over exactly-once vs at-least-once semantics + +### Consumer Group Behavior + +- A single consumer connection is created and reused across trigger executions; it is only rebuilt after a fatal connection/poll error or a configuration change +- Set `max_poll_interval` above your trigger interval so the broker does not evict the reused consumer between polls (this would force a rebalance) +- Consumer group ID ensures partitions are assigned consistently +- Offsets are committed to Kafka for durability +- Multiple plugin instances with the same `group_id` will share partitions (load balancing) + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/mad-check.md b/content/shared/influxdb3-plugins/plugins-library/official/mad-check.md index 6bdff13777..8e0f3b093e 100644 --- a/content/shared/influxdb3-plugins/plugins-library/official/mad-check.md +++ b/content/shared/influxdb3-plugins/plugins-library/official/mad-check.md @@ -1,4 +1,5 @@ - + + The MAD-Based Anomaly Detection Plugin provides real-time anomaly detection for time series data in {{% product-name %}} using Median Absolute Deviation (MAD). Detect outliers in your field values as data is written, with configurable thresholds for both count-based and duration-based alerts. The plugin maintains in-memory deques for efficient computation and integrates with the Notification Sender Plugin to deliver alerts via multiple channels. ## Configuration @@ -21,25 +22,29 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor ### MAD threshold parameters -| Component | Description | Example | -|----------------|------------------------------------------------|-------------| -| `field_name` | The numeric field to monitor | `temp` | -| `k` | MAD multiplier for anomaly threshold | `2.5` | -| `window_count` | Number of recent points for MAD computation | `20` | -| `threshold` | Count (integer) or duration (for example, "2m", "1h") | `5` or `2m` | +| Component | Description | Example | +|----------------|----------------------------------------------------------------|---------------| +| `field_name` | The numeric field to monitor | `temp` | +| `k` | MAD multiplier for the anomaly cutoff (float, ≥ 0) | `2.5` | +| `window_count` | Number of recent points for MAD computation (integer, 2–10000) | `20` | +| `threshold` | Consecutive outliers (integer, ≥ 1) or a duration | `5` or `2min` | + +Multiple thresholds are separated by `@`: `temp:2.5:20:5@load:3:10:2min` + +Durations use the format ``, where unit is `us` (microseconds), `ms` (milliseconds), `s` (seconds), `min` (minutes), `h` (hours), `d` (days), or `w` (weeks). -Multiple thresholds are separated by `@`: `temp:2.5:20:5@load:3:10:2m` +Thresholds that share a field and `window_count` share one MAD window, so you can combine a count-based and a duration-based alert on the same detector: `temp:2.5:20:5@temp:2.5:20:2min`. Invalid thresholds are skipped with a warning; if none remain, the plugin logs an error and stops. Repeated identical thresholds are also skipped with a warning, because they would share one counter. ### Optional parameters -| Parameter | Type | Default | Description | -|---------------------------|--------|--------------------------------------|-------------------------------------------------------------------------------------------| -| `influxdb3_auth_token` | string | env var | API token for {{% product-name %}} (or use INFLUXDB3_AUTH_TOKEN env var) | -| `state_change_count` | string | "0" | Maximum allowed value flips before suppressing notifications | -| `notification_count_text` | string | see *Default notification templates* | Template for count-based alerts with variables: $table, $field, $threshold_count, $tags | -| `notification_time_text` | string | see *Default notification templates* | Template for duration-based alerts with variables: $table, $field, $threshold_time, $tags | -| `notification_path` | string | "notify" | URL path for the notification sending plugin | -| `port_override` | string | "8181" | Port number where InfluxDB accepts requests | +| Parameter | Type | Default | Description | +|---------------------------|--------|--------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `influxdb3_auth_token` | string | env var | API token for {{% product-name %}} (or use INFLUXDB3_AUTH_TOKEN env var) | +| `state_change_count` | string | "0" | Number of transitions between normal and outlier state, within the MAD window, at which notifications are suppressed. Use 2 or greater; `1` is treated as `0`. See *Flip Detection* | +| `notification_count_text` | string | see *Default notification templates* | Template for count-based alerts with variables: $table, $field, $threshold_count, $tags | +| `notification_time_text` | string | see *Default notification templates* | Template for duration-based alerts with variables: $table, $field, $threshold_time, $tags | +| `notification_path` | string | "notify" | URL path for the notification sending plugin | +| `port_override` | string | "8181" | Port number where InfluxDB accepts requests | #### Default notification templates @@ -84,7 +89,11 @@ Multiple thresholds are separated by `@`: `temp:2.5:20:5@load:3:10:2m` |--------------------|--------|---------|----------------------------------------------------------------------------------| | `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (required for TOML configuration) | -*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. Relative paths are resolved against the first directory that is set: `PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent of `VIRTUAL_ENV`. Only that directory is used — the file is not looked up in the remaining ones. + +When `config_file_path` is set, the TOML file provides the whole configuration and inline trigger arguments are ignored. `INFLUXDB3_AUTH_TOKEN` from the environment still applies when `influxdb3_auth_token` is not set in the file. In TOML, `senders` and `mad_thresholds` use native structures (a list and a list of entries) instead of the inline string formats, though the inline strings are also accepted. + +The plugin caches the loaded configuration for 10 minutes to keep the write path fast, so configuration changes take effect within that window. #### Example TOML configuration @@ -96,6 +105,7 @@ For more information on using TOML configuration files, see the Using TOML Confi - **{{% product-name %}}**: with the Processing Engine enabled. - **Python packages**: + - `influxdata-plugin-utils>=0.3.0` (configuration loading, parsing, and schema introspection) - `requests` (for notification delivery) - **Notification Sender Plugin** *(optional)*: Required if using the `senders` parameter. See the [influxdata/notifier plugin](/influxdb3/version/plugins/library/official/notifier/). @@ -113,6 +123,7 @@ For more information on using TOML configuration files, see the Using TOML Confi 2. Install required Python packages: ```bash + influxdb3 install package influxdata-plugin-utils influxdb3 install package requests ``` 3. *(Optional)* For notifications, install the [influxdata/notifier plugin](/influxdb3/version/plugins/library/official/notifier/) and create an HTTP trigger for it. @@ -132,7 +143,7 @@ influxdb3 create trigger \ --database mydb \ --path "gh:influxdata/mad_check/mad_check_plugin.py" \ --trigger-spec "all_tables" \ - --trigger-arguments 'measurement=cpu,mad_thresholds="temp:2.5:20:5@load:3:10:2m",senders=slack,slack_webhook_url="$SLACK_WEBHOOK_URL"' \ + --trigger-arguments 'measurement=cpu,mad_thresholds="temp:2.5:20:5@load:3:10:2min",senders=slack,slack_webhook_url="$SLACK_WEBHOOK_URL"' \ mad_anomaly_detector ``` Set `SLACK_WEBHOOK_URL` to your Slack incoming webhook URL. @@ -183,7 +194,7 @@ influxdb3 create trigger \ --database monitoring \ --path "gh:influxdata/mad_check/mad_check_plugin.py" \ --trigger-spec "all_tables" \ - --trigger-arguments 'measurement=system_metrics,mad_thresholds="cpu_load:3:30:2m@memory_used:2.5:30:5m",senders=slack.discord,slack_webhook_url="$SLACK_WEBHOOK_URL",discord_webhook_url="$DISCORD_WEBHOOK_URL"' \ + --trigger-arguments 'measurement=system_metrics,mad_thresholds="cpu_load:3:30:2min@memory_used:2.5:30:5min",senders=slack.discord,slack_webhook_url="$SLACK_WEBHOOK_URL",discord_webhook_url="$DISCORD_WEBHOOK_URL"' \ system_anomaly_detector ``` Set `SLACK_WEBHOOK_URL` and `DISCORD_WEBHOOK_URL` to your webhook URLs. @@ -191,8 +202,8 @@ Set `SLACK_WEBHOOK_URL` and `DISCORD_WEBHOOK_URL` to your webhook URLs. **Expected output** - Monitors two fields independently: - - `cpu_load`: Alerts when exceeds 3 MADs for 2 minutes - - `memory_used`: Alerts when exceeds 2.5 MADs for 5 minutes + - `cpu_load`: Alerts when exceeds 3 MADs for 2 minutes + - `memory_used`: Alerts when exceeds 2.5 MADs for 5 minutes - Sends notifications to both Slack and Discord ### Example 3: Anomaly detection with flip suppression @@ -213,7 +224,7 @@ Set `HTTP_WEBHOOK_URL` to your HTTP webhook endpoint. **Expected output** - Detects vibration anomalies exceeding 2 MADs for 10 consecutive points -- If values flip between normal/anomalous more than 3 times in the 50-point window, suppresses notifications +- Suppresses notifications once the value has switched between normal and outlier state 3 times within the 50-point window, so two switches are still tolerated - Sends custom formatted message to HTTP endpoint ## Using TOML Configuration Files @@ -245,7 +256,7 @@ This plugin supports using TOML configuration files to specify all plugin argume ```toml # Required parameters measurement = "cpu" - mad_thresholds = "temp:2.5:20:5@load:3:10:2m" + mad_thresholds = "temp:2.5:20:5@load:3:10:2min" senders = "slack" # Notification settings @@ -270,6 +281,9 @@ This plugin supports using TOML configuration files to specify all plugin argume - `mad_check_plugin.py`: The main plugin code containing the handler for data write triggers - `mad_anomaly_config_data_writes.toml`: Example TOML configuration file +- `test_mad_check.py`: Pytest suite, runs without a live {{% product-name %}} server +- `requirements.txt`: Runtime dependencies (`influxdata-plugin-utils>=0.3.0`, `requests`) +- `requirements-dev.txt`: Development dependencies (`pytest`) ### Logging @@ -309,9 +323,15 @@ mad = statistics.median([abs(x - median) for x in values]) threshold = k * mad is_anomaly = abs(value - median) > threshold ``` +When more than half of the values in the window are identical, `mad` is `0` and the bounds collapse onto the median, so any different value counts as an outlier no matter how large `k` is. This affects flat signals: a stable sensor, a metric that is usually `0`, or a low-resolution integer field. The effect also works in reverse — once outliers fill more than half of the window they become the new median and stop being detected. + #### Flip Detection -Counts transitions between normal and anomalous states within the window to prevent alert fatigue from rapidly changing values. +The plugin keeps the recent outlier flags of each threshold in a deque the size of `window_count` and counts transitions between normal and outlier state. Once the number of transitions reaches `state_change_count`, the alert is computed as usual but not delivered, and a warning is logged instead. This prevents alert fatigue from values that switch in and out of the outlier state. + +An alert that follows normal data always records one normal-to-outlier transition, so `state_change_count` must be 2 or greater to leave sustained anomalies alone. A value of `1` would suppress every alert; the plugin logs a warning and treats it as `0`. + +A count threshold needs consecutive outliers, so when it fires the last `threshold` flags are all outliers and only `window_count - threshold` transitions can remain in the window. Suppression therefore requires `window_count >= threshold + state_change_count`; otherwise the plugin logs a `Flip suppression never triggers` warning naming the field. Duration thresholds have no such limit. ## Troubleshooting @@ -327,13 +347,18 @@ Counts transitions between normal and anomalous states within the window to prev ` 3. Ensure notification channel parameters are provided for selected senders -#### Issue: "Invalid MAD thresholds format" error +#### Issue: "No valid MAD thresholds provided" error -**Solution**: Check threshold format is correct: +**Solution**: Each invalid threshold is logged as a warning naming the part that failed. Check the format: -- Count-based: `field:k:window:count` (for example, `temp:2.5:20:5`) -- Duration-based: `field:k:window:duration` (for example, `temp:2.5:20:2m`) +- Count-based: `field:k:window_count:count` (for example, `temp:2.5:20:5`) +- Duration-based: `field:k:window_count:duration` (for example, `temp:2.5:20:2min`) - Multiple thresholds separated by `@` +- `k` must not be negative, `window_count` must be 2 or greater, the count must be 1 or greater + +#### Issue: Alerts are logged but never delivered + +**Solution**: Look for `Suppressed count alert` or `Suppressed duration alert` warnings. They mean flip suppression is active. Raise `state_change_count`, or remove it to disable suppression. #### Issue: Too many false positive alerts @@ -344,6 +369,8 @@ Counts transitions between normal and anomalous states within the window to prev 3. Enable flip suppression with `state_change_count` 4. Increase the window size for more stable statistics +If the log line reports `mad=0.000`, the window has no spread and `k` has no effect. Require the change to persist with a count or duration threshold instead. + #### Issue: Missing anomalies (false negatives) **Solution**: @@ -354,24 +381,26 @@ Counts transitions between normal and anomalous states within the window to prev ### Debugging tips -1. **Monitor deque sizes**: +1. **Check whether windows are still filling up**: ```bash - influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine_logs WHERE log_text LIKE '%Deque%'" + influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine_logs WHERE log_text LIKE '%Waiting for%points for MAD%'" ``` -2. **Check MAD calculations**: +2. **Check MAD calculations** (logged for detected outliers only): ```bash - influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine_logs WHERE log_text LIKE '%MAD:%'" + influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine_logs WHERE log_text LIKE '%MAD calculation%'" ``` 3. **Test with known anomalies**: Write test data with obvious outliers to verify detection ### Performance considerations -- **Memory usage**: Each field maintains a deque of `window_count` values +- **Memory usage**: Each field and series maintains a deque of `window_count` values - **Computation**: MAD is computed on every data write for monitored fields -- **Caching**: Measurement and tag names are cached for 1 hour -- **Notification retries**: Failed notifications retry up to 3 times with exponential backoff +- **Caching**: Measurement and tag names are cached for 1 hour, the loaded configuration for 10 minutes +- **Early exit**: Writes that contain no rows of the configured measurement return before thresholds, senders and tags are parsed; the configuration and the table list come from the cache +- **Notification delivery**: Each alert is sent in a single attempt with a 5-second timeout; retries would hold up the write path +- **Logging**: MAD calculations are logged only for points detected as outliers, so a calm table produces two log lines per write ## Report an issue @@ -380,4 +409,6 @@ For plugin issues, see the Plugins repository [issues page](https://github.com/i ## Find support for {{% product-name %}} The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. -For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. \ No newline at end of file +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/mqtt-subscriber.md b/content/shared/influxdb3-plugins/plugins-library/official/mqtt-subscriber.md new file mode 100644 index 0000000000..f8a8cf7700 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/mqtt-subscriber.md @@ -0,0 +1,543 @@ + + +> **Note:** This plugin requires {{% product-name %}}.8.2 or later. + + +The MQTT Subscriber Plugin enables real-time ingestion of MQTT messages into {{% product-name %}}. Subscribe to MQTT broker topics and automatically transform messages into time-series data with support for JSON, Line Protocol, and custom text formats. The plugin uses persistent MQTT sessions (`clean_session=False`) to ensure message delivery between executions and provides comprehensive error tracking and statistics. + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. This plugin supports TOML configuration files for complex mapping scenarios, which can be specified using the `config_file_path` parameter. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Required parameters + +In TOML configuration, `broker_host` and `topics` are placed under the `[mqtt]` section; `table_name` and `table_name_field` are placed under `[mapping.json]` or `[mapping.text]` section. + +| Parameter | Type | Default | Description | +|---------------|--------|---------------------------|-------------------------------------------------------------------------------------| +| `broker_host` | string | required | MQTT broker hostname or IP address | +| `topics` | string | required | Space-separated list of topics (for example, "sensors/temp sensors/humidity") | +| `table_name` | string | required (json/text only) | InfluxDB measurement name for storing data. Not required for `lineprotocol` format or when `table_name_field` is set. | +| `table_name_field` | string | none | JSON field name or regex pattern to extract table name dynamically from each message. Alternative to static `table_name`. | + +### Connection parameters + +In TOML configuration, these parameters are placed under the `[mqtt]` section. + +| Parameter | Type | Default | Description | +|-------------------|---------|----------------|------------------------------------------------------------------------------------------| +| `broker_port` | integer | 1883 | MQTT broker port (1883 for non-TLS, 8883 for TLS) | +| `qos` | integer | 1 | MQTT Quality of Service level (0, 1, or 2) | +| `client_id` | string | auto-generated | MQTT client identifier (must be unique per broker) | +| `max_queue_bytes` | integer | 67108864 | Maximum total bytes of MQTT payloads buffered between scheduled drains (default: 64 MiB) | + + +**Recommendation:** Use QoS 1 for most IoT scenarios. It provides reliable delivery with minimal overhead. + +**About `max_queue_bytes`:** Between two scheduled fires, paho's network thread buffers incoming MQTT messages in an in-memory queue inside the plugin. To protect InfluxDB from a misbehaving or malicious broker, the plugin tracks the running total of buffered payload sizes; once it would exceed `max_queue_bytes`, new messages are dropped (the existing queue is still processed normally). Drop counts are reported per-topic in the `mqtt_stats` table (`messages_dropped` field) and a summary error is logged at the end of each cycle when any drops occurred. Raise this value if you expect high-throughput bursts; lower it on memory-constrained hosts. + +### Authentication parameters + +In TOML configuration, `username` and `password` are placed under the `[mqtt.auth]` section. + +| Parameter | Type | Default | Description | +|------------|--------|---------|-------------------------------------------------| +| `username` | string | none | MQTT broker username (required with password) | +| `password` | string | none | MQTT broker password (required with username) | + +**Note:** Both `username` and `password` must be provided together for authentication. + +`allow_insecure_auth` is placed directly under the `[mqtt]` section (not `[mqtt.auth]`): + +| Parameter | Type | Default | Description | +|-----------------------|---------|---------|-----------------------------------------------------------------------------| +| `allow_insecure_auth` | boolean | false | Permit sending `username`/`password` when TLS is not configured | + +**Security note:** When `username`/`password` are provided without a `ca_cert` (TLS), +credentials are transmitted in cleartext over an unencrypted connection. The plugin +refuses this by default and raises a configuration error. Configure TLS (`ca_cert`), +or explicitly set `allow_insecure_auth = true` to permit cleartext credentials — only +do this on a trusted network. + +### TLS/SSL parameters + +In TOML configuration, these parameters are placed under the `[mqtt.tls]` section. + +| Parameter | Type | Default | Description | +|---------------|--------|---------|-------------------------------------------| +| `ca_cert` | string | none | Path to CA certificate file | +| `client_cert` | string | none | Path to client certificate for mutual TLS | +| `client_key` | string | none | Path to client private key for mutual TLS | + +**Note:** For mutual TLS, both `client_cert` and `client_key` must be provided together. + +### Logging parameters + +In TOML configuration, `enable_full_logging` is placed directly under the `[mqtt]` section. + +| Parameter | Type | Default | Description | +|-----------------------|---------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `enable_full_logging` | boolean | false | When `true`, full exception messages are written to logs. When `false` (default), only the exception type is logged, to avoid leaking sensitive values (credentials, payloads, paths) into log output. Enable temporarily for debugging. | + +### Message format parameters + +In TOML configuration, `format` is placed under the `[mqtt]` section; `timestamp_field` is placed under `[mapping.json]` or `[mapping.text]` section. + +| Parameter | Type | Default | Description | +|-------------------|--------|---------|----------------------------------------------------------------| +| `format` | string | "json" | Message format: json, lineprotocol, or text | +| `timestamp_field` | string | none | Field containing timestamp (format depends on message format) | + +**Format-specific timestamp_field syntax:** + +The timestamp field format differs between JSON and Text formats: + +| Format | Syntax | Split Method | Example (CLI) | Example (TOML) | +|----------|---------------------|--------------------|------------------|--------------------| +| JSON | `field_name:format` | Split by first `:` | `"timestamp:ms"` | `"$.timestamp:ms"` | +| Text | `regex:format` | Split by last `:` | `"ts:(\\d+):ms"` | `"ts:(\\d+):ms"` | + +**Note:** In CLI arguments, JSON paths are specified without `$.` prefix (added automatically). In TOML configuration, use full JSONPath syntax with `$.` prefix. + +**Note:** Text format uses the last colon to split, allowing regex patterns to contain colons (for example, time patterns). + +**Supported timestamp formats:** +- `ns` - nanoseconds (Unix timestamp) +- `ms` - milliseconds (Unix timestamp) +- `s` - seconds (Unix timestamp) +- `datetime` - ISO 8601 string (for example, "2021-12-01T12:00:00Z") + +### JSON format parameters + +In TOML configuration, `tags` are placed under `[mapping.json.tags]` section and `fields` under `[mapping.json.fields]` section. + +| Parameter | Type | Default | Description | +|-----------|--------|----------|---------------------------------------------------------------------------| +| `tags` | string | none | Space-separated tag names. Example: "room sensor location" | +| `fields` | string | required | Space-separated field mappings. Format: "name:type=jsonpath" without `$.` | + +**Field specification format:** `"temp:float=temperature hum:int=humidity status:bool=online"` + +**Supported field types:** `int`, `uint`, `float`, `string`, `bool` + +**JSONPath restriction:** The regex-based JSONPath operators `=~` (filter +regex-match) and `sub()` (regex substitution) are disabled. They run a +configured regex against untrusted broker data and are vulnerable to +backtracking (ReDoS). A configuration that uses them is rejected +at startup. All other JSONPath constructs — nesting, recursive descent (`..`), +wildcards (`*`), and comparison filters such as `[?(@.type=="temp")]` — are +fully supported. + +### Text format parameters + +In TOML configuration, `tags` are placed under `[mapping.text.tags]` section and `fields` under `[mapping.text.fields]` section. + +| Parameter | Type | Default | Description | +|-----------|--------|-----------|-------------------------------------------------------------------| +| `tags` | string | none | Space-separated tag patterns. Format: "name=regex_pattern" | +| `fields` | string | required | Space-separated field patterns. Format: "name:type=regex_pattern" | + +**Tag specification format:** `"room=room:([^,\\s]+) sensor=sensor:(\\w+)"` + +**Field specification format:** `"temp:float=temp:([\\d.]+) status:bool=(true|false)"` + +### TOML configuration + +| Parameter | Type | Default | Description | +|--------------------|--------|---------|-------------------------------------------------| +| `config_file_path` | string | none | Path to TOML config file (absolute or relative) | + +*To use a TOML configuration file, specify the `config_file_path` in the trigger arguments.* + +### File path resolution + +All file paths in the plugin (configuration file, TLS certificates) follow the same resolution logic: + +- **Absolute paths** (for example, `/etc/mqtt/config.toml`) are used as-is +- **Relative paths** (for example, `config.toml`, `certs/ca.crt`) are resolved from `PLUGIN_DIR` environment variable + +If a relative path is specified and `PLUGIN_DIR` is not set, the plugin will return an error. + +#### Example TOML configuration + +[mqtt_config_example.toml](https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/mqtt_subscriber/mqtt_config_example.toml) - comprehensive configuration example with all three message formats + +## Data requirements + +The plugin automatically creates the target measurement table on first write. Field mappings are required for JSON and Text formats to specify which fields to extract and their data types. + +### Message encoding requirements + +- **UTF-8 text only**: The plugin only processes UTF-8 encoded text messages. Binary messages are automatically skipped with a warning logged. +- **Non-empty payloads**: Empty or whitespace-only messages are automatically skipped. + +## Software Requirements + +- **{{% product-name %}}**: with the Processing Engine enabled +- **Python packages**: + - `paho-mqtt` (MQTT client library) + - `jsonpath-ng` (JSON path parsing for JSON format) + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install required Python packages: + + ```bash + influxdb3 install package paho-mqtt + influxdb3 install package jsonpath-ng + ``` +## Trigger setup + +### Scheduled ingestion with TOML configuration + +Recommended for production use with complex mappings: + +```bash +# 1. Set PLUGIN_DIR environment variable +export PLUGIN_DIR=~/.plugins + +# 2. Copy and edit configuration file +cp mqtt_config_example.toml $PLUGIN_DIR/my_mqtt_config.toml +# Edit my_mqtt_config.toml with your broker and mapping settings + +# 3. Create the trigger +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/mqtt_subscriber/mqtt_subscriber.py \ + --trigger-spec "every:10m" \ + --trigger-arguments config_file_path=my_mqtt_config.toml \ + mqtt_ingestion +``` +### Scheduled ingestion with command-line arguments + +For simple JSON message ingestion: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/mqtt_subscriber/mqtt_subscriber.py \ + --trigger-spec "every:5m" \ + --trigger-arguments 'broker_host=broker.hivemq.com,topics=sensors/temperature sensors/humidity,format=json,table_name=sensor_data,fields=temp:float=temperature hum:int=humidity,tags=location sensor_id' \ + mqtt_sensors +``` +### Secure MQTT connection with TLS + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/mqtt_subscriber/mqtt_subscriber.py \ + --trigger-spec "every:10m" \ + --trigger-arguments 'broker_host=secure-broker.example.com,broker_port=8883,topics=secure/data,format=json,table_name=secure_data,ca_cert=certs/ca.crt,username=myuser,password=mypass,fields=value:float=value' \ + secure_mqtt +``` +## MQTT Topic Wildcards + +The plugin supports standard MQTT wildcard patterns in topic subscriptions: + +- `+` - Single level wildcard (matches one topic level) +- `#` - Multi-level wildcard (matches any number of levels, must be last) + +**Examples:** +- `sensors/+/temperature` - matches `sensors/room1/temperature`, `sensors/room2/temperature` +- `sensors/#` - matches `sensors/temp`, `sensors/room1/humidity`, `sensors/building/floor1/co2` +- `+/+/data` - matches `home/kitchen/data`, `office/room1/data` + +**Note:** Wildcards are processed by the MQTT broker, not the plugin. Ensure your broker supports the wildcard patterns you use. + +## Message Formats + +### JSON Format + +The primary use case for structured IoT data. Supports nested fields using JSONPath expressions. + +#### TOML Configuration + +```toml +[mqtt] +broker_host = "broker.hivemq.com" +broker_port = 1883 +topics = ["sensors/temperature"] +qos = 1 +format = "json" + +[mapping.json] +table_name = "sensor_data" +timestamp_field = "$.timestamp:ms" + +[mapping.json.tags] +location = "$.location" +sensor_id = "$.sensor.id" + +[mapping.json.fields] +temperature = ["$.temp", "float"] +humidity = ["$.humidity", "int"] +status = ["$.online", "bool"] +``` +#### Example Message + +```json +{ + "timestamp": 1638360000000, + "location": "warehouse_a", + "sensor": { + "id": "sensor_001" + }, + "temp": 22.5, + "humidity": 65, + "online": true +} +``` +#### Resulting Data + +``` +sensor_data,location=warehouse_a,sensor_id=sensor_001 temperature=22.5,humidity=65i,status=true 1638360000000000000 +``` +#### JSON Array Support + +Process batch messages containing arrays of JSON objects: + +```json +[ + {"timestamp": 1638360000000, "sensor_id": "001", "temperature": 22.5}, + {"timestamp": 1638360001000, "sensor_id": "002", "temperature": 23.1}, + {"timestamp": 1638360002000, "sensor_id": "003", "temperature": 21.8} +] +``` +**Array processing behavior:** +- Each array element is processed independently as a separate data point +- If one element fails to parse, the others continue processing (partial success) +- Parse errors for individual elements are logged to `mqtt_exceptions` table +- Statistics count 1 MQTT message = 1 unit (regardless of array size) + +### Line Protocol Format + +For messages already in InfluxDB line protocol format, use passthrough mode. No mapping configuration needed - messages are validated and written directly. + +#### TOML Configuration + +```toml +[mqtt] +broker_host = "broker.example.com" +topics = ["influxdb/metrics"] +format = "lineprotocol" +``` +#### CLI Configuration + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename mqtt_subscriber.py \ + --trigger-spec "every:10m" \ + --trigger-arguments 'broker_host=broker.example.com,topics=influxdb/metrics,format=lineprotocol' \ + mqtt_lineprotocol +``` +#### Example Message + +``` +sensor_data,location=warehouse_a,sensor_id=001 temperature=22.5,humidity=65i 1638360000000000000 +``` +#### Supported Line Protocol Types + +| Type | Suffix | Example | +|------------------|----------------|--------------------| +| Float | none | `temperature=22.5` | +| Integer | `i` | `count=100i` | +| Unsigned Integer | `u` | `bytes=1024u` | +| String | `"..."` | `status="running"` | +| Boolean | `true`/`false` | `active=true` | + +### Text Format + +Parse plain text messages using regular expressions: + +#### TOML Configuration + +```toml +[mqtt] +format = "text" + +[mapping.text] +table_name = "sensor_logs" +timestamp_field = "ts:(\\d+):ms" + +[mapping.text.tags] +location = "location=([^,\\s]+)" + +[mapping.text.fields] +temperature = ["temp:([\\d.]+)", "float"] +humidity = ["hum:(\\d+)", "int"] +status = ["status:(true|false)", "bool"] +``` +#### Example Message + +``` +location=warehouse_a,temp:22.5,hum:65,status:true,ts:1638360000000 +``` +## Statistics and Monitoring + +The plugin tracks comprehensive statistics and writes them to the `mqtt_stats` table on every plugin invocation. + +**Important notes:** +- Statistics are written **on every plugin invocation** +- Each topic is tracked separately with independent statistics +- Statistics persist across plugin restarts using the InfluxDB cache + +### mqtt_stats Table + +| Field | Type | Description | +|-----------------------|-------|---------------------------------------------------------------------------------------------------| +| `topic` (tag) | tag | MQTT topic name | +| `broker_host` (tag) | tag | MQTT broker address | +| `messages_received` | int | Total messages received on this topic (includes dropped messages) | +| `messages_processed` | int | Successfully processed messages | +| `messages_failed` | int | Failed messages | +| `messages_dropped` | int | Messages dropped because the `max_queue_bytes` budget was exhausted | +| `success_rate` | float | Percentage of successfully processed messages: `processed / (processed + failed + dropped) * 100` | + +### Querying Statistics + +```bash +# Get latest statistics +influxdb3 query --database mydb \ + "SELECT * FROM mqtt_stats ORDER BY time DESC LIMIT 10" + +# Success rate over time +influxdb3 query --database mydb \ + "SELECT topic, success_rate, messages_processed, messages_failed + FROM mqtt_stats + WHERE time > now() - INTERVAL '1 hour' + ORDER BY time DESC" +``` +## Error Handling + +Parse errors and message processing failures are logged to the `mqtt_exceptions` table: + +### mqtt_exceptions Table + +| Field | Type | Description | +|-------------------|--------|------------------------------------------| +| `topic` (tag) | tag | MQTT topic where error occurred | +| `error_type` (tag)| tag | Type of error (for example, JSONDecodeError) | +| `error_message` | string | Detailed error message | +| `raw_message` | string | Original MQTT message (truncated to 1KB) | + +### Checking for Errors + +```bash +influxdb3 query --database mydb \ + "SELECT * FROM mqtt_exceptions ORDER BY time DESC LIMIT 10" +``` +## Troubleshooting + +### Check Plugin Logs + +```bash +influxdb3 query --database _internal \ + "SELECT * FROM system.processing_engine_logs + WHERE trigger_name = 'mqtt_ingestion' + ORDER BY time DESC LIMIT 20" +``` +### Common Issues + +#### "paho-mqtt library not installed" + +```bash +influxdb3 install package paho-mqtt +``` +#### "Configuration file not found" + +- For relative paths, ensure `PLUGIN_DIR` environment variable is set +- For absolute paths, verify the file exists at the specified location + +```bash +# For relative paths +export PLUGIN_DIR=~/.plugins +ls $PLUGIN_DIR/my_mqtt_config.toml + +# Or use absolute path +ls /etc/mqtt/my_mqtt_config.toml +``` +#### "Failed to connect to MQTT broker" + +- Verify broker address and port +- Check network connectivity +- For TLS connections, verify certificate paths +- Verify authentication credentials (both username and password required) + +#### "Both username and password must be provided for authentication" + +Either provide both `username` and `password`, or omit both for anonymous connection. + +#### "Refusing to send username/password over an unencrypted connection" + +`username`/`password` were provided without a `ca_cert`, so the credentials would be +sent in cleartext. Configure TLS by providing `ca_cert` (and `client_cert`/`client_key` +for mutual TLS), or set `allow_insecure_auth = true` to permit cleartext credentials on +a trusted network. + +#### "Both client_cert and client_key must be provided for mutual TLS" + +For mutual TLS authentication, both client certificate and key are required. + +#### "No fields were mapped from JSON data" + +- Verify JSONPath expressions in field mappings (use `$.` prefix) +- Check that JSON structure matches your paths +- Review `mqtt_exceptions` table for detailed errors + +#### "Unsupported JSONPath operator '=~' / 'sub'" + +A tag, field, timestamp, or `table_name_field` mapping uses a regex-based +JSONPath operator. These are disabled for security (ReDoS). Remove the `=~` +filter or `sub()` call — use a comparison filter (`[?(@.x=="value")]`) +instead, or extract the raw value and transform it downstream. + +#### Messages not being processed + +- Check trigger status: `influxdb3 show summary --database mydb` +- Verify MQTT connection in plugin logs +- Increase trigger frequency (for example, from `every:5s` to `every:1s`) + +## Architecture + +### How It Works + +1. **Scheduled Trigger**: Plugin runs on schedule (for example, `every:10m`) +2. **Configuration Caching**: Plugin configuration is parsed once and cached between executions +3. **Message Queue**: Incoming MQTT messages are queued in-memory during callback execution +4. **Batch Processing**: Each trigger execution processes all queued messages +5. **Parse & Write**: Messages parsed according to format and written to InfluxDB +6. **Error Tracking**: Parse errors logged to `mqtt_exceptions` table +7. **Statistics**: Written to `mqtt_stats` table on every plugin invocation + +### Performance Optimization + +The plugin includes several optimizations for high-throughput scenarios: + +- **Configuration Caching**: Plugin configuration is parsed once and reused across all trigger executions +- **Pre-compiled Patterns**: JSONPath expressions and regex patterns are compiled once during parser initialization, not per-message +- **Persistent Session**: MQTT client uses `clean_session=False` - broker preserves subscriptions and queued messages between connections + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/nori-regression.md b/content/shared/influxdb3-plugins/plugins-library/official/nori-regression.md new file mode 100644 index 0000000000..aa61b915d9 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/nori-regression.md @@ -0,0 +1,558 @@ + + +> **Note:** This plugin requires {{% product-name %}}.8.2 or later (it uses the synchronous write API). + +Predict a numeric field in an {{% product-name %}} measurement from other columns on the same rows with +**Nori**, Synthefy's in-context-learning tabular regression model, called through the Synthefy +inference gateway. The plugin reads a window of rows, trains on the rows where the target field is +present, predicts the rows where it is null (imputation / backfill), and writes the predicted values +back into InfluxDB. + + +Nori is a tabular regression foundation model: you give it labeled feature rows (`X_train`, +`y_train`) and query rows (`X_test`) in a single request, and it predicts a value for each query row +in one forward pass, with no training or fine-tuning step. + +This plugin applies Nori to an InfluxDB measurement. You choose a target field and a set of feature +columns; the plugin uses the rows where the target is present as the in-context training set and +predicts the target for the rows where it is null, writing each prediction back at its own row's +timestamp. It is plain tabular regression: Nori sees only the feature columns you name, with no time +or ordering assumptions. + +Typical uses: + +- Backfill a field that dropped out (a sensor went offline while its neighbors kept reporting). +- Impute a missing metric from correlated ones (for example, predict `pressure` from `temperature` + and `humidity`). +- Derive a field that is expensive to measure directly from cheaper ones recorded alongside it. + +Key features: + +- **In-context tabular regression**: no training step; the recent labeled rows are the context. +- **Imputation / backfill**: predicts the rows where the target is null and writes them back. +- **Scheduled or on-demand**: run on an interval, or call an HTTP endpoint with an explicit window. +- **Idempotent by default**: rows that already hold a prediction are skipped, so a repeating + schedule does not re-send and re-pay for the same rows. +- **Bounded cost**: row caps and a batch size keep one run's billed rows predictable. +- **Single-series guarantee**: a run that resolves to more than one series fails before it calls the + gateway, rather than training one model on two mixed series. + +## Configuration + +Plugin parameters may be given as key-value pairs in the `--trigger-arguments` flag of +`influxdb3 create trigger`, in the `trigger_arguments` field of the API, or entirely from a TOML +file via `config_file_path` — see [TOML configuration](#toml-configuration). For the HTTP trigger, a +documented subset may also be sent in the JSON request body. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that declares the supported trigger +types (`scheduled`, `http`) and every parameter each accepts, so the +[InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI can render a configuration +form. + +### Authentication for the Nori gateway + +The Nori gateway API key is a secret and is **never** read from trigger arguments or the request +body (both are logged). It is resolved, in order: + +1. a non-empty `X-Nori-Api-Key: ` request header (HTTP trigger only), then +2. the `SYNTHEFY_NORI_API_KEY` environment variable set on the InfluxDB host (required for the + scheduled trigger). + +The key is intentionally **not** accepted in the `Authorization` header: InfluxDB parses +`Authorization` for its own request authorization, so a key placed there never reaches the plugin. +Use the custom `X-Nori-Api-Key` header instead. + +Get a Nori API key from the [Synthefy console](https://console.synthefy.com/). One key covers every +model slug its group is granted (see [Supported models](#supported-models)), so you do not normally +need a key per variant. This plugin does not create keys. + +The gateway endpoint itself is **not** a parameter: the request carries the operator's API key and +the training data, so a caller must never be able to choose its destination. An operator running a +private gateway can point the plugin at it with the `NORI_GATEWAY_URL` environment variable on the +InfluxDB host. It must be an `https://` URL; plain `http://` is accepted only for a loopback host +(`localhost`, `127.0.0.1` or `::1`), so a local mock gateway still works in testing. + +### Required parameters + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `measurement` | string | required | Source measurement (table) to read from. | +| `field` | string | required | The numeric field to predict. The plugin trains on the rows where it is present and predicts the rows where it is null. | +| `feature_fields` | string | required | Numeric feature columns (X) used to predict `field`, **space-separated** (for example `temp humidity`). Use spaces, not commas (`--trigger-arguments` splits argument pairs on commas) and not dots (a field name may contain a `.`). | +| `model` | string | required | The Nori gateway slug to call. There is no default: the slug selects a priced model, so the plugin will not choose one for you. See [Supported models](#supported-models). Trigger argument only. | + +A column name that contains a space cannot be expressed in `feature_fields` as a trigger argument, +because every string form splits on whitespace. Name such a column from a TOML array +(`feature_fields = ["air temp", "humidity"]`) or from a JSON list in the HTTP body. + +### Optional parameters + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `window` | string | `30d` | Time window of rows to read, ending at the trigger's call time. Units: `s`, `min`, `h`, `d`, `w`, with an integer magnitude. | +| `start_time` | string | *(none)* | ISO 8601 start of a fixed window. Given alone, the window ends now. | +| `end_time` | string | *(none)* | ISO 8601 end of a fixed window. Given alone, the window starts one `window` earlier. | +| `tags` | string | *(none)* | Filter to a single series. Format: `key:val key2:val2` (space-separated pairs, one value per key). A token without a `:` is rejected. Required when the window holds more than one series. | +| `output_measurement` | string | `_regressed` | Measurement to write predictions to. Must differ from `measurement`. | +| `target_database` | string | *(trigger db)* | Write predictions to a different database. | +| `dry_run` | boolean | `false` | Log the first few predictions and return them all, without writing anything. | +| `skip_existing` | boolean | `true` | Skip rows that already hold a prediction in `output_measurement`. Set `false` to refresh earlier predictions with newer training data. | +| `min_history` | integer | `50` | Minimum labeled rows required to train; the run is skipped below this. | +| `max_train_rows` | integer | `1000` | Cap on labeled rows sent as the training context; the most recent rows are kept. This is the main cost control — the gateway bills per training row and column. | +| `max_predict_rows` | integer | `5000` | Cap on rows predicted per run; the most recent rows are kept and the rest wait for a later run. | +| `max_read_rows` | integer | `50000` | Ceiling on rows read from InfluxDB in one run, applied as a `LIMIT` on the query. The most recent rows are read, and a truncated read is logged with a warning. This bounds the plugin's memory: a row costs roughly 0.7 KB while it is held, so the default is about 35 MB. | +| `predict_batch_size` | integer | `1000` | Rows per gateway call. Each batch re-sends the training context and is billed separately, so a larger value costs less. | +| `request_timeout` | string | `300s` | Timeout for one gateway call. A model that has scaled to zero cold-starts on the first request, measured between roughly one and four minutes depending on the variant, so keep this well above the warm response time. | +| `max_retries` | integer | `3` | Maximum attempts per gateway call and per write. `1` disables retry. | +| `config_file_path` | string | *(none)* | Path to a TOML file supplying every parameter, relative to `PLUGIN_DIR`. Cannot be combined with other inline arguments or a request body. | + +Two constraints are checked before anything runs: `min_history` must not exceed `max_train_rows` +(no run could otherwise ever qualify), and `output_measurement` must differ from `measurement`. All +of the integer parameters must be at least `1`. + +### HTTP request body parameters + +On the HTTP trigger, these keys may be sent in the JSON request body: + +`measurement`, `field`, `feature_fields`, `tags`, `window`, `start_time`, `end_time`, `dry_run`. + +`feature_fields` may be a JSON list (`{"feature_fields": ["temp", "humidity"]}`) or a +space-separated string, and `tags` may be a JSON object (`{"tags": {"site": "A"}}`). + +**A trigger argument pins its value.** The body may fill in what the trigger left open, but it +cannot change what the trigger already set — that is rejected. So an operator who wants the request +to choose the measurement creates the trigger without one, and an operator who wants it fixed sets +it as a trigger argument. This matters because `output_measurement` defaults to +`_regressed`: without the pin, a body-supplied `measurement` would move the write +target too. + +Every other parameter is **operator-only** and is rejected by name if it appears in the body. The +endpoint is reachable by anyone holding a database token, so the model slug (which selects a billed +model), the write targets (`output_measurement`, `target_database`), the row caps, the timeout and +`config_file_path` stay under the control of whoever created the trigger. + +`gateway_url` is not a parameter at all, in either place — use the `NORI_GATEWAY_URL` environment +variable. Passing it (or a parameter from the plugin's earlier forecasting revision: `mode`, +`horizon`, `step`, `lags`, `rolling`, `tz`) is rejected with a message naming the replacement, rather +than ignored. + +### TOML configuration + +Set the `PLUGIN_DIR` environment variable and reference the file with the `config_file_path` trigger +argument (relative paths resolve against `PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent +of `VIRTUAL_ENV`). The TOML file then supplies **all** parameters — it is mutually exclusive with +inline trigger arguments and with an HTTP request body. See +[`nori_regression_config_scheduler.toml`](nori_regression_config_scheduler.toml) for an annotated +template. + +```bash +influxdb3 create trigger \ + --database mydb \ + --path "gh:influxdata/nori_regression/nori_regression.py" \ + --trigger-spec "every:1h" \ + --trigger-arguments config_file_path=nori_regression_config_scheduler.toml \ + nori_from_toml +``` +## Requirements + +### Software requirements + +- **{{% product-name %}} Core or Enterprise**, version 3.8.2 or later, with the Processing Engine enabled + (`influxdb3 serve --plugin-dir /path/to/plugins`). +- **Python packages**: `influxdata-plugin-utils>=0.3.0`, `requests`. +- A **Nori API key** from the [Synthefy console](https://console.synthefy.com/), reachable from the + InfluxDB host over HTTPS. + +### Installation steps + +1. Install the Python dependencies into the {{% product-name %}} Processing Engine environment: + + ```bash + influxdb3 install package influxdata-plugin-utils requests + ``` +2. Reference the plugin directly from this repository with the `gh:` prefix (the form used in the + examples below): `--path "gh:influxdata/nori_regression/nori_regression.py"`. Alternatively, copy + `nori_regression.py` into your plugin directory (the one passed to `influxdb3 serve + --plugin-dir`) and use `--path nori_regression.py`. + +3. Set the Nori gateway key on the InfluxDB host, so the scheduled trigger can read it: + + ```bash + export SYNTHEFY_NORI_API_KEY="" + ``` +### Data requirements + +- The measurement holds at least `min_history` rows where the target `field` is present **and** + every `feature_fields` column is present. Those rows are the training context. +- It holds at least one row where the target is null and every feature is present. Those rows are + what gets predicted; if there are none, the run is a no-op. +- The window resolves to a **single series**. If the measurement holds several series (one per + `site`, say), pass a `tags` filter that isolates one, or create one trigger per series. +- The features actually explain the target. Nori sees no time and no row order, so a target that + depends on time rather than on the feature columns is not a good fit for this plugin. + +### Schema requirements + +The plugin reads `information_schema.columns` before it queries data, and fails with a message +naming the offending column if the schema cannot serve the request: + +| Column | Required type | +|---|---| +| `time` | timestamp (every InfluxDB measurement has one) | +| `field` (the target) | numeric field: `Int64`, `UInt64`, `Int32`, `Float64` or `Float32` | +| each `feature_fields` entry | numeric field, and neither the target nor `time` | +| each `tags` key | a tag column (`Dictionary(Int32, Utf8)`), not a field | +| the source tag columns | none named `model`, `source` or `target` | + +A string or boolean column named as a feature is rejected here rather than coerced to null, which +would otherwise surface much later as `only 0 labeled rows`. + +The last row matters because every output point carries `model`, `source` and `target` tags for +provenance. A source tag with one of those names would overwrite the provenance on write *and* make +the `skip_existing` lookup contradict itself, so the run would silently re-send and re-pay for the +same rows on every tick. The plugin refuses the configuration instead. + +## Trigger setup + +### Scheduled trigger + +Every 15 minutes, fill any rows of `sensors` (for `site=A`) that are missing `pressure`, predicting +it from `temp` and `humidity`: + +```bash +influxdb3 create trigger \ + --database mydb \ + --path "gh:influxdata/nori_regression/nori_regression.py" \ + --trigger-spec "every:15m" \ + --trigger-arguments measurement=sensors,field=pressure,feature_fields="temp humidity",tags=site:A,model=synthefy/nori-30m \ + nori_sensors_pressure +``` +Because `skip_existing` defaults to `true`, each subsequent run only sends the rows that still have +no prediction. Once the window is fully imputed, the trigger stops calling the gateway entirely. + +### HTTP trigger + +```bash +influxdb3 create trigger \ + --database mydb \ + --path "gh:influxdata/nori_regression/nori_regression.py" \ + --trigger-spec "request:nori_regress" \ + nori_http +``` +## Example usage + +### Example 1: impute a missing field on a schedule + +Write sample data. The plugin needs at least `min_history` complete rows to train on, so this +example lowers that to `3` — a real deployment should leave it at the default and train on far more. +The last two rows carry `temp` and `humidity` but no `pressure`, and those are the ones that get +imputed: + +```bash +influxdb3 write --database mydb --precision s " +sensors,site=A temp=20.0,humidity=40.0,pressure=1000.0 1767225600 +sensors,site=A temp=22.0,humidity=41.0,pressure=1000.7 1767225660 +sensors,site=A temp=24.0,humidity=42.0,pressure=1001.4 1767225720 +sensors,site=A temp=25.0,humidity=45.0 1767229200 +sensors,site=A temp=21.0,humidity=41.0 1767229260 +" +``` +```bash +influxdb3 create trigger \ + --database mydb \ + --path "gh:influxdata/nori_regression/nori_regression.py" \ + --trigger-spec "every:15m" \ + --trigger-arguments measurement=sensors,field=pressure,feature_fields="temp humidity",tags=site:A,model=synthefy/nori-30m,min_history=3 \ + nori_example +``` +Read the predictions back after the trigger runs: + +```bash +influxdb3 query --database mydb " +SELECT time, value, model, target, site +FROM sensors_regressed +ORDER BY time DESC +LIMIT 5 +" +``` +**Expected output:** + +``` ++---------------------+--------+-------------------+----------+------+ +| time | value | model | target | site | ++---------------------+--------+-------------------+----------+------+ +| 2026-01-01T01:01:00 | 998.2 | synthefy/nori-30m | pressure | A | +| 2026-01-01T01:00:00 | 999.0 | synthefy/nori-30m | pressure | A | ++---------------------+--------+-------------------+----------+------+ +``` +### Example 2: on-demand HTTP regression + +Call the HTTP endpoint (exposed at `/api/v3/engine/`), passing the Nori key in the header: + +```bash +curl -X POST http://localhost:8181/api/v3/engine/nori_regress \ + -H "X-Nori-Api-Key: $SYNTHEFY_NORI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"measurement":"sensors","field":"pressure","feature_fields":["temp","humidity"],"tags":{"site":"A"}}' +``` +**Expected output:** + +```json {lint="false"} +{"status": "success", "task_id": "...", "result": {"status": "success", "written": 24}} +``` +A run that had nothing to do reports its real outcome instead of a bare success: + +```json {lint="false"} +{"status": "skipped", "task_id": "...", "result": {"status": "skipped", "written": 0}} +``` +A run stopped part-way by a gateway fault keeps the batches it already paid for and reports the +shortfall, so a caller never reads a partial result as a complete one: + +```json {lint="false"} +{"status": "partial", "task_id": "...", "result": {"status": "partial", "written": 8, "remaining": 12}} +``` +The top-level `status` is one of `success`, `partial`, `skipped`, `dry_run` or `failed`. + +### Example 3: backfill a specific window + +```bash +curl -X POST http://localhost:8181/api/v3/engine/nori_regress \ + -H "X-Nori-Api-Key: $SYNTHEFY_NORI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"measurement":"sensors","field":"pressure","feature_fields":["temp","humidity"],"tags":{"site":"A"},"start_time":"2026-01-01T00:00:00Z","end_time":"2026-02-01T00:00:00Z"}' +``` +Either bound may be given alone: `start_time` on its own reads up to now, and `end_time` on its own +reads the `window` before it. + +### Example 4: dry run (preview without writing) + +```bash +curl -X POST http://localhost:8181/api/v3/engine/nori_regress \ + -H "X-Nori-Api-Key: $SYNTHEFY_NORI_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{"measurement":"sensors","field":"pressure","feature_fields":["temp","humidity"],"tags":{"site":"A"},"dry_run":true}' +``` +## Output format + +Each prediction is written as a point: + +- **Measurement:** `output_measurement` (default `_regressed`). +- **Tags:** `model` (the slug), `source` (the input measurement), `target` (the predicted field), + plus every tag of the source series (not only the tags you filtered on), so a point can always be + traced back to the series it was predicted for. +- **Field:** `value` (float): the predicted target value. +- **Timestamp:** the predicted row's own timestamp (nanoseconds). + +Example line protocol: + +``` +sensors_regressed,model=synthefy/nori-30m,source=sensors,target=pressure,site=A value=1001.2 1767225600000000000 +``` +## Cost and metering + +Every gateway call is a billed request, priced from the **training** rows and columns you send +(`max_train_rows` x the number of `feature_fields`), with a per-request floor. Three settings +control what a run costs: + +- `max_train_rows` bounds the priced rows in every call. +- `predict_batch_size` bounds the number of calls: each batch re-sends the same training context and + is billed again, so a larger batch size is cheaper. +- `skip_existing` (on by default) stops a repeating schedule from paying for rows it has already + predicted. With it off, an `every:15m` trigger over a 30-day window re-sends each row roughly + 2,880 times. + +## Querying predictions + +```bash +influxdb3 query --database mydb " +SELECT date_trunc('hour', time) AS hour, count(*) AS predicted, avg(value) AS mean_value +FROM sensors_regressed +WHERE target = 'pressure' +GROUP BY 1 +ORDER BY 1 DESC +" +``` +## Notes + +- **What it predicts:** rows in the window where the target `field` is null but every + `feature_fields` column is present. Rows where the target is already present become the training + set. It never overwrites an existing target value. +- **One series per run:** the plugin counts the distinct tag combinations in the window and fails + before calling the gateway if there is more than one, because predictions are written back at each + row's own timestamp and two series would train as one model. +- **Features only:** Nori sees just the columns you name in `feature_fields`. Row order does not + matter, and no time-derived features are added. +- **Non-finite predictions:** the gateway returns JSON `null` for a row it cannot produce a finite + value for. Those rows are skipped and counted in the log; a batch that is entirely null fails + rather than reporting a successful run that wrote nothing. +- **A partial run keeps what it paid for, and says so:** if a later batch fails, the predictions the + earlier batches already returned are still written, because those batches were already billed. The + run reports `{"status": "partial", "written": N, "remaining": M}` rather than `success`, and the + remaining rows are picked up by the next run. + +## Supported models + +The `model` argument is the Nori gateway slug your API key is granted. It is **required**: the +slug selects a priced model, so the plugin will not choose one on your behalf. Synthefy's own +client and local package take the same position. + +Synthefy publishes the current models, their sizes and their slugs at +[docs.synthefy.com/nori/quickstart#models](https://docs.synthefy.com/nori/quickstart#models). That list is the authoritative one: +it changes when Synthefy releases a variant, and a slug not on it will not route. + +Which model predicts better depends on your data, and the larger ones cost more per request and +take longer to cold-start. Try a couple with `dry_run=true` before committing a schedule to one. + +The bare `synthefy/nori` slug has been retired and no longer routes; the plugin rejects it with a +pointed message rather than letting the gateway answer `404`. One API key from the +[Synthefy console](https://console.synthefy.com/) works for every slug it is granted. + +## Code overview + +### Files + +- `nori_regression.py`: the plugin (metadata docstring and implementation). +- `nori_regression_config_scheduler.toml`: annotated TOML configuration template. +- `test_nori_regression.py`: unit tests (`pytest influxdata/nori_regression/`); no engine or + network needed. +- `requirements.txt`: Python dependencies. +- `manifest.toml`: packaging metadata. + +### Key functions + +- `process_scheduled_call(influxdb3_local, call_time, args)`: scheduled entry point; anchors the + window to `call_time`. +- `process_request(influxdb3_local, query_parameters, request_headers, request_body, args)`: HTTP + entry point; applies the request-body allowlist. +- `_load_config(args, body)`: merges trigger arguments, the TOML file and the allowlisted body keys, + then validates them. +- `_resolve_schema(influxdb3_local, cfg)`: reads column names *and* types, rejecting a + non-numeric target or feature. +- `_resolve_window(cfg, now)`: resolves `window` / `start_time` / `end_time` into one range, + honouring each bound on its own. +- `_regress(...)`: enforces the single-series rule, splits labeled from null-target rows, applies + the caps and `skip_existing`, and batches the gateway calls. +- `_call_nori(...)`: sends the in-context regression request and validates the response. +- `_write_predictions(...)`: writes the predictions with `write_sync` so a write error surfaces + during trigger execution. + +## Troubleshooting + +### Common issues + +Each heading below quotes the text the plugin actually logs or returns, so a message can be +searched for directly. Every failure is logged with a `task_id`; use it to correlate the +caller-facing message with the full detail in `processing_engine_logs`. + +#### Missing API key + +The plugin cannot find a Nori gateway key. + +**Solution:** set `SYNTHEFY_NORI_API_KEY` on the InfluxDB host, or pass an +`X-Nori-Api-Key: ` header when calling the HTTP trigger (see +[Authentication](#authentication-for-the-nori-gateway)). An empty header value is ignored and the +environment variable is used instead. + +#### Gateway returns 403 or 404 + +- **`HTTP 403 ... please check the api-key you provided`:** the key is wrong, revoked, or malformed. +- **`HTTP 404 ... please check the model you provided`:** the `model` slug does not exist or your + key's group was not granted it. Confirm the spelling against + [Supported models](#supported-models). + +**Solution:** re-copy the key from the [Synthefy console](https://console.synthefy.com/) and check +the slug. Neither status is retried, because neither is transient. + +#### Request body may not set ... + +The HTTP request body contained an operator-only parameter (for example `model` or +`target_database`). + +**Solution:** set it as a trigger argument or in the TOML config file. Only the query-shape keys +listed in [HTTP request body parameters](#http-request-body-parameters) may come from the body. + +#### `gateway_url` is not a parameter + +The endpoint moved out of the configuration entirely, because the request carries the Nori API key. + +**Solution:** set `NORI_GATEWAY_URL` on the InfluxDB host. It must be an `https://` URL. + +#### Not enough labeled rows, or nothing to predict + +- **`only N labeled rows (< min_history)`:** fewer than `min_history` rows have both the target and + every feature present. Widen `window`, lower `min_history`, or check that + `measurement`/`field`/`feature_fields`/`tags` select the data you expect. +- **`no rows to predict`:** every target value in the window is already present. The plugin only + fills rows where the target is null. +- **`every row in the window already holds a prediction`:** `skip_existing` did its job. Set + `skip_existing=false` to recompute them with newer training data. + +#### The window holds N series + +The measurement holds more than one series and your `tags` filter did not isolate one, so a single +model would be trained on mixed series. + +**Solution:** add a `tags` filter that selects one series, or create one trigger per series. The +error message lists the first few series it found. + +#### Feature or target column rejected + +A column does not exist, is not a numeric field, or clashes with the target field or the reserved +names `time`/`y`. + +**Solution:** fix the column names, and check the types with +`SELECT column_name, data_type FROM information_schema.columns WHERE table_name = 'sensors'`. + +#### Cold-start latency and timeouts + +The models scale to zero, so the first request after an idle period is slow: measurements have +ranged from roughly one minute to nearly four, with the larger variants slower, and it can return +a `503` or a non-JSON body from the fronting proxy once. + +**Solution:** the default `request_timeout` of `300s` and `max_retries` of `3` are set to absorb +this; a `503`, a `429` and a connection error are retried with backoff. A read timeout is **not** +retried — it has already spent the whole budget, and it usually means the key's group was never +granted the slug. Raise `request_timeout` only if you see genuine timeouts on a warm model. + +## Limitations + +- One series per run; create one trigger per series for a multi-series measurement. Multi-series + imputation in a single run is a possible enhancement. +- Imputes only rows where the target is null; it never overwrites an existing value. +- Prediction quality depends on how well `feature_fields` explain the target. Nori adds no + time-derived features, so this plugin is not a time-series forecaster. +- A feature column whose name contains a space is only reachable via a TOML array or the HTTP JSON + body, not via `--trigger-arguments`. +- Each gateway call is billed; see [Cost and metering](#cost-and-metering). + +## License + +Apache 2.0. + + +## Logging + +Logs are stored in the `_internal` database (or the database where the trigger is created) in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" +``` + +Log columns: +- **event_time**: Timestamp of the log event +- **trigger_name**: Name of the trigger that generated the log +- **log_level**: Severity level (INFO, WARN, ERROR) +- **log_text**: Message describing the action or error + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/notifier.md b/content/shared/influxdb3-plugins/plugins-library/official/notifier.md index c35ecb94f6..97d8a0792a 100644 --- a/content/shared/influxdb3-plugins/plugins-library/official/notifier.md +++ b/content/shared/influxdb3-plugins/plugins-library/official/notifier.md @@ -1,4 +1,5 @@ - + + The Notifier Plugin provides multi-channel notification capabilities for {{% product-name %}}, enabling real-time alert delivery through various communication channels. Send notifications via Slack, Discord, HTTP webhooks, SMS, or WhatsApp based on incoming HTTP requests. Acts as a centralized notification dispatcher that receives data from other plugins or external systems and routes notifications to the appropriate channels. ## Configuration @@ -65,8 +66,8 @@ The `senders_config` object accepts channel configurations where keys are sender - **{{% product-name %}}**: with the Processing Engine enabled. - **Python packages**: - - `httpx` (for HTTP requests) - - `twilio` (for SMS/WhatsApp notifications) + - `httpx` (for HTTP requests) + - `twilio` (for SMS/WhatsApp notifications) ### Installation steps @@ -187,7 +188,7 @@ influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine ``` ### Main functions -#### `process_http_request(influxdb3_local, request_body, args)` +#### `process_request(influxdb3_local, query_parameters, request_headers, request_body, args)` Handles incoming HTTP notification requests. Parses the request body, extracts notification text and sender configurations, and dispatches notifications to configured channels. @@ -237,4 +238,6 @@ For plugin issues, see the Plugins repository [issues page](https://github.com/i ## Find support for {{% product-name %}} The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. -For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. \ No newline at end of file +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/nws-weather.md b/content/shared/influxdb3-plugins/plugins-library/official/nws-weather.md new file mode 100644 index 0000000000..3c1702ce40 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/nws-weather.md @@ -0,0 +1,415 @@ + + +The US NWS Weather Sampler Plugin provides real-time weather data from the National Weather Service API for demonstration and sample data purposes. Fetch live observations from multiple weather stations across the United States with zero authentication required. Perfect for demos, training, testing, and providing realistic IoT-like time-series data streams. + +- **Zero authentication**: No API keys or signup required +- **Real-time data**: Live weather updates from NOAA weather stations +- **Multiple metrics**: Temperature, humidity, wind, pressure, visibility, and more +- **Configurable stations**: Choose from thousands of US weather stations +- **Parallel fetching**: Efficient concurrent data retrieval from multiple stations +- **Statistics tracking**: Built-in monitoring of plugin performance + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. + +If a plugin supports multiple trigger specifications, some parameters may depend on the trigger specification that you use. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Required parameters + +| Parameter | Type | Default | Description | +|------------|--------|----------------------------------------------------|------------------------------------------| +| `stations` | string | KSEA.KORD.KJFK.KDEN.KATL.KDFW.KLAX.KMIA.KPHX.KBOS | Dot-separated list of NWS station IDs | + +### Optional parameters + +| Parameter | Type | Default | Description | +|-----------------------|---------|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `measurement` | string | weather_observations | Destination measurement for weather data | +| `user_agent` | string | InfluxDB3-NWS-Plugin/1.0 | Custom User-Agent header for NWS API requests | +| `use_data_timestamp` | boolean | true | Use timestamp from weather observation data instead of current time when writing data | +| `enable_full_logging` | boolean | false | When `true`, full exception messages are written to logs. When `false` (default), only the exception type is logged, to avoid leaking sensitive values. Enable temporarily for debugging. | + +**Note:** Station IDs are separated by dots (`.`) in trigger arguments. Example: `stations=KSEA.KSFO.KLAX` + +## Software Requirements + +- **{{% product-name %}}**: with the Processing Engine enabled +- **Python packages**: No additional packages required (uses Python standard library only) +- **Network access**: Outbound HTTPS access to `api.weather.gov` + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. No additional Python packages are required for this plugin. + +## Trigger setup + +### Scheduled trigger + +Create a trigger for periodic weather data collection: + +```bash +influxdb3 create trigger \ + --database weather_demo \ + --path "gh:influxdata/nws_weather/nws_weather_sampler.py" \ + --trigger-spec "every:5m" \ + nws_weather_trigger + +# Enable the trigger +influxdb3 enable trigger --database weather_demo nws_weather_trigger +``` +## Example usage + +### Example 1: Basic weather data collection + +Collect weather data from multiple stations every 5 minutes: + +```bash +# Create the trigger +influxdb3 create trigger \ + --database weather_demo \ + --path "gh:influxdata/nws_weather/nws_weather_sampler.py" \ + --trigger-spec "every:5m" \ + --trigger-arguments "stations=KSEA.KSFO.KORD" \ + nws_basic_weather + +# Enable the trigger +influxdb3 enable trigger --database weather_demo nws_basic_weather + +# Query weather data (after a few minutes) +influxdb3 query \ + --database weather_demo \ + "SELECT station_id, temperature_c, relative_humidity_percent, time FROM weather_observations ORDER BY time DESC LIMIT 10" +``` +### Expected output + + station_id | temperature_c | relative_humidity_percent | time + -----------|---------------|---------------------------|----- + KSEA | 15.6 | 72.0 | 2025-11-26T10:00:00Z + KSFO | 18.3 | 65.0 | 2025-11-26T10:00:00Z + KORD | 12.2 | 68.0 | 2025-11-26T10:00:00Z + +### Example 2: Custom measurement name and user agent + +```bash +# Create trigger with custom configuration +influxdb3 create trigger \ + --database weather_demo \ + --path "gh:influxdata/nws_weather/nws_weather_sampler.py" \ + --trigger-spec "every:10m" \ + --trigger-arguments "stations=KJFK.KATL.KDFW,measurement=us_weather,user_agent=MyDemo/1.0" \ + nws_custom_weather +``` +### Example 3: Using current time instead of API observation timestamp + +By default (`use_data_timestamp=true`), the plugin uses the timestamp from the weather observation data provided by the NWS API. However, since `api.weather.gov` may not update data regularly, this can lead to scenarios where no new records are written: + +**Scenario example:** +- Weather station updates data every 10 minutes +- Plugin runs every 3 minutes +- First run: New record written (timestamp from API: 4:10 PM) +- Second run (3 min later): No new record (API still returns data timestamped 4:10 PM - a duplicate that will overwrite the existing record) +- Third run (6 min later): No new record (API still returns data timestamped 4:10 PM - a duplicate that will overwrite the existing record) +- Fourth run (9 min later): New record written (timestamp from API: 4:20 PM) + +To ensure data is written on every plugin execution (useful for demos or populating tables with regular data), set `use_data_timestamp=false`: + +```bash +# Create trigger that writes data on every execution using current time +influxdb3 create trigger \ + --database weather_demo \ + --path "gh:influxdata/nws_weather/nws_weather_sampler.py" \ + --trigger-spec "every:3m" \ + --trigger-arguments "stations=KSEA.KSFO,use_data_timestamp=false" \ + nws_current_time +``` +**Note:** The NWS API data is typically delayed. For example: +- Request at 4:30 PM → Returns data timestamped 4:10 PM +- Request at 4:37 PM → Returns data timestamped 4:25 PM + +## Output data structure + +### Measurement: `weather_observations` + +**Tags:** +- `station_id`: NWS station identifier (for example, "KSEA") +- `station`: Full station name +- `conditions`: Current weather conditions (for example, "Fair", "Partly Cloudy") +- `longitude`: Station longitude (4 decimal places) +- `latitude`: Station latitude (4 decimal places) + +**Fields:** +- `temperature_c` (float): Temperature in Celsius +- `dewpoint_c` (float): Dew point in Celsius +- `wind_speed_kmh` (float): Wind speed in km/h +- `wind_direction_degrees` (float): Wind direction (0-360°) +- `wind_gust_kmh` (float): Wind gust speed in km/h +- `barometric_pressure_pa` (float): Barometric pressure in Pascals +- `visibility_m` (float): Visibility in meters +- `relative_humidity_percent` (float): Relative humidity (0-100%) +- `elevation_m` (float): Station elevation in meters + +**Timestamp:** +- Automatically set from NWS observation time (nanosecond precision) + +### Measurement: `nws_plugin_stats` + +Tracks plugin execution statistics: + +**Tags:** +- `plugin`: Always "nws_weather_sampler" + +**Fields:** +- `success_count` (int64): Number of successful station fetches +- `error_count` (int64): Number of failed station fetches +- `total_count` (int64): Total stations attempted +- `success_rate` (float64): Success percentage (0-100) +- `task_id` (string): Unique identifier for each plugin execution + +## Finding Weather Stations + +### Popular Station IDs + +Major US airports make excellent weather stations: + +**West Coast:** +- `KSEA` - Seattle-Tacoma International, WA +- `KPDX` - Portland International, OR +- `KSFO` - San Francisco International, CA +- `KLAX` - Los Angeles International, CA +- `KSAN` - San Diego International, CA + +**East Coast:** +- `KBOS` - Boston Logan International, MA +- `KJFK` - New York JFK International, NY +- `KEWR` - Newark Liberty International, NJ +- `KPHL` - Philadelphia International, PA +- `KATL` - Atlanta Hartsfield-Jackson International, GA +- `KMIA` - Miami International, FL + +**Central:** +- `KORD` - Chicago O'Hare International, IL +- `KDFW` - Dallas/Fort Worth International, TX +- `KDEN` - Denver International, CO +- `KPHX` - Phoenix Sky Harbor International, AZ +- `KLAS` - Las Vegas McCarran International, NV + +### How to Find Stations + +1. **Airport Codes**: Most US airport weather stations use the ICAO code (4 letters starting with 'K') + - Example: Seattle-Tacoma airport code SEA → Station ID KSEA + +2. **NWS Station List**: Browse available stations at: + - https://www.weather.gov/documentation/services-web-api + - API endpoint: `https://api.weather.gov/stations?state=WA` (replace WA with your state) + +3. **Test a Station**: + ```bash + curl -H "User-Agent: test" https://api.weather.gov/stations/KSEA/observations/latest + ``` +## Example Queries + +### Current temperature across all stations + +```sql +SELECT + station_id, + temperature_c, + time +FROM weather_observations +WHERE time > now() - INTERVAL '1 hour' +ORDER BY time DESC; +``` +### Average temperature by location + +```sql +SELECT + station_id, + AVG(temperature_c) as avg_temp, + MAX(temperature_c) as max_temp, + MIN(temperature_c) as min_temp +FROM weather_observations +WHERE time > now() - INTERVAL '24 hours' +GROUP BY station_id +ORDER BY avg_temp DESC; +``` +### Wind speed trends + +```sql +SELECT + time_bucket(time, INTERVAL '1 hour') as hour, + station_id, + AVG(wind_speed_kmh) as avg_wind_speed, + MAX(wind_gust_kmh) as max_gust +FROM weather_observations +WHERE time > now() - INTERVAL '24 hours' +GROUP BY hour, station_id +ORDER BY hour, station_id; +``` +### Check plugin statistics + +```sql +SELECT + time, + success_count, + error_count, + success_rate +FROM nws_plugin_stats +WHERE time > now() - INTERVAL '1 hour' +ORDER BY time DESC; +``` +## Code overview + +### Files + +- `nws_weather_sampler.py`: The main plugin code containing the scheduled handler for weather data collection + +### Logging + +Logs are stored in the trigger's database in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'nws_weather_trigger'" +``` +### Main functions + +#### `process_scheduled_call(influxdb3_local, call_time, args)` + +Handles scheduled weather data fetching. Fetches observations from configured NWS stations in parallel and writes to InfluxDB. + +Key operations: + +1. Parses configuration from trigger arguments +2. Fetches weather data from stations in parallel using ThreadPoolExecutor +3. Writes weather observations and plugin statistics to InfluxDB +4. Implements comprehensive error handling and logging + +## Troubleshooting + +### Common issues + +#### Issue: No data appearing + +**Solution**: Check trigger status, review plugin logs, and verify network connectivity: + +```bash +# Check trigger status +influxdb3 show summary --database weather_demo --token YOUR_TOKEN + +# Check plugin logs +influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine_logs WHERE log_text LIKE '%NWS%' ORDER BY event_time DESC LIMIT 10" + +# Verify network connectivity +curl -H "User-Agent: test" https://api.weather.gov/stations/KSEA/observations/latest +``` +#### Issue: Rate limiting from NWS API + +**Solution**: The NWS API has generous rate limits. If you encounter rate limiting, reduce polling frequency, reduce number of stations, or add delays between station requests. + +#### Issue: HTTP 404 errors for specific stations + +**Solution**: Station may not exist or be decommissioned. Check station ID spelling and try a different station from the same area. + +#### Issue: JSON decode errors + +**Solution**: NWS API may be temporarily unavailable. Check if station is reporting data: `curl -H "User-Agent: test" https://api.weather.gov/stations/STATION_ID/observations/latest` + +### Debugging tips + +1. **Check trigger status**: + ```bash + influxdb3 show summary --database weather_demo --token YOUR_TOKEN + ``` +2. **Enable/Disable trigger**: + ```bash + influxdb3 disable trigger nws_weather --database weather_demo --token YOUR_TOKEN + influxdb3 enable trigger nws_weather --database weather_demo --token YOUR_TOKEN + ``` +## Integration with Grafana + +1. **Add {{% product-name %}} Data Source** in Grafana + - URL: `http://localhost:8181` + - Database: `weather_demo` + - Token: Your admin token + +2. **Create a Dashboard Panel**: + + **Query:** + ```sql + SELECT + time, + station_id, + temperature_c + FROM weather_observations + WHERE time > now() - INTERVAL '1 hour' + ORDER BY time + ``` + **Visualization**: Time series graph + +3. **Watch real-time weather data update** every 5 minutes! + +## Common Customizations + +### Change Polling Frequency + +**Every 1 minute** (for faster demos): +```bash +influxdb3 delete trigger nws_weather --database weather_demo --token YOUR_TOKEN + +influxdb3 create trigger \ + --trigger-spec "every:1m" \ + --path "gh:influxdata/nws_weather/nws_weather_sampler.py" \ + --database weather_demo \ + --trigger-arguments "use_data_timestamp=false" \ + --token YOUR_TOKEN \ + nws_weather_fast +``` +### Select Different Cities + +Pick from major US cities: +```bash +# West Coast +--trigger-arguments "stations=KSEA.KPDX.KSFO.KLAX.KSAN" + +# East Coast +--trigger-arguments "stations=KBOS.KJFK.KPHL.KBWI.KATL" + +# Central +--trigger-arguments "stations=KORD.KMSP.KSTL.KCLE.KDEN" + +# Texas +--trigger-arguments "stations=KDFW.KIAH.KAUS.KSAT.KHOU" +``` +## API Reference + +**National Weather Service API** +- Base URL: https://api.weather.gov +- Documentation: https://www.weather.gov/documentation/services-web-api +- Authentication: None required (User-Agent header required) +- Rate Limits: Not publicly documented, but generous +- Data Update Frequency: Typically every 1-5 minutes +- Coverage: Thousands of stations across the United States + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/opcua.md b/content/shared/influxdb3-plugins/plugins-library/official/opcua.md new file mode 100644 index 0000000000..cf8ec7c05f --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/opcua.md @@ -0,0 +1,895 @@ + + +scheduled opcua, ingestion, iot, industrial {{% product-name %}} + +> **Note:** This plugin requires {{% product-name %}}.8.2 or later. + + +The OPC UA Plugin enables periodic ingestion of OPC UA node values into {{% product-name %}}. Connect to an OPC UA server, read current values from configured nodes, and automatically write them as time-series data with support for automatic data type detection. Designed for industrial IoT scenarios — PLCs, SCADA systems, CNC machines, and other OPC UA-enabled equipment. + +**Key characteristics:** +- **Polling-based**: Reads current node values on each scheduled trigger call (not subscription-based) +- **Two operating modes**: Explicit node listing for precise control, or browse mode for auto-discovery of thousands of devices +- **Auto type detection**: OPC UA data types are automatically mapped to InfluxDB field types +- **Namespace URI support**: Use stable namespace URIs instead of numeric indexes that may change on server restart +- **Quality filtering**: Accept or reject OPC UA values based on quality category (good, uncertain, bad) +- **Persistent connection**: OPC UA connection is cached and reused across scheduled calls with automatic reconnect on failure +- **Scalar values only**: Arrays, structures, and other complex OPC UA types are not supported + +## Operating Modes + +The plugin supports two mutually exclusive modes for specifying which OPC UA nodes to read: + +### Explicit nodes mode + +Specify each node ID manually. Best for small setups where you need precise control over field names and types. All nodes are written as fields in a **single data point** per trigger execution. + +### Browse mode + +Automatically discover devices and their variables by browsing the OPC UA address space from a root node. Best for large-scale deployments with hundreds or thousands of devices sharing the same variable structure. The plugin maps the Object hierarchy to InfluxDB tags and writes **one data point per unique tag combination** per trigger execution. + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. This plugin supports TOML configuration files for complex setups, which can be specified using the `config_file_path` parameter. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Required parameters + +| Parameter | Type | TOML Section | Description | +|--------------|--------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `server_url` | string | `[opcua]` | OPC UA server endpoint URL (for example, `opc.tcp://localhost:4840`). Must use the `opc.tcp://` or `opc.tls://` scheme; other schemes (`file://`, `http://`, etc.) are rejected. | +| `table_name` | string | `[opcua]` | InfluxDB measurement name for storing data | + +**One of the following is required** (mutually exclusive): + +| Parameter | Type | TOML Section | Description | +|---------------|---------|------------------|-------------------------------------------------------------------------------------------------------------------------------------| +| `nodes` | string | `[opcua.nodes]` | Space-separated node mappings. Format: `field_name:namespace:identifier[:type]`. See [Node mapping](#node-mapping-format). | +| `browse_root` | string | `[opcua.browse]` | Root node ID for auto-discovery mode (for example, `ns=2;s=Devices` or `nsu=;s=Devices`). See [Browse mode](#browse-mode-parameters). | + +### Namespace parameters + +| Parameter | Type | Default | TOML Section | Description | +|--------------|--------|---------|-----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `namespaces` | string | none | `[opcua.namespaces]` | Space-separated namespace alias mappings. Format: `alias=namespace_uri`. Aliases can be used instead of numeric namespace indexes in `nodes` and `tag_nodes` for stable configuration that survives server restarts. The plugin resolves URIs to numeric indexes at connection time. | + +**CLI format:** + +```bash +namespaces="siemens=urn:vendor:s7 beckhoff=urn:beckhoff:ua" +nodes="temperature:siemens:s=SpindleTemp pressure:beckhoff:s=CoolantPressure:float" +``` +**TOML format:** + +In TOML configuration, namespace aliases are not used — write node IDs directly with `nsu=;...` format: + +```toml +[opcua.nodes] +temperature = "nsu=urn:vendor:s7;s=SpindleTemp" +pressure = ["nsu=urn:beckhoff:ua;s=CoolantPressure", "float"] +``` +The plugin resolves `nsu=` URIs to numeric `ns=` indexes after connecting to the server. + +### Browse mode parameters + +| Parameter | Type | Default | TOML Section | Description | +|--------------------|--------|-----------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `browse_root` | string | none | `[opcua.browse]` | Root node ID for auto-discovery. Accepts a numeric namespace (`ns=2;s=Devices`) or a namespace URI (`nsu=;s=Devices`) resolved at connection time for restart stability. See [Namespace URI resolution](#namespace-uri-resolution). | +| `browse_depth` | int | 2 | `[opcua.browse]` | Maximum browse depth. 1 = direct children, 2 = children of children, etc. | +| `browse_cache_ttl` | int | 3600 | `[opcua.browse]` | Seconds to cache the discovered node set before re-browsing. Discovery runs on the first call and every `browse_cache_ttl` seconds thereafter; cached node IDs are read in between. Set low for a fast-changing address space, high to run discovery infrequently while collecting on a short trigger interval. See [Discovery caching](#discovery-caching). | +| `path_tags` | list | `[]` | `[opcua.browse]` | Tag names mapping Object hierarchy levels to InfluxDB tags. First entry maps to depth-1 Objects, second to depth-2, etc. Length must be strictly less than `browse_depth`. Objects beyond `path_tags` length become field name prefixes. **Required in TOML** (specify `[]` explicitly for no hierarchy tags); defaults to `[]` in CLI. | +| `filter` | string | none | `[opcua.browse]` | Regex pattern to filter discovered variable names. Only matching variables are included. Matches the original Variable browse name, not the prefixed field name (see [filter in nested hierarchies](#filter-in-nested-hierarchies)). | +| `exclude_branches` | string | none | `[opcua.browse]` | Regex pattern to exclude Object nodes (branches) by browse name. Matched Objects and their entire subtree are skipped. Works at all browse depths. See [Exclude branches](#exclude-branches). | +| `browse_tags` | list | none | `[opcua.browse]` | Variable names that should be stored as InfluxDB tags instead of fields. Values are converted to strings. See [Browse tags](#browse-tags). | +| `name_separator` | string | none | `[opcua.browse]` | Separator for splitting Variable browse names into segments for tag extraction. Required when `name_tags` is set. See [Name tags](#name-tags). | +| `name_tags` | list | none | `[opcua.browse]` | Tag names extracted from leading segments of Variable browse names split by `name_separator`. Remaining segments form the field name (joined with `_`). See [Name tags](#name-tags). | + +> **Note:** integer parameters (`browse_depth`, `browse_cache_ttl`) also accept numeric strings, so `browse_depth = "2"` is valid in TOML. `browse_cache_ttl` is capped at 30 days. + +### Tag parameters + +| Parameter | Type | Default | TOML Section | Description | +|----------------|--------|---------|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `default_tags` | string | none | `[opcua.default_tags]`| Space-separated static tags added to every point. Format: `key=value`. | +| `tag_nodes` | string | none | `[opcua.tag_nodes]` | Space-separated tag node mappings. Format: `tag_name:namespace:identifier`. Values are read from OPC UA nodes at each execution and used as string tags. See [Static and dynamic tags](#static-and-dynamic-tags). | + +### Quality filter parameters + +| Parameter | Type | Default | TOML Section | Description | +|------------------|--------|---------|--------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `quality_filter` | string | "good" | `[opcua]` | Space-separated list of OPC UA quality categories to accept: `good`, `uncertain`, `bad`. Values with non-matching quality are skipped and logged to `opcua_exceptions`. | + +**CLI format:** + +```bash +quality_filter="good uncertain" +``` +**TOML format:** + +```toml +[opcua] +quality_filter = ["good", "uncertain"] +``` +### Authentication parameters + +| Parameter | Type | Default | TOML Section | Description | +|------------|--------|---------|-----------------|-------------------------------------------------------------------| +| `username` | string | none | `[opcua.auth]` | Username for UserToken authentication | +| `password` | string | none | `[opcua.auth]` | Password for UserToken authentication | + +**Note:** Both `username` and `password` must be provided together. + +**Security:** When `security_policy` is not set, the connection is unencrypted and credentials are sent in cleartext. The plugin refuses to send `username`/`password` in that case unless `allow_insecure_auth` is explicitly set to `true`. Prefer configuring a `security_policy` for an encrypted connection. + +### Security parameters + +| Parameter | Type | Default | TOML Section | Description | +|-------------------|--------|------------------|---------------------|--------------------------------------------------------------------------------------------------------------------| +| `security_policy` | string | none | `[opcua.security]` | OPC UA security policy: `Basic128Rsa15`, `Basic256`, `Basic256Sha256`, `Aes128Sha256RsaOaep`, `Aes256Sha256RsaPss` | +| `security_mode` | string | "SignAndEncrypt" | `[opcua.security]` | OPC UA security mode: `Sign`, `SignAndEncrypt`. Used when `security_policy` is set. | +| `certificate` | string | none | `[opcua.security]` | Path to client certificate (DER format). Required when `security_policy` is set. | +| `private_key` | string | none | `[opcua.security]` | Path to client private key (PEM format). Required when `security_policy` is set. | + +**Note:** When `security_policy` is set, both `certificate` and `private_key` are required. The OPC UA server must trust the client certificate. + +### Advanced parameters + +| Parameter | Type | Default | TOML Section | Description | +|------------------------|--------|---------|--------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `config_file_path` | string | none | CLI only | Path to TOML config file (absolute or relative to `PLUGIN_DIR`) | +| `config_cache_ttl` | int | 3600 | `[opcua]` | Seconds to cache the parsed configuration (max 2592000). Controls how quickly credential, endpoint, table, or filter changes take effect. Independent of `browse_cache_ttl`. | +| `disable_config_cache` | bool | false | `[opcua]` | Reload configuration on every call instead of caching for `config_cache_ttl` seconds. Also disables caching of the discovered browse structure, so in browse mode the address space is re-walked on every call. Useful during development. | +| `allow_insecure_auth` | bool | false | `[opcua]` | Permit sending `username`/`password` when `security_policy` is not set (credentials sent in cleartext over an unencrypted connection). Only enable on a trusted network. | +| `enable_full_logging` | bool | false | `[opcua]` | When `true`, full exception messages are written to logs. When `false` (default), only the exception type is logged, to avoid leaking sensitive values (credentials, payloads, paths). Enable temporarily for debugging. | + +### Static and dynamic tags + +The plugin supports two ways to attach tags to every written data point: + +- **Static tags** (`default_tags`): Fixed key-value pairs defined at configuration time. Do not change between trigger calls. +- **Dynamic tags** (`tag_nodes`): Values read from OPC UA nodes on each trigger execution, then converted to strings and used as tag values. Useful for identifiers stored on the device itself — serial number, model name, firmware version, or any other variable property. + +Dynamic tags are merged with static tags before writing. If a key appears in both, the dynamic value takes precedence. + +**CLI format:** + +```bash +# Static tags +default_tags="location=factory_1 line=A" + +# Dynamic tags — same syntax as nodes, type suffix is ignored (values always stored as strings) +tag_nodes="machine_id:2:s=MachineID serial:2:s=SerialNumber" +``` +**TOML format:** + +```toml +[opcua.default_tags] +location = "factory_1" +line = "A" + +[opcua.tag_nodes] +machine_id = "ns=2;s=MachineID" +serial = "ns=2;s=SerialNumber" +# List format is accepted but type is ignored — values are always read as strings +# firmware = ["ns=2;s=FirmwareVersion", "string"] +``` +### Browse tags + +In browse mode, some discovered variables may represent identifiers (room name, zone, asset ID) rather than measurements. Use `browse_tags` to store them as InfluxDB tags instead of fields. + +**CLI format:** + +```bash +browse_tags="room zone" +``` +**TOML format:** + +```toml +[opcua.browse] +browse_root = "ns=2;s=Sensors" +browse_depth = 2 +browse_tags = ["room", "zone"] +``` +**Example:** + +Given this OPC UA address space: +``` +Sensors + +-- Sensor_001 + | +-- room (value="Building_A") -> tag + | +-- zone (value="Zone_3") -> tag + | +-- temperature (value=22.5) -> field + | +-- humidity (value=45.0) -> field +``` +The plugin writes: +``` +sensor_data,device=Sensor_001,room=Building_A,zone=Zone_3 temperature=22.5,humidity=45.0 +``` +### Exclude branches + +Use `exclude_branches` to skip entire Object nodes (and all their children) during auto-discovery. The pattern is a Python regex matched against Object browse names using `re.search()` (partial match). Works at all browse depths — both within `path_tags` range (skipping entire device groups) and beyond (skipping sub-object branches that would become field prefixes). + +**CLI format:** + +```bash +exclude_branches="Debug_.*|Test_.*" +``` +**TOML format:** + +```toml +[opcua.browse] +browse_root = "ns=2;s=Factory" +browse_depth = 3 +path_tags = ["line", "station"] +exclude_branches = "Debug_.*|Test_.*" +``` +**Example:** + +Given this OPC UA address space with `path_tags = ["line", "station"]`: +``` +Factory + +-- Line_A + | +-- Station_01 -> included + | +-- Station_02 -> included + | +-- Debug_Station -> EXCLUDED (matches "Debug_.*") + +-- Test_Line -> EXCLUDED (matches "Test_.*") + +-- Station_01 -> not discovered (parent excluded) +``` +The plugin writes only data from non-excluded branches: +``` +factory_data,line=Line_A,station=Station_01 Temperature=42.5,Pressure=3.2 +factory_data,line=Line_A,station=Station_02 Temperature=38.1,Pressure=2.8 +``` +`Debug_Station` and `Test_Line` (with all its children) are completely skipped. + +**Note:** `exclude_branches` filters Object nodes (branches), while `filter` filters Variable nodes (leaves). They are independent and can be used together. + +### Name tags + +Use `name_tags` with `name_separator` when Variable browse names encode hierarchy in a flat format (for example, `BuildingA.Zone3.Temperature`). The plugin splits each Variable name by the separator, extracts leading segments as InfluxDB tags, and uses remaining segments as the field name. Variables are then grouped by extracted tag combinations — one point per unique combination. + +Variables that don't have enough segments (fewer than `len(name_tags) + 1`) are skipped with a warning. + +**CLI format:** + +```bash +name_separator="." +name_tags="building zone" +``` +**TOML format:** + +```toml +[opcua.browse] +browse_root = "ns=2;s=FlatSensors" +browse_depth = 1 +path_tags = [] +name_separator = "." +name_tags = ["building", "zone"] +``` +**Example:** + +Given Variables directly under `browse_root`: +``` +FlatSensors + +-- BuildingA.Zone3.Temperature (value=22.5) + +-- BuildingA.Zone3.Humidity (value=45.0) + +-- BuildingB.Zone1.Temperature (value=20.1) + +-- BuildingB.Zone1.Humidity (value=50.2) +``` +The plugin splits by `.`, extracts 2 leading segments as tags, and groups: +``` +flat_sensors,building=BuildingA,zone=Zone3 Temperature=22.5,Humidity=45.0 +flat_sensors,building=BuildingB,zone=Zone1 Temperature=20.1,Humidity=50.2 +``` +**Note:** Tag names in `name_tags` must not overlap with `path_tags` tag names. + +### File path resolution + +All file paths in the plugin (configuration file, certificates) follow the same resolution logic: + +- **Absolute paths** (for example, `/etc/opcua/client.der`) are used as-is +- **Relative paths** (for example, `certs/client.der`) are resolved from `PLUGIN_DIR` environment variable + +If a relative path is specified and `PLUGIN_DIR` is not set, the plugin will return an error. + +### Example TOML configuration + +[opcua_config_example.toml](https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/opcua/opcua_config_example.toml) - comprehensive configuration example with multiple scenarios + +### TOML section reference + +| TOML section | Description | +|------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `[opcua]` | Required. `server_url`, `table_name`, `quality_filter`, `config_cache_ttl`, `disable_config_cache`, `allow_insecure_auth` | +| `[opcua.default_tags]` | Static tags as `key = "value"` pairs | +| `[opcua.namespaces]` | Namespace alias mappings as `alias = "uri"` pairs. Validated but not used for alias substitution in TOML — use `nsu=;...` directly in node IDs | +| `[opcua.tag_nodes]` | Dynamic tags from OPC UA nodes | +| `[opcua.nodes]` | Explicit node mappings (mutually exclusive with `[opcua.browse]`) | +| `[opcua.browse]` | Browse mode settings: `browse_root`, `browse_depth`, `browse_cache_ttl`, `path_tags`, `filter`, `exclude_branches`, `browse_tags`, `name_separator`, `name_tags` | +| `[opcua.security]` | `security_policy`, `security_mode`, `certificate`, `private_key` | +| `[opcua.auth]` | `username`, `password` | + +## Node Mapping Format + +### CLI arguments format + +``` +field_name:namespace:identifier[:type] +``` +| Component | Required | Description | +|--------------|----------|------------------------------------------------------------------------------------------| +| `field_name` | yes | InfluxDB field name for this node value | +| `namespace` | yes | OPC UA namespace index (non-negative integer) or alias defined in `namespaces` parameter | +| `identifier` | yes | OPC UA node identifier with type prefix (`s=`, `i=`, `g=`, `b=`) | +| `type` | no | Explicit type override: `int`, `uint`, `float`, `string`, `bool` | + +When namespace is a numeric index, the node ID is built as `ns=;`. +When namespace is an alias, the corresponding URI is looked up in `namespaces` and the node ID is built as `nsu=;`, then resolved to a numeric index at connection time. + +**Examples:** + +```bash +# Auto-detected types +"temperature:2:s=SpindleTemp pressure:2:s=CoolantPressure" + +# With explicit type override +"rpm:2:i=1234:uint status:3:s=MachineStatus:string" + +# Using namespace alias (requires namespaces="siemens=urn:vendor:s7") +"temperature:siemens:s=SpindleTemp" +``` +### TOML format + +```toml +[opcua.nodes] +# Auto-detected type (string format) +temperature = "ns=2;s=SpindleTemp" + +# Explicit type override (list format) +rpm = ["ns=2;s=SpindleRPM", "uint"] + +# Using namespace URI (resolved at connection time) +pressure = "nsu=urn:vendor:s7;s=CoolantPressure" +``` +### Type auto-detection + +When no explicit type is specified, the plugin automatically maps OPC UA data types to InfluxDB field types: + +| OPC UA VariantType | InfluxDB type | LineBuilder method | +|---------------------------------------|---------------|--------------------| +| Boolean | bool | `bool_field()` | +| SByte, Int16, Int32, Int64 | int | `int64_field()` | +| Byte, UInt16, UInt32, UInt64 | uint | `uint64_field()` | +| Float, Double | float | `float64_field()` | +| String, DateTime, and all other types | string | `string_field()` | + +## Browse Mode + +Browse mode automatically discovers devices and their variables by traversing the OPC UA address space hierarchy. This is designed for large-scale deployments where manually listing every node ID is impractical. + +### How browse works + +The plugin uses `path_tags` to map Object hierarchy levels to InfluxDB tags: + +``` +browse_root (specified by browse_root parameter) + +-- Device_001 (Object, depth 1) -> path_tags[0] = "device" -> tag: device=Device_001 + | +-- Temperature (Variable) -> field + | +-- Pressure (Variable) -> field + | +-- Status (Variable) -> field + +-- Device_002 (Object, depth 1) -> path_tags[0] = "device" -> tag: device=Device_002 + | +-- Temperature (Variable) -> field + +-- ... (hundreds/thousands of devices) +``` +- **Object nodes** at depths within `path_tags` range become **tag values** — the Object browse name is stored as the value of the corresponding tag +- **Object nodes** beyond `path_tags` range become **field name prefixes** (for example, `Position_X`) +- **Variable nodes** become **fields** — their browse name (with optional prefix) becomes the field name +- **Variable nodes** directly under `browse_root` (with `path_tags = []`) are written as a single point without hierarchy tags + +Each unique combination of tag values produces one data point. All Variables within that combination are collected as fields of that point. Tags on the final data point come from multiple sources (in merge order): `default_tags`, `tag_nodes`, `path_tags`, `browse_tags`, and `name_tags`. See the corresponding parameter sections below for details. + +### Nested hierarchies + +For servers with deeper nesting, increase `browse_depth` and configure `path_tags` to control which Object levels become tags vs field prefixes: + +```bash +# path_tags = ["device"], browse_depth = 3 +browse_root (depth 0) + +-- Robot_001 (Object, depth 1) -> tag: device=Robot_001 + +-- Speed (Variable, depth 2) -> field: "Speed" + +-- Position (Object, depth 2) -> field prefix: "Position_" + | +-- X (Variable, depth 3) -> field: "Position_X" + | +-- Y (Variable, depth 3) -> field: "Position_Y" + | +-- Z (Variable, depth 3) -> field: "Position_Z" + +-- Gripper (Object, depth 2) -> field prefix: "Gripper_" + +-- Force (Variable, depth 3) -> field: "Gripper_Force" +``` +Result — one data point per device, all Variables (including nested) collected as fields: +``` +robot_data,device=Robot_001 Speed=100i,Position_X=1.5,Position_Y=2.3,Position_Z=0.8,Gripper_Force=5.2 +``` +With multi-level `path_tags = ["line", "station"]` and `browse_depth = 3`, each unique line×station combination produces one data point: + +``` +browse_root (depth 0) + +-- Line_A (Object, depth 1) -> tag: line=Line_A + | +-- Station_01 (Object, depth 2) -> tag: station=Station_01 + | | +-- Temperature (Variable) -> field + | | +-- Pressure (Variable) -> field + | +-- Station_02 (Object, depth 2) -> tag: station=Station_02 + | +-- Temperature (Variable) -> field + | +-- Pressure (Variable) -> field + +-- Line_B (Object, depth 1) -> tag: line=Line_B + +-- Station_01 (Object, depth 2) -> tag: station=Station_01 + +-- Temperature (Variable) -> field +``` +Result — one data point per unique tag combination (3 points): +``` +factory_data,line=Line_A,station=Station_01 Temperature=42.5,Pressure=3.2 +factory_data,line=Line_A,station=Station_02 Temperature=38.1,Pressure=2.8 +factory_data,line=Line_B,station=Station_01 Temperature=40.0 +``` +Objects beyond `path_tags` length become field name prefixes with underscore separator (for example, `Position_X`). + +### Browse structure caching + +The discovered node structure (which devices have which variables) is cached for **1 hour**. Only values are re-read on each trigger call. This means: +- First call: browse + read (slower) +- Subsequent calls: read only (fast) +- After 1 hour: browse is repeated to pick up any new devices or variables +- Config reload also invalidates the browse cache + +### TOML configuration for browse mode + +```toml +[opcua] +server_url = "opc.tcp://192.168.1.100:4840" +table_name = "cnc_data" + +[opcua.browse] +browse_root = "ns=2;s=Devices" +browse_depth = 2 +path_tags = ["device"] +# filter = "Temperature|Pressure|Status" # optional regex +# browse_tags = ["room", "zone"] # optional: store these as tags instead of fields + +[opcua.default_tags] +location = "factory_1" +``` +### Browse mode example output + +Given 3 devices with 3 variables each, the plugin writes: + +``` +cnc_data,location=factory_1,device=Device_001 Temperature=42.5,Pressure=3.2,Status="running" +cnc_data,location=factory_1,device=Device_002 Temperature=38.1,Pressure=2.8,Status="idle" +cnc_data,location=factory_1,device=Device_003 Temperature=40.0,Pressure=3.0,Status="running" +``` +## Data requirements + +The plugin automatically creates the target measurement table on first write. No pre-existing schema is required. + +### OPC UA node requirements + +- **Scalar values only**: The plugin reads scalar (single) values. Arrays, structures, ExtensionObjects, and other complex types are not supported — they will be converted to string representation. +- **Readable nodes**: Nodes must have the `Read` access level. Nodes with `Bad` status codes are skipped with errors logged. +- **Explicit mode**: Node IDs must be known in advance. +- **Browse mode**: The `browse_root` node must exist and contain Object/Variable child nodes. + +## Software Requirements + +- **{{% product-name %}}**: with the Processing Engine enabled +- **Python packages**: + - `asyncua` (async Python OPC UA client library) + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install required Python packages: + + ```bash + influxdb3 install package asyncua + ``` +## Trigger setup + +### Scheduled reading with TOML configuration + +Recommended for production use: + +```bash +# 1. Set PLUGIN_DIR environment variable +export PLUGIN_DIR=~/.plugins + +# 2. Copy and edit configuration file +cp opcua_config_example.toml $PLUGIN_DIR/my_opcua_config.toml +# Edit my_opcua_config.toml with your OPC UA server and node settings + +# 3. Create the trigger +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/opcua/opcua.py \ + --trigger-spec "every:10s" \ + --trigger-arguments config_file_path=my_opcua_config.toml \ + opcua_ingestion +``` +### Scheduled reading with command-line arguments (explicit mode) + +For simple setups: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/opcua/opcua.py \ + --trigger-spec "every:5s" \ + --trigger-arguments 'server_url=opc.tcp://192.168.1.100:4840,table_name=cnc_machine,nodes=temperature:2:s=SpindleTemp pressure:2:s=CoolantPressure:float rpm:2:i=1234:uint,default_tags=location=factory_1,tag_nodes=machine_id:2:s=MachineID serial:2:s=SerialNumber' \ + opcua_cnc +``` +### Explicit mode with namespace aliases + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/opcua/opcua.py \ + --trigger-spec "every:5s" \ + --trigger-arguments 'server_url=opc.tcp://192.168.1.100:4840,table_name=cnc_machine,namespaces=siemens=urn:vendor:s7,nodes=temperature:siemens:s=SpindleTemp pressure:siemens:s=CoolantPressure:float' \ + opcua_ns_alias +``` +### Browse mode with command-line arguments + +For auto-discovery of many devices: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/opcua/opcua.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'server_url=opc.tcp://192.168.1.100:4840,table_name=cnc_data,browse_root=ns=2;s=Devices,browse_depth=2,path_tags=device,default_tags=location=factory_1' \ + opcua_browse +``` +### Browse mode with multi-level path_tags + +Map multiple Object hierarchy levels to tags: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/opcua/opcua.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'server_url=opc.tcp://192.168.1.100:4840,table_name=factory_data,browse_root=ns=2;s=Factory,browse_depth=3,path_tags=line station,default_tags=plant=north' \ + opcua_multi_level +``` +### Browse mode with filter and browse_tags + +Read only specific variables from each device, storing some as tags: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/opcua/opcua.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'server_url=opc.tcp://192.168.1.100:4840,table_name=plc_data,browse_root=ns=3;s=Floor2,browse_depth=2,path_tags=device,filter=Temperature|Pressure|Status|room,browse_tags=room' \ + opcua_filtered +``` +### Browse mode with exclude_branches + +Skip specific Object branches during auto-discovery: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/opcua/opcua.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'server_url=opc.tcp://192.168.1.100:4840,table_name=factory_data,browse_root=ns=2;s=Factory,browse_depth=3,path_tags=line station,exclude_branches=Debug_.*|Test_.*,default_tags=plant=north' \ + opcua_exclude +``` +### Browse mode with name_tags (flat namespace) + +Extract tags from Variable browse names in flat namespaces: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/opcua/opcua.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'server_url=opc.tcp://192.168.1.100:4840,table_name=flat_sensors,browse_root=ns=2;s=FlatSensors,browse_depth=1,path_tags=,name_separator=.,name_tags=building zone' \ + opcua_flat +``` +### Secure connection with certificates + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/opcua/opcua.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'server_url=opc.tcp://secure-plc.company.com:4840,table_name=plc_data,nodes=speed:3:s=Motor.Speed torque:3:s=Motor.Torque,security_policy=Basic256Sha256,security_mode=SignAndEncrypt,certificate=certs/client.der,private_key=certs/client.pem,username=operator,password=secret' \ + secure_opcua +``` +### Accept uncertain quality values + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/opcua/opcua.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'server_url=opc.tcp://192.168.1.100:4840,table_name=sensor_data,nodes=temperature:2:s=Temperature pressure:2:s=Pressure,quality_filter=good uncertain' \ + opcua_with_uncertain +``` +## Statistics and Monitoring + +The plugin tracks statistics and writes them to the `opcua_stats` table every 10 plugin calls. + +### opcua_stats Table + +| Field | Type | Description | +|------------------------|-------|------------------------------------------------| +| `server_url` (tag) | tag | OPC UA server URL | +| `table_name` (tag) | tag | Target InfluxDB measurement | +| `nodes_read` | int | Nodes successfully read in current period | +| `nodes_failed` | int | Nodes failed to read in current period | +| `points_written` | int | Data points written in current period | +| `success_rate` | float | Success rate for current period (%) | +| `total_nodes_read` | int | Total nodes successfully read (all time) | +| `total_nodes_failed` | int | Total nodes failed (all time) | +| `total_points_written` | int | Total data points written (all time) | +| `total_success_rate` | float | Total success rate (all time, %) | + +### Querying Statistics + +```bash +# Get latest statistics +influxdb3 query --database mydb \ + "SELECT * FROM opcua_stats ORDER BY time DESC LIMIT 10" + +# Success rate over time +influxdb3 query --database mydb \ + "SELECT server_url, success_rate, nodes_read, nodes_failed, points_written + FROM opcua_stats + WHERE time > now() - INTERVAL '1 hour' + ORDER BY time DESC" +``` +## Error Handling + +Node read errors, type conversion failures, and quality filter rejections are logged to the `opcua_exceptions` table: + +### opcua_exceptions Table + +| Field | Type | Description | +|----------------------|--------|--------------------------------------------------------------------------| +| `node_id` (tag) | tag | OPC UA node ID that failed | +| `field_name` (tag) | tag | Configured field name for this node | +| `error_type` (tag) | tag | Error type: `NodeReadError`, `TypeConversionError`, `QualityFilterError` | +| `error_message` | string | Detailed error message | + +### Checking for Errors + +```bash +influxdb3 query --database mydb \ + "SELECT * FROM opcua_exceptions ORDER BY time DESC LIMIT 10" +``` +## Important Behaviors + +### Timestamp handling + +All fields within a single data point share the **same timestamp** — the latest OPC UA timestamp among successfully read nodes that passed the quality filter. Per node, `SourceTimestamp` is preferred; if unavailable, `ServerTimestamp` is used. The maximum across all accepted nodes becomes the point timestamp. If no OPC UA timestamps are available, system time is used. + +### Connection lifecycle + +The OPC UA connection is **persistent** — it is cached and reused across scheduled trigger calls. This avoids the overhead of connecting/disconnecting on every call. The connection is automatically re-established if it fails during a read operation (one reconnect attempt per call). The connection is dropped and all caches are cleared if a call fails after reconnect. + +### Dynamic tags ignore quality_filter + +The `tag_nodes` feature only accepts values with **"good"** quality, regardless of the `quality_filter` setting. This is by design — tag values are identifiers (serial number, machine ID) and must be reliable. An uncertain or bad serial number should not become an InfluxDB tag. + +### Quality filter with "bad" category + +Including `"bad"` in `quality_filter` has no practical effect. Nodes with bad StatusCode have no value on the OPC UA server — the plugin cannot write a non-existent value. These nodes will still generate `NodeReadError` exceptions even if "bad" is in the filter. + +### filter applies to browse_tags variables + +The `filter` regex applies to **all** discovered variables, including those listed in `browse_tags`. If you use `filter` and `browse_tags` together, the browse_tags variable names must also match the filter pattern, otherwise they won't be discovered. + +Example: with `browse_tags = ["room"]` and `filter = "temp|hum"`, the `room` variable will **not** be found. Use `filter = "room|temp|hum"` instead. + +### filter in nested hierarchies + +The `filter` regex is applied to the **original Variable browse name**, not to the resulting prefixed field name. In nested hierarchies where Objects beyond `path_tags` become field prefixes, `filter` matches the variable's own name at its level in the tree. + +Example with `path_tags = ["device"]` and `browse_depth = 3`: +``` +Robot_001 (Object, depth 1) -> tag: device=Robot_001 + +-- Position (Object, depth 2) -> prefix: "Position_" + | +-- X (Variable) -> field: "Position_X" + | +-- Y (Variable) -> field: "Position_Y" + +-- Speed (Variable) -> field: "Speed" +``` +- `filter = "X|Speed"` matches `X` and `Speed` — writes fields `Position_X` and `Speed` +- `filter = "Position_X"` matches **nothing** — `Position_X` is the field name, not the browse name + +### exclude_branches vs filter + +`exclude_branches` and `filter` are independent parameters that operate on different node types: + +- **`exclude_branches`** matches **Object** node browse names — skips the entire branch (Object + all children) +- **`filter`** matches **Variable** node browse names — includes/excludes individual leaf values + +They can be used together. For example, `exclude_branches = "Debug_.*"` with `filter = "Temperature|Pressure"` will first skip all debug Object branches, then among remaining branches only discover Temperature and Pressure variables. + +### browse_tags follow quality_filter + +Unlike `tag_nodes` which always require "good" quality, `browse_tags` variables are subject to the normal `quality_filter` setting. If `quality_filter = ["good", "uncertain"]`, browse_tags with uncertain quality will be accepted. + +### Groups with only browse_tags produce no data point + +If all variables in a device group are listed in `browse_tags` (no regular fields remain), no data point is written for that group. InfluxDB requires at least one field per point — browse_tags alone are not sufficient. + +### Discovery caching + +Two independent caches, each with its own TTL: + +- **Parsed configuration** — cached for `config_cache_ttl` seconds (default **1 hour**). Controls how quickly a change to credentials, endpoint, table, or filters is picked up. +- **Discovered browse structure** — cached for `browse_cache_ttl` seconds (default **1 hour**). Discovery runs on the first call and every `browse_cache_ttl` seconds thereafter; cached node IDs are read in between. + +This means: +- Set a low `browse_cache_ttl` for a fast-changing address space, or set it much larger than the trigger interval so discovery runs rarely while collection stays frequent — the recommended pattern for large or high-latency servers where a full browse is expensive +- New devices added to the OPC UA server appear after at most `browse_cache_ttl` seconds +- Changes to the config take effect after at most `config_cache_ttl` seconds. Changes to the browse-relevant part (`server_url`, `browse_root`, `browse_depth`, `filter`, `exclude_branches`, …) also trigger an immediate re-browse on the next config reload; other changes keep the cached structure +- A failed connection drops the cached config, so a credential or endpoint fix is retried on the next call rather than after `config_cache_ttl` +- `disable_config_cache = true` disables both caches, so config and address space are re-read on every call +- Any unhandled error clears all caches, forcing a fresh reload on the next call + +### Namespace URI resolution + +When using `nsu=` node IDs (either from `namespaces` aliases in CLI mode or directly in TOML), the URI-to-index resolution happens **once at connection time**. The resolved numeric indexes are stored in the config object. If the server assigns different namespace indexes after a restart, the plugin will re-resolve them on the next reconnect. + +This applies to `nodes`, `tag_nodes`, and `browse_root` — use `browse_root = "nsu=;s=..."` in browse mode for the same restart stability. If the URI is not found on the server, connection fails with a configuration error rather than browsing into an empty result. + +### Namespace aliases in TOML vs CLI + +In **CLI arguments** mode, you can define short aliases (for example, `siemens`) via the `namespaces` parameter and use them in `nodes` and `tag_nodes` definitions. In **TOML** mode, aliases are not supported — use the full `nsu=;...` format directly in node IDs. The `[opcua.namespaces]` section in TOML is validated but not used for alias substitution. + +## Troubleshooting + +### Check Plugin Logs + +```bash +influxdb3 query --database _internal \ + "SELECT * FROM system.processing_engine_logs + WHERE trigger_name = 'opcua_ingestion' + ORDER BY time DESC LIMIT 20" +``` +### Common Issues + +#### "asyncua library not installed" or "No module named 'asyncua'" + +```bash +influxdb3 install package asyncua +``` +#### "Configuration file not found" + +- For relative paths, ensure `PLUGIN_DIR` environment variable is set +- For absolute paths, verify the file exists at the specified location + +```bash +# For relative paths +export PLUGIN_DIR=~/.plugins +ls $PLUGIN_DIR/my_opcua_config.toml + +# Or use absolute path +ls /etc/opcua/my_opcua_config.toml +``` +#### "Failed to connect to OPC UA server" + +- Verify server URL and port (`opc.tcp://host:port`) +- Check network connectivity to the OPC UA server +- Ensure the OPC UA server is running and accepting connections +- For secure connections, verify that the server trusts the client certificate + +#### "Certificate and private_key required when security_policy is set" + +When using a security policy, both files are required: +- `certificate`: Client certificate in DER format (`.der`) +- `private_key`: Client private key in PEM format (`.pem`) + +#### "Invalid 'server_url' scheme" + +- `server_url` must use the `opc.tcp://` or `opc.tls://` scheme +- Other schemes (`file://`, `http://`, etc.) are rejected + +#### "Refusing to send username/password over an unencrypted connection" + +- `username`/`password` are set but no `security_policy` is configured +- Configure a `security_policy` for an encrypted connection (recommended) +- Or, only on a trusted network, set `allow_insecure_auth = true` to permit cleartext credentials + +#### "Namespace URI not found on server" + +- The `nsu=` URI does not match any namespace registered on the server +- Use an OPC UA browser (UaExpert) to see the server's namespace array +- Namespace URIs are case-sensitive + +#### "Bad status" errors for specific nodes + +- Verify the node ID exists on the server (use an OPC UA browser like UaExpert) +- Check that the configured user has read access to the node +- The node may be temporarily unavailable — check the server's diagnostic info + +#### "Null value" for a node + +- The node exists but has no value assigned +- The server may not have initialized the variable yet +- Check the node's status in an OPC UA client tool + +#### All nodes failing + +- Verify the namespace index is correct (namespaces may differ between server restarts — consider using `nsu=` URIs instead) +- Use an OPC UA browser to confirm the exact node IDs +- Check server logs for access control issues + +#### "Browse discovered no variables" (browse mode) + +- Verify `browse_root` points to a valid node that contains Object or Variable children +- If `browse_root` uses a numeric `ns=` index, confirm the index is still current — it can shift after a server restart. Use `nsu=;s=...` instead so the plugin resolves the index at connection time +- Check that `browse_depth` is sufficient to reach Variable nodes +- The root node may contain nodes of other types (for example, Methods) — only Objects and Variables are traversed +- If using `filter`, verify the regex pattern matches your variable names +- If using `exclude_branches`, verify the pattern is not excluding all Object nodes at a required depth +- Use an OPC UA browser (UaExpert) to inspect the address space structure + +#### Config changes not taking effect + +- Configuration is cached for `config_cache_ttl` seconds (default 1 hour) — either wait for expiry, lower `config_cache_ttl`, or set `disable_config_cache = true` +- Any plugin error automatically clears the cache + +#### "Previous opcua call still running, skipping this tick" + +- The plugin reuses a single cached connection and event loop, which cannot be used by two calls at once. With an asynchronous trigger (`--run-asynchronous`), scheduled invocations may overlap; if a call is still running when the next tick fires, that tick is skipped instead of failing. +- Occasional skips are harmless. Frequent skips mean each read takes longer than the interval — reduce the per-call runtime or give it more time: + - Read fewer nodes per trigger, or narrow browse mode (smaller `browse_depth`, tighter `filter`/`exclude_branches`) + - Increase the trigger interval (e.g. `--trigger-spec "every:30s"` instead of `every:10s`) + - Check for a slow or unreachable server adding connection latency + +## Limitations + +- **Polling only**: The plugin reads current values on each scheduled call. It does not use OPC UA subscriptions for change-based monitoring. Fast-changing signals may be missed between polling intervals. +- **No Historical Data Access (HDA)**: Only current values are read, not historical data from the server. +- **No write support**: The plugin only reads from OPC UA nodes. Writing values or calling methods is not supported. +- **Scalar values only**: Arrays, structures, ExtensionObjects, and other complex data types are converted to string representation. +- **Single table**: All nodes (or all browsed devices) are written to one measurement. For multiple tables, create separate plugin triggers. +- **Browse mode requires hierarchy**: Browse discovers nodes based on the OPC UA address space hierarchy. If the server uses a flat namespace with dotted identifiers (for example, `API15_Histo1.zone_5130.Signal`) rather than a folder tree, browse mode will not discover child signals — use `name_tags` with `name_separator` to extract hierarchy from flat names, or use explicit nodes mode instead. +- **No OPC-DA support**: Only OPC UA (Unified Architecture) is supported. Legacy OPC-DA (COM/DCOM-based) connections require a separate gateway. + +## Logging + +Logs are stored in the `_internal` database (or the database where the trigger is created) in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" +``` + +Log columns: +- **event_time**: Timestamp of the log event +- **trigger_name**: Name of the trigger that generated the log +- **log_level**: Severity level (INFO, WARN, ERROR) +- **log_text**: Message describing the action or error + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/prophet-forecasting.md b/content/shared/influxdb3-plugins/plugins-library/official/prophet-forecasting.md index a6e3ab7009..b9b4f987b9 100644 --- a/content/shared/influxdb3-plugins/plugins-library/official/prophet-forecasting.md +++ b/content/shared/influxdb3-plugins/plugins-library/official/prophet-forecasting.md @@ -1,11 +1,12 @@ - + + The Prophet Forecasting Plugin enables time series forecasting for data in {{% product-name %}} using Facebook's Prophet library. Generate predictions for future data points based on historical patterns, including seasonality, trends, and custom events. Supports both scheduled batch forecasting and on-demand HTTP-triggered forecasts with model persistence and validation capabilities. - **Model persistence**: Save and reuse trained models for consistent predictions - **Forecast validation**: Built-in accuracy assessment using Mean Squared Relative Error (MSRE) - **Holiday support**: Built-in holiday calendars and custom holiday configuration - **Advanced seasonality**: Configurable seasonality modes and changepoint detection -- **Flexible time intervals**: Support for seconds, minutes, hours, days, weeks, months, quarters, and years +- **Flexible time intervals**: Support for microseconds through years ## Configuration @@ -21,57 +22,66 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor Set these parameters with `--trigger-arguments` when creating a scheduled trigger: -| Parameter | Type | Default | Description | -|----------------------|--------|----------|-------------------------------------------------------------------| -| `measurement` | string | required | Source measurement containing historical data | -| `field` | string | required | Field name to forecast | -| `window` | string | required | Historical data window. Format: `` (for example, "30d") | -| `forecast_horizont` | string | required | Forecast duration. Format: `` (for example, "2d") | -| `tag_values` | string | required | Dot-separated tag filters (for example, "region:us-west.device:sensor1") | -| `target_measurement` | string | required | Destination measurement for forecast results | -| `model_mode` | string | required | Operation mode: "train" or "predict" | -| `unique_suffix` | string | required | Unique model identifier for versioning | +| Parameter | Type | Default | Description | +|----------------------|--------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------| +| `measurement` | string | required | Source measurement containing historical data | +| `field` | string | required | Field name to forecast | +| `window` | string | required | Historical data window, ending at the trigger's call time. Format: `` (for example, "30d") | +| `forecast_horizont` | string | required | Forecast duration. Format: `` (for example, "2d") | +| `tag_values` | string | required | Tag filters as dot-separated `tag:value` pairs (for example, "region:us-west.device:sensor1") | +| `target_measurement` | string | required | Destination measurement for forecast results | +| `model_mode` | string | required | `train` trains an in-memory model on every run; `predict` loads the saved model for `unique_suffix`, or trains and saves it when no file exists | +| `unique_suffix` | string | required | Model version identifier, also used as the model file name suffix. Up to 64 characters from letters, digits, `.`, `_` and `-` | ### HTTP request parameters -Send these parameters as JSON in the HTTP POST request body: +Send these parameters as JSON in the HTTP POST request body. Trigger arguments are not used by the HTTP endpoint; a JSON `null` means "not set", so the default applies. -| Parameter | Type | Default | Description | -|----------------------|--------|----------|----------------------------------------------------------| -| `measurement` | string | required | Source measurement containing historical data | -| `field` | string | required | Field name to forecast | -| `forecast_horizont` | string | required | Forecast duration. Format: `` (for example, "7d") | -| `tag_values` | object | required | Tag filters as JSON object (for example, {"region":"us-west"}) | -| `target_measurement` | string | required | Destination measurement for forecast results | -| `unique_suffix` | string | required | Unique model identifier for versioning | -| `start_time` | string | required | Historical window start (ISO 8601 format) | -| `end_time` | string | required | Historical window end (ISO 8601 format) | +| Parameter | Type | Default | Description | +|----------------------|---------------|----------|----------------------------------------------------------------------------------------------------| +| `measurement` | string | required | Source measurement containing historical data | +| `field` | string | required | Field name to forecast | +| `forecast_horizont` | string | required | Forecast duration. Format: `` (for example, "7d") | +| `tag_values` | object/string | required | Tag filters as a JSON object (for example, `{"region": "us-west"}`) or a dot-separated string | +| `target_measurement` | string | required | Destination measurement for forecast results | +| `unique_suffix` | string | required | Model version identifier, also used as the model file name suffix | +| `start_time` | string | required | Historical window start, ISO 8601 with timezone | +| `end_time` | string | required | Historical window end, ISO 8601 with timezone. Forecast points are written from this moment onward | +| `save_mode` | boolean | false | When true, load the saved model for `unique_suffix`, or train and save it when no file exists | ### Advanced parameters -| Parameter | Type | Default | Description | -|---------------------------|--------------|------------|----------------------------------------------------------| -| `seasonality_mode` | string | "additive" | Prophet seasonality mode: "additive" or "multiplicative" | -| `changepoint_prior_scale` | number | 0.05 | Flexibility of trend changepoints | -| `changepoints` | string/array | none | Changepoint dates (ISO format) | -| `holiday_date_list` | string/array | none | Custom holiday dates (ISO format) | -| `holiday_names` | string/array | none | Holiday names corresponding to dates | -| `holiday_country_names` | string/array | none | Country codes for built-in holidays | -| `inferred_freq` | string | auto | Manual frequency specification (for example, "1D", "1H") | -| `validation_window` | string | "0s" | Validation period duration | -| `msre_threshold` | number | infinity | Maximum acceptable Mean Squared Relative Error | -| `target_database` | string | current | Database for forecast storage | -| `save_mode` | string | "false" | Whether to save/load models (HTTP only) | +Available to both trigger types: + +| Parameter | Type | Default | Description | +|---------------------------|--------------|------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `seasonality_mode` | string | "additive" | Prophet seasonality mode: "additive" or "multiplicative" | +| `changepoint_prior_scale` | number | 0.05 | Flexibility of trend changepoints; must be greater than 0 | +| `changepoints` | string/array | none | Changepoint dates (ISO format), space-separated or as a list | +| `holiday_date_list` | string/array | none | Custom holiday dates (ISO format), space-separated or as a list | +| `holiday_names` | string/array | none | Holiday names matching `holiday_date_list`, dot-separated or as a list | +| `holiday_country_names` | string/array | none | Country code for built-in holidays, dot-separated or as a list. Prophet supports one country, so only the first entry is used | +| `inferred_freq` | string | auto | Pandas frequency alias, fixed ("30min", "1h") or calendar ("D", "W-SUN", "MS", "QS"); inferred from the data when omitted | +| `validation_window` | string | "0s" | Duration held back from training and used to validate the forecast | +| `validation_alignment` | string | "position" | How actual and forecasted values are paired: `position` pairs them in time order, `nearest` pairs each actual value with the closest forecast point within half a frequency step | +| `msre_threshold` | number | infinity | Maximum acceptable Mean Squared Relative Error; must be 0 or greater | +| `max_forecast_points` | integer | 10000 | Maximum number of forecast points per run, counting the validation window | +| `target_database` | string | "default" | Database for forecast results. Without this parameter, results go to a database named `default`, created on the first write | ### Notification parameters -| Parameter | Type | Default | Description | -|------------------------|--------|----------|-------------------------------------| -| `is_sending_alert` | string | "false" | Enable alerts on validation failure | -| `notification_text` | string | template | Custom alert message template | -| `senders` | string | none | Dot-separated notification channels | -| `notification_path` | string | "notify" | Notification endpoint path | -| `influxdb3_auth_token` | string | env var | Authentication token | +Scheduled triggers only: + +| Parameter | Type | Default | Description | +|------------------------|---------|----------|----------------------------------------------------------------------------------------------------------------------------| +| `is_sending_alert` | boolean | false | Send an alert when validation fails | +| `notification_text` | string | template | Alert message template. Variables: `$version`, `$measurement`, `$field`, `$start_time`, `$end_time`, `$output_measurement` | +| `senders` | string | none | Dot-separated notification channels; required when `is_sending_alert` is true | +| `notification_path` | string | "notify" | URL path of the notification sender plugin | +| `influxdb3_auth_token` | string | env var | Token for the notification request; falls back to `INFLUXDB3_AUTH_TOKEN` | +| `port_override` | integer | 8181 | Port for notification dispatch (1–65535) | + +Each channel listed in `senders` needs its own keys (`slack_webhook_url`, `discord_webhook_url`, `http_webhook_url`, `twilio_sid`, `twilio_token`, `twilio_from_number`, `twilio_to_number`, and the optional `*_headers`). See the [influxdata/notifier plugin](/influxdb3/version/plugins/library/official/notifier/). ### TOML configuration @@ -79,7 +89,9 @@ Send these parameters as JSON in the HTTP POST request body: |--------------------|--------|---------|----------------------------------------------------------------------------------| | `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (required for TOML configuration) | -*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. Relative paths are resolved against the first directory that is set: `PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent of `VIRTUAL_ENV`. Only that directory is used — the file is not looked up in the remaining ones. + +When `config_file_path` is set, the TOML file provides the whole configuration and inline trigger arguments are ignored. `INFLUXDB3_AUTH_TOKEN` from the environment still applies when `influxdb3_auth_token` is not set in the file. In TOML, `tag_values`, `senders`, `changepoints`, `holiday_date_list`, `holiday_names` and `holiday_country_names` can use native structures (a table or a list) instead of the inline string formats, though the inline strings are also accepted. The HTTP endpoint ignores `config_file_path`. #### Example TOML configuration @@ -91,8 +103,8 @@ For more information on using TOML configuration files, see the Using TOML Confi - **{{% product-name %}}**: with the Processing Engine enabled. - **Python packages**: - - `pandas` (for data manipulation) - - `numpy` (for numerical operations) + - `influxdata-plugin-utils>=0.3.0` (configuration loading, parsing, and writing) + - `pandas` (for data manipulation; 2.x and 3.x are both supported) - `requests` (for HTTP requests) - `prophet` (for time series forecasting) - **Notification Sender Plugin** *(optional)*: Required if using the `senders` parameter. See the [influxdata/notifier plugin](/influxdb3/version/plugins/library/official/notifier/). @@ -111,8 +123,8 @@ For more information on using TOML configuration files, see the Using TOML Confi 2. Install required Python packages: ```bash + influxdb3 install package influxdata-plugin-utils influxdb3 install package pandas - influxdb3 install package numpy influxdb3 install package requests influxdb3 install package prophet ``` @@ -129,7 +141,7 @@ influxdb3 create trigger \ --database mydb \ --path "gh:influxdata/prophet_forecasting/prophet_forecasting.py" \ --trigger-spec "every:1d" \ - --trigger-arguments "measurement=temperature,field=value,window=30d,forecast_horizont=2d,tag_values=region:us-west.device:sensor1,target_measurement=temperature_forecast,model_mode=train,unique_suffix=20250619_v1" \ + --trigger-arguments "measurement=temperature,field=value,window=30d,forecast_horizont=2d,tag_values=region:us-west.device:sensor1,target_measurement=temperature_forecast,model_mode=train,unique_suffix=20250619_v1,target_database=mydb" \ prophet_forecast_trigger ``` ### HTTP trigger @@ -166,7 +178,7 @@ influxdb3 create trigger \ --database mydb \ --path "gh:influxdata/prophet_forecasting/prophet_forecasting.py" \ --trigger-spec "every:1d" \ - --trigger-arguments "measurement=temperature,field=value,window=30d,forecast_horizont=2d,tag_values=region:us-west.device:sensor1,target_measurement=temperature_forecast,model_mode=train,unique_suffix=v1" \ + --trigger-arguments "measurement=temperature,field=value,window=30d,forecast_horizont=2d,tag_values=region:us-west.device:sensor1,target_measurement=temperature_forecast,model_mode=train,unique_suffix=v1,target_database=mydb" \ prophet_forecast influxdb3 enable trigger --database mydb prophet_forecast @@ -188,8 +200,6 @@ influxdb3 query \ ``` ### Example 2: On-demand HTTP forecasting -Example HTTP request for on-demand forecasting: - ```bash curl -X POST http://localhost:8181/api/v3/engine/forecast \ -H "Authorization: Bearer YOUR_TOKEN" \ @@ -200,16 +210,23 @@ curl -X POST http://localhost:8181/api/v3/engine/forecast \ "forecast_horizont": "7d", "tag_values": {"region":"us-west","device":"sensor1"}, "target_measurement": "temperature_forecast", + "target_database": "mydb", "unique_suffix": "model_v1_20250722", "start_time": "2025-05-20T00:00:00Z", "end_time": "2025-06-19T00:00:00Z", "seasonality_mode": "additive", "changepoint_prior_scale": 0.05, "validation_window": "3d", + "validation_alignment": "nearest", "msre_threshold": 0.05 }' ``` -### Advanced forecasting with holidays +**Expected response** + +```json +{"message": "[] Forecast written to temperature_forecast"} +``` +### Example 3: Advanced forecasting with holidays ```bash curl -X POST http://localhost:8181/api/v3/engine/forecast \ @@ -221,7 +238,9 @@ curl -X POST http://localhost:8181/api/v3/engine/forecast \ "forecast_horizont": "30d", "tag_values": {"store":"main_branch"}, "target_measurement": "revenue_forecast", + "target_database": "mydb", "unique_suffix": "retail_model_v2", + "save_mode": true, "start_time": "2024-01-01T00:00:00Z", "end_time": "2025-06-01T00:00:00Z", "holiday_country_names": ["US"], @@ -233,30 +252,33 @@ curl -X POST http://localhost:8181/api/v3/engine/forecast \ ``` ## Output data structure -Forecast results are written to the target measurement with the following structure: +Forecast results are written to the target measurement in `target_database`, or to a database named `default` when that parameter is omitted. ### Tags -- `model_version`: Model identifier from unique_suffix parameter -- Additional tags from original measurement query filters +- `model_version`: Model identifier from the `unique_suffix` parameter +- One tag per entry in `tag_values` ### Fields -- `forecast`: Predicted value (yhat from Prophet model) -- `yhat_lower`: Lower bound of confidence interval -- `yhat_upper`: Upper bound of confidence interval -- `run_time`: Forecast execution timestamp (ISO 8601 format) +- `forecast`: Predicted value (`yhat` from the Prophet model) +- `yhat_lower`: Lower bound of the confidence interval +- `yhat_upper`: Upper bound of the confidence interval +- `run_time`: Time the forecast ran, ISO 8601 with UTC offset ### Timestamp - `time`: Forecast timestamp in nanoseconds +Points where a forecast value is not finite are skipped, and their count is logged as a warning. + ## Code overview ### Files - `prophet_forecasting.py`: The main plugin code containing handlers for scheduled and HTTP triggers - `prophet_forecasting_scheduler.toml`: Example TOML configuration file for scheduled triggers +- `requirements.txt`, `requirements-dev.txt`: Runtime and development dependencies ### Logging @@ -269,19 +291,20 @@ influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine #### `process_scheduled_call(influxdb3_local, call_time, args)` -Handles scheduled forecasting tasks. Queries historical data, trains or loads Prophet model, generates forecasts, and writes results. +Handles scheduled forecasting. The training window ends at `call_time - validation_window` and starts at `call_time - window`; forecast points are written from `call_time` onward. Key operations: -1. Parses configuration from arguments or TOML file -2. Queries historical data within specified window -3. Trains Prophet model or loads existing model -4. Generates forecasts for specified horizon -5. Optionally validates against actual data and sends alerts +1. Loads and validates the configuration from the trigger arguments or the TOML file +2. Queries the historical window with the configured tag filters +3. Trains a model or loads the saved one, depending on `model_mode` +4. Forecasts at the resolved frequency, from one step after the last queried point up to `call_time` plus `forecast_horizont` +5. Validates the forecast when `validation_window` is set, and sends an alert on failure +6. Writes the forecast points -#### `process_http_request(influxdb3_local, request_body, args)` +#### `process_request(influxdb3_local, query_parameters, request_headers, request_body, args)` -Handles on-demand forecast requests via HTTP. Supports backfill operations with configurable time ranges. +Handles on-demand forecasts over an explicit window. The training window is `start_time` to `end_time - validation_window`, and forecast points are written from `end_time` onward. Returns `{"message": ...}` describing the outcome. ## Troubleshooting @@ -289,42 +312,53 @@ Handles on-demand forecast requests via HTTP. Supports backfill operations with #### Issue: Model training failures -**Solution**: Ensure sufficient historical data points for the specified window. Verify data contains required time column and forecast field. Check for data gaps that might affect frequency inference. Set `inferred_freq` manually if automatic detection fails. +**Solution**: Ensure sufficient historical data points for the specified window. Verify the data contains the forecast field with numeric values; rows where the field is missing or non-numeric are dropped and counted in a warning. Set `inferred_freq` manually when the frequency cannot be inferred (at least three points are required); a frequency that does not move time forward, such as `0h`, is rejected. #### Issue: Validation failures -**Solution**: Review MSRE threshold settings - values too low may cause frequent failures. Ensure validation window provides sufficient data for comparison. Check that validation data aligns temporally with forecast period. +**Solution**: Review the `msre_threshold` setting — values that are too low cause frequent failures. Ensure the validation window holds enough data. With `validation_alignment=nearest`, an actual value is only compared when a forecast point falls within half a frequency step of it, and validation is skipped when nothing matches; check that `inferred_freq` matches the real cadence of the data. + +#### Issue: `Invalid unique_suffix` + +**Solution**: `unique_suffix` becomes part of the model file name and accepts up to 64 characters from letters, digits, `.`, `_` and `-`. #### Issue: HTTP trigger issues -**Solution**: Verify JSON request body format matches expected schema. Check authentication tokens and database permissions. Ensure start_time and end_time are in valid ISO 8601 format with timezone. +**Solution**: Verify the JSON request body matches the expected schema. Check authentication tokens and database permissions. Ensure `start_time` and `end_time` are valid ISO 8601 values with a timezone. -#### Issue: Model persistence problems +#### Issue: Forecast results are not in the expected database -**Solution**: Verify plugin directory permissions for model storage. Check disk space availability in plugin directory. Ensure unique_suffix values don't conflict between different model versions. +**Solution**: Set `target_database`. Without it, results are written to a database named `default`, which is created on the first write. ### Model storage -- **Location**: Models stored in `prophet_models/` directory within plugin directory +- **Location**: `prophet_models/` under the resolved plugin directory (`PLUGIN_DIR`, `INFLUXDB3_PLUGIN_DIR`, or the parent of `VIRTUAL_ENV`) - **Naming**: Files named `prophet_model_{unique_suffix}.json` -- **Versioning**: Use descriptive unique_suffix values for model management +- **Writing**: Models are written to a temporary file and renamed into place +- **Versioning**: Use descriptive `unique_suffix` values for model management +- **Reuse**: A model loaded from disk forecasts the timestamps derived from the freshly queried data, so it stays usable after the data has moved on; its coefficients still come from the data it was trained on, so the further the run is from that training window, the wider the extrapolation + +### Frequency support + +The forecast step comes from `inferred_freq` or is inferred from the data. Fixed aliases (`1s`, `30min`, `1h`) and calendar aliases (`D`, `W-SUN`, `MS`, `QS`, `YS`) are both supported; calendar steps follow the calendar, so a monthly forecast lands on month starts. Forecast timestamps start one step after the last queried point and run until `forecast_horizont` past the trigger's call time (or `end_time` for HTTP requests). ### Time format support -Supported time units for window, forecast_horizont, and validation_window: +Supported units for `window`, `forecast_horizont` and `validation_window`: -- `s` (seconds), `min` (minutes), `h` (hours) +- `us` (microseconds), `ms` (milliseconds), `s` (seconds), `min` (minutes), `h` (hours) - `d` (days), `w` (weeks) - `m` (months ≈30.42 days), `q` (quarters ≈91.25 days), `y` (years = 365 days) ### Validation process -When validation_window is set: +When `validation_window` is set: -1. Training data: `current_time - window` to `current_time - validation_window` -2. Validation data: `current_time - validation_window` to `current_time` -3. MSRE calculation: `mean((actual - predicted)² / actual²)` -4. Threshold comparison and optional alert dispatch +1. Training data: window start to `window_end - validation_window` +2. Validation data: `window_end - validation_window` to `window_end` +3. Actual and forecasted values are paired according to `validation_alignment`; `nearest` compares each actual value with the forecast point closest in time — at most half a frequency step away — and ignores actual values that fall outside the forecast range +4. MSRE: `mean((actual - predicted)² / actual²)`, computed over non-zero actual values +5. Validation fails when MSRE exceeds `msre_threshold` or cannot be computed at all — the validation window holds no data, no actual value is close enough to a forecast point, or every actual value is zero. A failed validation withholds the forecast and sends an alert if configured ## Report an issue @@ -333,4 +367,6 @@ For plugin issues, see the Plugins repository [issues page](https://github.com/i ## Find support for {{% product-name %}} The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. -For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. \ No newline at end of file +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/resampler.md b/content/shared/influxdb3-plugins/plugins-library/official/resampler.md new file mode 100644 index 0000000000..43e45812de --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/resampler.md @@ -0,0 +1,265 @@ + + +> **Note:** This plugin requires {{% product-name %}}.8.2 or later (uses the synchronous write API). + + +The Resampler Plugin interpolates time series with non-uniform timestamps +(sensor timestamp jitter) onto a uniform time grid and writes the result to a +separate measurement. Uniform sampling is required by most signal-processing +and forecasting algorithms (FIR/IIR filters, FFT, ML models). + +Numeric fields are interpolated; all other fields (strings, booleans) are +carried onto the same grid points by last known value, so the full data set +is preserved. A separate `snap` mode skips interpolation entirely: each +point's timestamp is rounded to the nearest grid node and all values and +types stay unchanged (on node collisions the latest point wins). + +Each run processes a sliding window of history. Output timestamps are exact +multiples of the grid interval, so overlapping runs overwrite the same points +idempotently and late-arriving data is picked up automatically. Every unique +tag combination is resampled as an independent series, with all tags preserved. + +The plugin does not fill large data gaps: when two neighboring source points +are more than `max_gap` apart, grid points between them are not written, so +sensor outages stay visible in the output. + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. This plugin supports TOML configuration files, which can be specified using the `config_file_path` parameter. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Required parameters + +| Parameter | Type | Default | Description | +|----------------------|--------|----------|-----------------------------------------------------------------------------| +| `measurement` | string | required | Source measurement with non-uniform timestamps | +| `target_measurement` | string | required | Output measurement; must differ from the source measurement | + +### Grid parameters + +| Parameter | Type | Default | Description | +|------------|--------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `interval` | string | `1s` | Uniform grid step, at least `1ms`. Units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`. (`window` + `max_gap`)/`interval` is capped at 1,000,000 grid points per run, so sub-second grids need a proportionally small `window` | +| `window` | string | `10min` | How much history each run processes; at least one `interval` | +| `offset` | string | `0s` | Processing delay for late-arriving data | + +### Processing parameters + +| Parameter | Type | Default | Description | +|------------------------|--------|----------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `mode` | string | `interpolate` | `interpolate` recalculates numeric fields at grid nodes; `snap` only rounds timestamps to the nearest node, values and types unchanged (`interpolation_method` and `max_gap` are ignored) | +| `fields` | string | all numeric | Space-separated numeric fields to interpolate; numeric fields not listed are dropped. In snap mode: fields of any type to carry (default all) | +| `excluded_fields` | string | none | Space-separated fields of any type to exclude from the output; listing a field in both `fields` and `excluded_fields` is an error | +| `interpolation_method` | string | `linear` | `linear`, `nearest`, `cubic`, `previous`, or `next`. Use `previous` (last known value) for step-like signals such as states or counters. `nearest`/`previous`/`next` return source values unchanged (integer fields keep their type and precision); `linear`/`cubic` always produce float values | +| `max_gap` | string | 2 × `interval` | Source-point spacing above which grid points in between are not written; at least one `interval` | + +### Output parameters + +| Parameter | Type | Default | Description | +|-------------------|---------|------------|------------------------------------------| +| `target_database` | string | trigger DB | Optional target database for the output | +| `max_retries` | integer | `5` | Maximum number of write attempts | + +### TOML configuration + +| Parameter | Type | Default | Description | +|--------------------|--------|---------|-----------------------------------------------------------------------------------------------| +| `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (replaces trigger arguments entirely when set) | + +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. + +#### Example TOML configuration + +[resampler_config_scheduler.toml](https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/resampler/resampler_config_scheduler.toml) + +For more information on using TOML configuration files, see the Using TOML Configuration Files section in the [influxdb3_plugins/README.md](https://github.com/influxdata/influxdb3_plugins/blob/master/README.md). + +## Data requirements + +Interpolation needs at least 2 source points per series inside the window; a +grid node is written only when it has a source point on each side at most +`max_gap` apart (or an exact source match). Snap mode has no such requirement. + +## Software Requirements + +- **{{% product-name %}}**: with the Processing Engine enabled +- **Python packages** (declared in `manifest.toml`): + - `influxdata-plugin-utils>=0.3.0` + - `scipy` (installs `numpy`) + +## Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install required Python packages: + + ```bash + influxdb3 install package "influxdata-plugin-utils>=0.3.0" + influxdb3 install package scipy + ``` +## Trigger setup + +### Scheduled resampling + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/resampler/resampler.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'measurement=signal,target_measurement=signal_resampled,interval=1s,window=1min' \ + resampler_trigger +``` +### Scheduled resampling with TOML configuration + +```bash +# Copy and edit the configuration file +cp resampler_config_scheduler.toml $PLUGIN_DIR/resampler_config.toml + +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/resampler/resampler.py \ + --trigger-spec "every:10s" \ + --trigger-arguments config_file_path=resampler_config.toml \ + resampler_trigger +``` +## Example usage + +### Example 1: Linear interpolation onto a 1-second grid + +```bash +# Create the trigger +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/resampler/resampler.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'measurement=signal,target_measurement=signal_resampled,interval=1s,window=10min' \ + resampler_trigger + +# Query resampled data (after trigger runs) +influxdb3 query --database mydb "SELECT * FROM signal_resampled" +``` +Source data with timestamp jitter: + +``` +signal value=39.1 1750000000083000000 # 00.083 +signal value=40.2 1750000000947000000 # 00.947 +signal value=41.0 1750000002114000000 # 02.114 +``` +### Expected output + +``` +signal_resampled value=40.24 1750000001000000000 # 01.000 +signal_resampled value=40.92 1750000002000000000 # 02.000 +``` +Grid points inside gaps larger than `max_gap` are not written and can be +filled by a downstream fill plugin. Non-numeric fields follow the same gap +rules but are carried by last known value instead of being interpolated. + +### Example 2: Snap mode + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/resampler/resampler.py \ + --trigger-spec "every:10s" \ + --trigger-arguments 'measurement=signal,target_measurement=signal_resampled,interval=1s,mode=snap' \ + resampler_snap +``` +### Expected output + +Timestamps are aligned to the grid; values and types are unchanged: + +``` +signal_resampled value=39.1 1750000000000000000 # 00.083 -> 00.000 +signal_resampled value=40.2 1750000001000000000 # 00.947 -> 01.000 +signal_resampled value=41.0 1750000002000000000 # 02.114 -> 02.000 +``` +## Code overview + +### Files + +- `resampler.py`: The main plugin code containing the handler for scheduled resampling +- `resampler_config_scheduler.toml`: Example TOML configuration file + +### Logging + +Logs are stored in the `_internal` database in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal \ + "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'resampler_trigger' ORDER BY event_time DESC LIMIT 20" +``` +### Main functions + +#### `process_scheduled_call(influxdb3_local, call_time, args)` + +Entry point for the scheduled trigger. Parses and validates the configuration, +queries the source window, groups rows by tag set, resamples or snaps each +series, and writes the result with retries. + +#### `resample_series(...)` + +Resamples all fields of one series: numeric fields selected for interpolation +are recalculated at grid nodes (`resample_field`), all other carried fields +use last known value (`carry_field_previous`), with the same `max_gap` rules. + +#### `snap_series(...)` + +Rounds each source point's timestamp to the nearest grid node, keeping values +and types unchanged; on node collisions the latest point wins per field. + +## Troubleshooting + +### Common issues + +#### Issue: No output data + +**Solution**: Check that the window contains at least 2 source points per +series and that `interval` is not larger than `window`. + +#### Issue: Missing points near "now" + +**Solution**: The freshest grid points need a source neighbor on the right; +they are written by the next run once that neighbor arrives. Increase `offset` +to delay processing instead. + +#### Issue: Holes in the output + +**Solution**: Source gaps larger than `max_gap` are preserved by design. +Increase `max_gap` to interpolate across larger gaps. + +#### Issue: `cubic` falls back to linear + +**Solution**: Cubic interpolation needs at least 4 points per series in the +window; add more data or use `linear`. + +#### Issue: A numeric field is missing from the output + +**Solution**: `fields` is set and does not list it — numeric fields not +listed there are dropped by design. + +#### Issue: Duplicate-looking values in snap mode + +**Solution**: Two source points rounded to the same grid node — the later one +wins per field. Use a smaller `interval` to keep them apart. + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/river-anomaly-detector.md b/content/shared/influxdb3-plugins/plugins-library/official/river-anomaly-detector.md new file mode 100644 index 0000000000..86ea6f9e62 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/river-anomaly-detector.md @@ -0,0 +1,273 @@ + + +An adaptive {{% product-name %}} Processing Engine plugin that detects anomalies in incoming time-series data using the [River ML](https://riverml.xyz) library. The detector automatically adapts its detection strategy based on data characteristics learned by the companion auto-profiler plugin. + +## Features + +- **Adaptive detection:** Automatically selects detectors (Z-Score, Seasonal, ADWIN) based on profiler's pattern classification +- **Zero-config:** Works out of the box with sensible defaults — just create a trigger and go +- **Auto-tune by default:** Reads profiler recommendations when available, falls back to conservative defaults otherwise +- **Always-learn detectors:** Rolling, seasonal, and ADWIN detectors all learn from every observation; only the ones active in the current mode contribute votes to the anomaly decision +- **Seasonal awareness:** Learns hour-of-day (24 buckets) or hour-of-week (168 buckets) patterns — activated only when the profiler detects seasonality +- **Drift detection:** Uses ADWIN algorithm for trending data — activated only when profiler detects trends +- **Hysteresis on mode changes:** Detector mode only switches after 3 consecutive same-mode recommendations to prevent flip-flopping +- **Configurable combination:** Combine detector votes with "any" (OR), "majority", or "all" (AND) logic +- **Per-series models:** Each unique combination of table + tags + field gets its own detector set +- **Online learning:** Models update incrementally with each observation — no batch retraining needed +- **LRU eviction:** Configurable limit on tracked series to control memory usage +- **Model persistence:** Models are pickled, compressed, chunked, and checkpointed to the database so they survive server restarts + +## Prerequisites + +- {{% product-name %}} with Processing Engine enabled +- River ML library (`river>=0.23.0`) + +## Quick Start + +### 1. Install River + +```bash +influxdb3 install package river +``` +### 2. Create a trigger (zero-config) + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_anomaly_detector.py \ + --trigger-spec "all_tables" \ + anomaly_detector +``` +That's it! The plugin will start monitoring all numeric fields in all tables. With `auto_tune=true` (default), it reads profiler recommendations when available. + +### 3. Query anomalies + +```sql +SELECT * FROM "_anomalies.cpu" ORDER BY time DESC LIMIT 10 +``` +## Customized Example + +Monitor specific fields with explicit detector overrides: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_anomaly_detector.py \ + --trigger-spec "table:cpu" \ + --trigger-arguments 'include_fields=temperature humidity' 'rolling_std_threshold=3.0' 'enable_seasonal=true' \ + anomaly_detector_cpu +``` +## Trigger Arguments + +Arguments can be passed inline (space-separated lists) or via TOML config file (native TOML lists). Parameter names are the same in both modes. + +| Argument | Required | Default | Description | +|-------------------------------|------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `include_fields` | No | `""` | Fields to monitor (space-separated). If set, only these numeric fields are processed | +| `exclude_fields` | No | `""` | Fields to exclude from detection (space-separated) | +| `exclude_tables` | No | `""` | Tables to skip when using `all_tables` trigger (space-separated) | +| `string_fields` | No | `""` | String columns that are fields, not tags (space-separated). All other strings become tags | +| `rolling_std_threshold` | No | `5.0` | Standard deviations from the mean to flag as anomaly | +| `ew_fading_factor` | No | `0.3` | Fading factor for exponentially weighted stats (lower = longer memory) | +| `combination_mode` | No | `any` | How to combine detector votes: `any` (OR), `majority`, or `all` (AND) | +| `enable_seasonal` | No | `""` | Force seasonal detection: `true`/`false` to override, empty = auto from profiler | +| `enable_adwin` | No | `""` | Force ADWIN drift detection: `true`/`false` to override, empty = auto from profiler | +| `detector_mode` | No | `""` | Explicitly set detector mode components (space-separated or TOML list). Overrides profiler's mode recommendation. See [Adaptive Detection](#adaptive-detection) | +| `adwin_delta` | No | `0.002` | ADWIN sensitivity parameter. Lower values = more sensitive to drift | +| `seasonal_fading_factor` | No | `0.1` | Fading factor for seasonal bucket stats (lower = longer memory) | +| `seasonal_period` | No | `""` | Seasonal bucket period: `hourly` (24 buckets, hour-of-day) or `weekly` (168 buckets, hour-of-week). Empty = auto from profiler, falls back to `hourly` | +| `seasonal_threshold` | No | `3.0` | Std devs from seasonal bucket mean to flag as anomaly | +| `min_seasonal_observations` | No | `5` | Min observations per bucket before contributing to anomaly detection | +| `min_rolling_observations` | No | `10` | Min total observations per series before the rolling Z-score starts scoring (cold-start guard) | +| `max_series` | No | `1000` | Max unique series to track (LRU eviction) | +| `auto_tune` | No | `true` | Read per-series parameters from `_meta.series_profiles` (requires auto-profiler plugin) | +| `tune_refresh_interval` | No | `100` | Observations per series between auto-tune refreshes from `_meta.series_profiles` | +| `log_anomalies` | No | `true` | Log detected anomalies to server log | +| `checkpoint_interval_seconds` | No | `1800` | Seconds between model checkpoints to the database | +| `max_checkpoint_age_hours` | No | `24` | Ignore checkpoints older than this when restoring | +| `min_checkpoint_observations` | No | `10` | Skip checkpointing models with fewer observations than this (avoids persisting cold-start state) | +| `config_file_path` | No | — | Path to TOML config file. Supports absolute paths or relative paths (resolved via PLUGIN_DIR) | + +## Column Classification + +The plugin classifies each column in incoming data: + +1. `time` — skipped +2. String columns listed in `string_fields` — skipped from anomaly detection and not used as tags in the series key +3. Other string columns — treated as **tags** (used in series key) +4. Numeric columns (`int`, `float`, not `bool`) — treated as **fields** for anomaly detection + - Filtered by `include_fields` (if set, only listed fields are monitored) + - Filtered by `exclude_fields` (excluded fields are skipped) + +NaN and infinity values are skipped with a warning. Column types are inferred from the first row of each batch. + +## Table Filtering + +All tables starting with `_` are automatically skipped (for example, `_anomalies.*`, `_meta.*`, `_system.*`, `_forecasts.*`). Additional tables can be excluded via `exclude_tables`. + +## Adaptive Detection + +`detector_mode` is a space-separated list of components. Exactly one **base** component selects the Z-score behavior; optional **add-ons** enable seasonal and ADWIN voting: + +**Base components** (pick one, reflects profiler's pattern classification): + +| Base Mode | Used For | +|-----------------------|----------------------------------| +| `zscore_conservative` | No profile yet (default) | +| `zscore_low` | Stable data (lower threshold) | +| `zscore_high` | Noisy data (higher threshold) | +| `zscore_adaptive` | Bursty data (adaptive threshold) | + +**Add-on components** (zero or more): + +| Component | Effect | +|------------|-----------------------------------------------------------------| +| `seasonal` | Enables seasonal-bucket voting on top of the base Z-score | +| `adwin` | Enables ADWIN drift-detection voting on top of the base Z-score | + +Examples: `zscore_conservative`, `zscore_low seasonal`, `zscore_high adwin`, `zscore_conservative seasonal adwin`. + +> **Note:** All detectors always *learn* from every observation regardless of mode. The mode only controls which detectors *vote* on the anomaly decision. + +### How Adaptation Works + +Mode selection: +1. **Explicit `detector_mode`** — if set, overrides the base mode from the profiler +2. **Auto-tune** (`auto_tune=true`) — reads `recommended_detector_mode` from `_meta.series_profiles` +3. **Model default** — `zscore_conservative` + +After the base mode is determined, `enable_seasonal` and `enable_adwin` overrides are applied on top (can add or remove the `seasonal` / `adwin` components from any base mode). + +Even when `detector_mode` is explicitly set, the profiler is still consulted (with `auto_tune=true`) for per-series `threshold`, `ew_fading_factor`, `seasonal_fading_factor`, and `seasonal_period` — only the mode itself is overridden. + +**Hysteresis:** `detector_mode` only switches after **3 consecutive fresh profiler recommendations** of the same new mode (one recommendation = one `tune_refresh_interval` query). Non-mode fields (`threshold`, `fading_factor`, `seasonal_fading`, `seasonal_period`) update on every refresh without hysteresis. This prevents flip-flopping on noisy pattern classifications while still letting thresholds and fading factors track the data. + +**Seasonality readiness gate:** If the profiler recommends `seasonal` but hasn't marked `seasonality_ready=true` yet, the `seasonal` component is stripped before the model is configured (so the model doesn't score against an immature seasonal grid). + +### Detector Details + +**Rolling Z-Score (always active for scoring):** Exponentially weighted mean and variance track the "normal" range. When a value exceeds `mean ± N*std` (configurable threshold), it's flagged. Starts scoring after `min_rolling_observations` observations (default 10) to reduce false positives during cold start. When the fading factor changes via auto-tune, the EW stats are re-seeded from the prior mean/std to avoid a cold restart. + +**Seasonal:** Time-based buckets, each maintaining its own EW mean and variance. Values are compared against their bucket's learned pattern. The bucketing scheme is controlled by `seasonal_period`: +- `hourly` — 24 buckets (H00–H23), one per hour-of-day +- `weekly` — 168 buckets (Mon-H00 … Sun-H23), one per hour-of-week + +Each bucket matures independently after `min_seasonal_observations` (default 5) data points. Buckets always learn; they only *vote* when `seasonal` is part of the detector mode. Changing `seasonal_period` at runtime resets the seasonal grid. + +**ADWIN:** River ML's Adaptive Windowing algorithm on raw values. Detects when the statistical properties of the data stream change (concept drift). Sensitivity is controlled by `adwin_delta` (default 0.002; lower = more sensitive). ADWIN always learns but only votes when `adwin` is part of the detector mode. + +### Combination Modes + +The `combination_mode` parameter controls how active detector votes are combined: + +- `any` (default): Anomaly if **any** active detector flags it (OR logic) +- `majority`: Anomaly if **at least half** of active detectors flag it (⌈n/2⌉ votes). With 2 active detectors, 1 flag is enough; with 3, at least 2 are required +- `all`: Anomaly if **all** active detectors flag it (AND logic) + +Only detectors that are active in the current mode participate in voting. + +## Output Schema + +Anomalies are written to `_anomalies.{source_table}`. + +**Tags:** All original tags preserved + `field_name` + +**Fields:** + +| Field | Type | Description | +|-------------------------|---------|------------------------------------------------------------------------------| +| `original_value` | float | The value that triggered detection | +| `is_anomaly` | boolean | True if detector combination flagged | +| `observations` | integer | Total observations for this series | +| `detector_mode` | string | Active detector mode (for example, `zscore_conservative seasonal`) | +| `rolling_anomaly` | boolean | Rolling stats detector flagged | +| `rolling_mean` | float | Current exponentially weighted mean | +| `rolling_std` | float | Current exponentially weighted std deviation | +| `rolling_deviation` | float | How many std devs from mean | +| `rolling_threshold` | float | Configured std dev threshold | +| `seasonal_anomaly` | boolean | Seasonal detector flagged | +| `seasonal_mature` | boolean | Whether bucket has enough observations | +| `seasonal_mean` | float | Exponentially weighted mean for this bucket | +| `seasonal_std` | float | Exponentially weighted std dev for this bucket | +| `seasonal_deviation` | float | How many std devs from seasonal bucket mean | +| `seasonal_bucket` | string | Bucket identifier, for example, `H14` (2pm UTC) for hourly, or `Mon-H14` for weekly | +| `seasonal_observations` | integer | Observation count in this bucket | +| `adwin_anomaly` | boolean | ADWIN drift detector flagged | +| `drift_detected` | boolean | Whether ADWIN detected a concept drift | + +Only rows where `is_anomaly=true` are written. Seasonal bucket fields (`seasonal_mean`, `seasonal_std`, `seasonal_deviation`, `seasonal_bucket`, `seasonal_observations`) appear once the seasonal grid has accumulated enough data for the current bucket, regardless of whether the `seasonal` component is active — because seasonal stats always learn. `seasonal_anomaly` and `adwin_anomaly` only flip to `true` when their respective components are part of the active detector mode. + +## Model Persistence + +The plugin automatically checkpoints and restores models so they survive server restarts. + +### Checkpoint + +Every `checkpoint_interval_seconds` (default: 1800 = 30 minutes), the plugin pickles each per-series model, compresses with zlib, base64-encodes, and writes to `_system.model_checkpoints`. Large models are automatically chunked across multiple rows (60KB chunks). + +Models with fewer than `min_checkpoint_observations` observations (default: 10) are skipped — there is no point persisting cold-start state that would be rebuilt from scratch on restore anyway. + +### Restore + +On the first invocation after a server restart (detected by an empty model cache), the plugin queries `_system.model_checkpoints` for the latest checkpoint per series. Checkpoints older than `max_checkpoint_age_hours` (default: 24) are ignored. Restored models resume scoring immediately with their full learned history. + +### Checkpoint Schema (`_system.model_checkpoints`) + +| Column | Type | Description | +|-------------------------|---------|-----------------------------------------------------| +| `plugin` | tag | `"river_anomaly_detector"` | +| `series_key` | tag | Unique series identifier | +| `chunk_index` | tag | 0-based chunk index (for multi-row models) | +| `chunk_total` | tag | Total chunks for this model | +| `model_data` | string | Base64-encoded, zlib-compressed pickle of the model | +| `model_type` | string | `"SeriesModel"` | +| `observation_count` | integer | Observations the model has processed | +| `checkpoint_size_bytes` | integer | Size of the compressed data | + +## TOML Configuration + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_anomaly_detector.py \ + --trigger-spec "all_tables" \ + --trigger-arguments config_file_path=river_anomaly_detector_config.toml \ + anomaly_detector +``` +Or with an absolute path (no PLUGIN_DIR needed): + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_anomaly_detector.py \ + --trigger-spec "all_tables" \ + --trigger-arguments config_file_path=/etc/influxdb3/river_anomaly_detector_config.toml \ + anomaly_detector +``` +See `river_anomaly_detector_config.toml` for an example configuration. + + +## Logging + +Logs are stored in the `_internal` database (or the database where the trigger is created) in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" +``` + +Log columns: +- **event_time**: Timestamp of the log event +- **trigger_name**: Name of the trigger that generated the log +- **log_level**: Severity level (INFO, WARN, ERROR) +- **log_text**: Message describing the action or error + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/river-auto-profiler.md b/content/shared/influxdb3-plugins/plugins-library/official/river-auto-profiler.md new file mode 100644 index 0000000000..b498bc52a6 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/river-auto-profiler.md @@ -0,0 +1,271 @@ + + +A zero-config {{% product-name %}} Processing Engine plugin that incrementally profiles incoming time-series data, classifies data patterns, and writes recommended anomaly detection parameters per series. This is the "zero-config" enabler — it learns about your data so the anomaly detector can auto-tune. + +## Features + +- **Incremental profiling:** Uses River ML's streaming statistics to build profiles one observation at a time +- **Pattern classification:** Automatically classifies data as stable, noisy, trending, seasonal, or bursty +- **Adaptive recommendations:** Recommends detector mode, threshold, and fading factors based on data characteristics +- **Calibrated EW stats:** Collects initial observations to determine optimal fading factor before creating EW statistics, then replays calibration data +- **Exceedance-calibrated thresholds:** Automatically adjusts anomaly thresholds based on observed data distribution (targets ~1% anomaly rate) +- **Per-series profiles:** Each unique combination of table + tags + field gets its own profile +- **Low overhead:** Only writes profiles every N observations (default 50) or every 5 minutes for slow data +- **LRU eviction:** Configurable limit on tracked series to control memory usage + +## How It Works + +On every write, the profiler iterates over each numeric field in each row and updates streaming statistics for a unique series, keyed by `table + tag values + field name`. Tracked stats include EW mean/variance, skewness, kurtosis, write interval, and seasonal variance buckets (hourly or weekly). + +Each series starts in a short **calibration phase** (30 observations). During calibration values are buffered; once it ends, the profiler uses the observed write interval to pick an appropriate fading factor and replays the buffered values through the EW stats so the profile starts off already informed by early history. + +After calibration, every `profile_write_interval` observations (or every 5 minutes for slow streams) the plugin writes a profile snapshot to `_meta.series_profiles` — pattern label, recommended detector mode, threshold, and fading factors. Profiles with fewer than `min_observations` are flagged `profile_mature=false`, so downstream consumers can fall back to safe defaults until the profile has seen enough data. + +## Prerequisites + +- {{% product-name %}} with Processing Engine enabled +- River ML library (`river>=0.23.0`) + +## Quick Start + +### 1. Install River (if not already installed) + +```bash +influxdb3 install package river +``` +### 2. Create the profiler trigger + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_auto_profiler.py \ + --trigger-spec "all_tables" \ + auto_profiler +``` +### 3. Query profiles + +```sql +SELECT * FROM "_meta.series_profiles" ORDER BY time DESC LIMIT 10 +``` +## Trigger Arguments + +Arguments can be passed inline (space-separated lists) or via TOML config file (native TOML lists). Parameter names are the same in both modes. + +| Argument | Required | Default | Description | +|-------------------------------|-----------|------------|----------------------------------------------------------------------------------------------------| +| `include_fields` | No | `""` | Fields to profile (space-separated). If set, only these numeric fields are processed | +| `exclude_fields` | No | `""` | Fields to exclude from profiling (space-separated) | +| `exclude_tables` | No | `""` | Tables to skip when using `all_tables` trigger (space-separated) | +| `string_fields` | No | `""` | String columns that are fields, not tags (space-separated). All other strings become tags | +| `max_series` | No | `1000` | Max unique series to track (LRU eviction) | +| `initial_fading_factor` | No | `0.3` | Initial fading factor for EW stats. Auto-adapted based on write frequency after calibration | +| `seasonal_period` | No | `"hourly"` | Seasonal period for variance buckets: `"hourly"` (24 buckets) or `"weekly"` (168 buckets) | +| `profile_write_interval` | No | `50` | Write profile every N observations per series | +| `min_observations` | No | `50` | Observations needed before profile is considered mature | +| `log_profiles` | No | `false` | Log profile updates to server log | +| `checkpoint_interval_seconds` | No | `1800` | Seconds between model checkpoints to the database | +| `max_checkpoint_age_hours` | No | `24` | Ignore checkpoints older than this when restoring | +| `min_checkpoint_observations` | No | `10` | Skip checkpointing profiles with fewer observations than this (avoids persisting cold-start state) | +| `config_file_path` | No | — | Path to TOML config file. Supports absolute paths or relative paths (resolved via PLUGIN_DIR) | + +### Inline example + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_auto_profiler.py \ + --trigger-spec "all_tables" \ + --trigger-arguments 'include_fields=temperature humidity' 'exclude_tables=debug_info system_logs' \ + auto_profiler +``` +### TOML config example + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_auto_profiler.py \ + --trigger-spec "all_tables" \ + --trigger-arguments config_file_path=river_auto_profiler_config.toml \ + auto_profiler +``` +Or with an absolute path (no PLUGIN_DIR needed): + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_auto_profiler.py \ + --trigger-spec "all_tables" \ + --trigger-arguments config_file_path=/etc/influxdb3/river_auto_profiler_config.toml \ + auto_profiler +``` +See `river_auto_profiler_config.toml` for an example configuration. + +## Column Classification + +The plugin classifies each column in incoming data: + +1. `time` — skipped +2. String columns listed in `string_fields` — skipped (not a tag, not numeric) +3. Other string columns — treated as **tags** (used in series key) +4. Numeric columns (`int`, `float`, not `bool`) — treated as **fields** to profile + - Filtered by `include_fields` (if set, only listed fields are profiled) + - Filtered by `exclude_fields` (excluded fields are skipped) + +## Table Filtering + +All tables starting with `_` are automatically skipped (for example, `_meta.*`, `_anomalies.*`, `_system.*`, `_forecasts.*`). Additional tables can be excluded via `exclude_tables`. + +## Output Schema + +Profiles are written to `_meta.series_profiles`. + +**Tags:** `source_table`, `field_name`, plus all original tags from the source data. + +**Fields:** + +| Field | Type | Description | +|--------------------------------|----------|--------------------------------------------------------------------| +| `observations` | integer | Total observation count | +| `write_interval_seconds` | float | Average seconds between writes | +| `value_mean` | float | Exponentially weighted mean | +| `value_std` | float | Exponentially weighted standard deviation | +| `value_min` | float | Minimum observed value | +| `value_max` | float | Maximum observed value | +| `coefficient_of_variation` | float | std / mean (key metric for tuning) | +| `pattern_label` | string | Data pattern: stable, noisy, trending, seasonal, bursty | +| `recommended_detector_mode` | string | Recommended detector mode for anomaly detector | +| `seasonality_strength` | float | Seasonality strength (0.0-1.0) | +| `trend_strength` | float | Trend strength (0.0-1.0) | +| `data_skewness` | float | Data asymmetry (skewness) | +| `data_kurtosis` | float | Heavy-tailedness (kurtosis) | +| `recommended_threshold` | float | Exceedance-calibrated `rolling_std_threshold` for anomaly detector | +| `recommended_fading_factor` | float | Recommended `ew_fading_factor` for anomaly detector | +| `recommended_seasonal_fading` | float | Recommended seasonal fading factor | +| `profile_mature` | boolean | True if observations ≥ min_observations | +| `seasonality_ready` | boolean | True if enough seasonal buckets are filled for reliable detection | +| `seasonal_buckets_filled` | integer | Number of seasonal buckets with 2+ observations | + +## Integration with Anomaly Detector + +The recommended parameters written to `_meta.series_profiles` are consumed by the companion `river_anomaly_detector` plugin when it runs with `auto_tune=true` (default). Deploying both triggers on the same database lets the detector auto-tune per series: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_auto_profiler.py \ + --trigger-spec "all_tables" \ + auto_profiler + +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_anomaly_detector.py \ + --trigger-spec "all_tables" \ + anomaly_detector +``` +Until a profile matures (`profile_mature=true`, i.e. at least `min_observations` observations), the anomaly detector falls back to conservative defaults. + +## Pattern Classification + +The profiler classifies each series into one of five patterns based on streaming statistics: + +| Condition | Pattern | Recommended Detector Mode | +|----------------------------|-----------|------------------------------| +| Seasonality strength > 0.4 | seasonal | zscore_conservative seasonal | +| Trend strength > 0.7 | trending | zscore_conservative adwin | +| CV < 0.05 | stable | zscore_low | +| Kurtosis > 5 or CV > 1.0 | bursty | zscore_adaptive | +| CV > 0.2 | noisy | zscore_high | +| Otherwise | stable | zscore_low | + +### Seasonality Detection + +Uses hourly (24 buckets) or weekly (168 buckets) variance buckets. If the coefficient of variation across bucket variances is high, the data is seasonal. Requires at least 25% of buckets with 2+ observations each. The `seasonality_ready` field in the output indicates whether enough data has been collected for reliable seasonality detection. + +### Trend Detection + +Uses EW mean drift — compares a fast EW mean (fading factor 0.3) against a slow EW mean (fading factor 0.05). Large divergence relative to the standard deviation indicates trending data. Requires at least 20 observations. + +## Tuning Rules + +### Threshold Recommendation (exceedance-calibrated) + +The threshold is calibrated automatically using exceedance rate tracking. The profiler monitors what fraction of observations fall outside `mean ± threshold × std` and adjusts the threshold to target a ~1% anomaly rate. The threshold is bounded to [2.5, 10.0] and calibrated every 200 observations. Initial value is set from pattern classification: + +| Data Pattern | Initial Threshold | +|--------------------|---------------------| +| stable | 3.5 | +| noisy (CV 0.2-0.5) | 6.0 | +| noisy (CV > 0.5) | 8.0 | +| trending | 5.0 | +| seasonal | 5.0 | +| bursty | 7.0 | + +### Fading Factor Recommendation (based on write frequency) + +| Write Interval | Recommended Factor | +|----------------|---------------------| +| < 10 seconds | 0.1 | +| 10s - 60s | 0.2 | +| 1min - 5min | 0.3 | +| > 5 minutes | 0.5 | + +### Seasonal Fading Factor Recommendation + +| Write Interval | Recommended Factor | +|----------------|---------------------| +| < 60 seconds | 0.05 | +| 1min - 5min | 0.1 | +| > 5 minutes | 0.2 | + +## Model Persistence + +The plugin automatically checkpoints and restores models so they survive server restarts. + +### Checkpoint + +Every `checkpoint_interval_seconds` (default: 1800 = 30 minutes), the plugin pickles each per-series profile, compresses with zlib, base64-encodes, and writes to `_system.model_checkpoints`. Large models are automatically chunked across multiple rows (60KB chunks) to stay within InfluxDB's string field limit. + +Profiles with fewer than `min_checkpoint_observations` observations (default: 10) are skipped — there is no point persisting cold-start state that would be rebuilt from scratch on restore anyway. + +### Restore + +On the first invocation after a server restart (detected by an empty profile cache), the plugin queries `_system.model_checkpoints` for the latest checkpoint per series. Checkpoints older than `max_checkpoint_age_hours` (default: 24) are ignored. Restored profiles resume profiling immediately with their full statistical history. + +### Checkpoint Schema (`_system.model_checkpoints`) + +| Column | Type | Description | +|-------------------------|---------|-----------------------------------------------| +| `plugin` | tag | `"river_auto_profiler"` | +| `series_key` | tag | Unique series identifier | +| `chunk_index` | tag | 0-based chunk index (for multi-row models) | +| `chunk_total` | tag | Total chunks for this model | +| `model_data` | string | Base64-encoded, zlib-compressed pickle | +| `model_type` | string | `"SeriesProfile"` | +| `observation_count` | integer | Observations the profile has processed | +| `checkpoint_size_bytes` | integer | Size of the compressed data | + + +## Logging + +Logs are stored in the `_internal` database (or the database where the trigger is created) in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" +``` + +Log columns: +- **event_time**: Timestamp of the log event +- **trigger_name**: Name of the trigger that generated the log +- **log_level**: Severity level (INFO, WARN, ERROR) +- **log_text**: Message describing the action or error + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/river-forecaster.md b/content/shared/influxdb3-plugins/plugins-library/official/river-forecaster.md new file mode 100644 index 0000000000..c24396ef6c --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/river-forecaster.md @@ -0,0 +1,231 @@ + + +An {{% product-name %}} Processing Engine plugin that provides online time-series forecasting using [River ML](https://riverml.xyz)'s SNARIMAX model. The plugin learns incrementally from incoming data and periodically produces multi-step-ahead forecasts, all within a single WAL trigger. + +## Features + +- **Online learning:** SNARIMAX model updates incrementally with each observation — no batch retraining needed +- **Single-trigger design:** Learns from incoming data and produces forecasts when previous predictions are fully consumed — self-throttling per model +- **Auto-horizon:** Automatically determines forecast horizon from the model's own observed write frequency +- **Per-series models:** Each unique combination of table + tags + field gets its own forecast model +- **Explicit table selection:** You specify which tables and fields to forecast — no surprise cardinality explosions +- **LRU eviction:** Configurable limit on tracked series to control memory usage + +## Prerequisites + +- {{% product-name %}} with Processing Engine enabled +- River ML library (`river>=0.23.0`) + +## Quick Start + +### 1. Install River + +```bash +influxdb3 install package river +``` +### 2. Create the trigger + +Specify which tables to forecast with `include_tables`: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_forecaster.py \ + --trigger-spec "all_tables" \ + --trigger-arguments "include_tables=system_cpu system_memory" \ + forecaster +``` +The plugin will start learning from all numeric fields in the specified tables and produce forecasts once models warm up (default: 30 observations). Each model produces a new forecast automatically once its previous predictions have been fully consumed by incoming actuals. + +### 3. Query forecasts + +```sql +SELECT * FROM "_forecasts.system_cpu" +WHERE field_name = 'idle' +ORDER BY time DESC LIMIT 12 +``` +## Customized Example + +Forecast specific fields with tuned parameters: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_forecaster.py \ + --trigger-spec "all_tables" \ + --trigger-arguments "include_tables=system_cpu system_memory" "include_fields=idle used available" "max_series=100" "default_horizon=24" "log_forecasts=true" \ + forecaster +``` +## Trigger Arguments + +| Argument | Required | Default | Description | +|-------------------------------|------------|------------|--------------------------------------------------------------------------------------------------------------------------------------------------| +| `include_tables` | **Yes** | — | Space-separated tables to forecast (for example, `system_cpu system_memory`). In TOML config use a list. | +| `include_fields` | No | *(all)* | Space-separated fields to forecast (for example, `idle used`). Omit to use all numeric fields. In TOML config use a list. | +| `string_fields` | No | *(none)* | Space-separated string column names that should be treated as fields, not tags. All other string columns become tags. In TOML config use a list. | +| `max_series` | No | `50` | Max unique series to track (LRU eviction) | +| `min_observations` | No | `30` | Minimum observations before forecasting | +| `default_horizon` | No | `12` | Fallback horizon if the model's own write_interval estimate is unavailable | +| `forecast_target_seconds` | No | `3600` | Target forecast window for auto-horizon | +| `snarimax_p` | No | `0` (auto) | SNARIMAX autoregressive order. 0 = auto-tune from write frequency (~1h lookback, clamped 4-30) | +| `snarimax_d` | No | `1` | SNARIMAX differencing order | +| `snarimax_q` | No | `1` | SNARIMAX moving average order | +| `calibration_count` | No | `30` | Observations buffered before finalizing SNARIMAX `p` (only used when `snarimax_p=0`). Larger = more reliable interval estimate, longer warm-up. | +| `log_forecasts` | No | `false` | Log forecast details | +| `checkpoint_interval_seconds` | No | `1800` | Seconds between model checkpoints to the database | +| `max_checkpoint_age_hours` | No | `24` | Ignore checkpoints older than this when restoring | +| `min_checkpoint_observations` | No | `10` | Skip checkpointing models with fewer observations than this (avoids persisting cold-start models) | +| `config_file_path` | No | — | Path to TOML config file | + +## Output Schema + +Forecasts are written to `_forecasts.{source_table}`. + +**Tags:** All original tags from the source row, plus `field_name` (name of the forecasted field) and `model` (always `"snarimax"`). + +**Fields:** + +| Field | Type | Description | +|------------------|---------|--------------------------------------------------| +| `horizon_step` | integer | 1-based index of this step in the forecast run | +| `forecast_value` | float | Predicted value for this future step | +| `horizon_total` | integer | Total steps in this forecast | +| `observations` | integer | How many observations the model has learned from | + +**Timestamps:** Each forecast row gets a future timestamp: +``` +forecast_time = last_observed_time + (step * write_interval_seconds) +``` +## How It Works + +### SNARIMAX Model + +River's `time_series.SNARIMAX` is an online ARIMA variant. Default parameters: + +| Parameter | Default | Rationale | +|--------------------|------------|---------------------------------------------------------------------------------------------------------------| +| `p` (AR order) | `0` (auto) | Auto-tuned from write frequency to capture ~1 hour of lag history (clamped 4-30). Set explicitly to override. | +| `d` (differencing) | `1` | Handle non-stationary trends | +| `q` (MA order) | `1` | Basic error correction | + +### Calibration Phase + +When `snarimax_p=0` (auto — the default), the model cannot be created immediately because it does not yet know the write frequency. Instead the plugin enters a calibration phase: + +1. The first `calibration_count` observations (default: 30) are buffered in memory. A smoothed write-interval estimate is tracked in parallel. +2. Once the buffer is full, the plugin chooses `p = round(3600 / write_interval_s)` (clamped to `[4, 30]`), creates the SNARIMAX model, and replays the buffered observations through it. +3. From that point on the model learns online on every incoming observation — calibration is a one-time bootstrap. + +Calibration happens only once per series. Increasing `calibration_count` gives a more reliable write-interval estimate at the cost of a longer warm-up. When `snarimax_p` is set to a non-zero value, calibration is skipped entirely and the model is created immediately. + +### Warm-up Behavior + +Models need at least `min_observations` (default: 30) before producing forecasts. During warm-up, the plugin continues learning but skips the series when producing forecasts. + +When `snarimax_p=0`, the calibration phase counts toward the observation total: with the defaults (`calibration_count=30`, `min_observations=30`) the first forecast is produced right after calibration finishes. If `min_observations > calibration_count`, the model still needs extra observations after calibration before it starts forecasting. + +### Auto-Horizon + +The forecast horizon — how many future steps each run predicts — is derived from each model's own EW-smoothed write interval (tracked from incoming timestamps). The goal is to cover roughly `forecast_target_seconds` of future data per run: + +1. **Primary:** `horizon = max(1, round(forecast_target_seconds / write_interval))`. With defaults (1 hour target, 10 s write interval) this gives 360 future points per forecast. +2. **Fallback:** `default_horizon` (12) — used only when the interval estimate is not yet available (e.g. immediately after a checkpoint restore, before the second observation has arrived). + +### LRU Eviction + +When the number of tracked series exceeds `max_series`, the least recently used series is evicted. If an evicted series reappears, it starts fresh with a new warm-up period. + +## Model Persistence + +Models are stored in `influxdb3_local.cache` (in-memory) and persist across WAL flushes. To survive server restarts, the plugin automatically checkpoints and restores them. + +### Checkpoint + +Every `checkpoint_interval_seconds` (default: 1800 = 30 minutes), the plugin pickles each per-series model, zlib-compresses it, base64-encodes the result, and writes it to `_system.model_checkpoints`. Large checkpoints are split into chunks that fit within InfluxDB's 64 KB string field limit. This is time-gated so it only runs once per interval regardless of how often WAL flushes trigger the plugin. + +Models with fewer than `min_checkpoint_observations` observations (default: 10) are skipped — there is no point persisting cold-start state that would be rebuilt from scratch on restore anyway. + +### Restore + +On the first invocation after a server restart (detected by an empty model cache), the plugin queries `_system.model_checkpoints` for the latest checkpoint per series. Checkpoints older than `max_checkpoint_age_hours` (default: 24) are ignored. Chunks are grouped by their timestamp so that chunks from different checkpoint runs are never mixed together. Restored models resume learning and forecasting immediately. + +### Checkpoint Schema (`_system.model_checkpoints`) + +| Column | Type | Description | +|-------------------------|---------|---------------------------------------------| +| `plugin` | tag | `"river_forecaster"` | +| `series_key` | tag | Unique series identifier | +| `chunk_index` | tag | 0-based index of this chunk | +| `chunk_total` | tag | Total number of chunks for this checkpoint | +| `model_data` | string | Base64-encoded zlib-compressed pickle chunk | +| `model_type` | string | `"SeriesForecaster"` | +| `observation_count` | integer | Observations the model has processed | +| `checkpoint_size_bytes` | integer | Size of the compressed pickle | + +## Forecast Evaluation + +After each forecast run, the plugin stores predictions in memory and compares them step-by-step against incoming actual values. A new forecast is only produced once all steps from the previous forecast have been evaluated, so MAE always covers the full forecast horizon. + +MAE is tracked per model in memory (and persisted via checkpoints). When `log_forecasts=true`, it is written to the plugin log after each forecast run that accumulated new evaluation points, in the form `EVAL: MAE= ( points)`. + +## TOML Configuration + +Instead of passing trigger arguments inline, you can use a TOML config file: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename river_forecaster.py \ + --trigger-spec "all_tables" \ + --trigger-arguments config_file_path=river_forecaster_config.toml \ + forecaster +``` +See `river_forecaster_config.toml` for an example configuration. + +## Query Examples + +```sql +-- All forecasts for a specific host and field +SELECT time, horizon_step, forecast_value +FROM "_forecasts.system_cpu" +WHERE host = 'server01' AND field_name = 'idle' +ORDER BY time DESC LIMIT 12 + +-- Latest forecast run for one series (all horizon steps) +SELECT time, horizon_step, forecast_value, horizon_total +FROM "_forecasts.system_cpu" +WHERE host = 'server01' AND field_name = 'idle' + AND time > now() - interval '2 hours' +ORDER BY horizon_step ASC + +-- Recent actuals for the same series (for manual comparison against forecasts) +SELECT time, idle +FROM system_cpu +WHERE host = 'server01' +ORDER BY time DESC LIMIT 12 +``` + +## Logging + +Logs are stored in the `_internal` database (or the database where the trigger is created) in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" +``` + +Log columns: +- **event_time**: Timestamp of the log event +- **trigger_name**: Name of the trigger that generated the log +- **log_level**: Severity level (INFO, WARN, ERROR) +- **log_text**: Message describing the action or error + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/sagemaker.md b/content/shared/influxdb3-plugins/plugins-library/official/sagemaker.md new file mode 100644 index 0000000000..8d683072d8 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/sagemaker.md @@ -0,0 +1,620 @@ + + +The SageMaker Inference Plugin enables periodic, real-time scoring of time-series data in {{% product-name %}} against an Amazon SageMaker endpoint. On every scheduled tick the plugin queries recent rows from a measurement, builds a request body matching the shape your SageMaker model expects, calls `InvokeEndpoint`, parses the response, and writes the predictions back into InfluxDB as a new measurement. It supports both batch and per-row inference, single- and multi-output models, custom containers, AWS built-in algorithms, TensorFlow Serving, Hugging Face / JumpStart LLMs, SageMaker Canvas tabular models, multi-model endpoints, and time-series forecasting models such as Amazon Chronos-Bolt. + +Key features: +- Pulls rows from any measurement with a configurable lookback window and row limit +- Builds CSV or one of eight JSON body shapes per row or in batch +- Time-series forecasting mode (`inputs_timeseries` + `forecast_output=true`): collects N historical rows into one request and expands each array output field into N separate prediction rows +- Optional Hugging Face / LLM-style nested parameters via `extra_body`, including array values (`[v1;v2;v3]`) +- Multi-model endpoints via `target_model` +- Type-aware response extraction (JSON, JSON Lines, CSV, plain text) +- Per-row prediction extraction with [JMESPath](https://jmespath.org/) for JSON/JSONLines or column index for CSV +- Multi-output predictions (e.g. score + class label + confidence) via `output_fields` +- Optional response-driven timestamps for forecasting models +- Tag-based filtering of source data, with auto-tags identifying the SageMaker endpoint, source measurement, region, and model + +## Configuration + +Plugin parameters can be provided in two ways: as **trigger arguments** (inline key-value pairs) or via a **TOML configuration file**. Both approaches accept the same parameter names. The TOML approach is recommended when you have complex configs such as many `feature_order` tokens, `extra_body` entries, or multiple `tag_values` filters, because it is easier to read and maintain. + +### Option 1: Trigger arguments (inline) + +Pass parameters as a comma-separated list in `--trigger-arguments`. Use `|` as a separator inside list-valued parameters (`feature_order`, `output_fields`, `extra_body`, `tag_values`): + +```bash +influxdb3 create trigger \ + --database mydb \ + --path "sagemaker.py" \ + --trigger-spec "every:1m" \ + --trigger-arguments 'endpoint_name=my-endpoint,source_measurement=sensor_data,feature_order={motor_speed}|{ambient_temperature},output_fields=score=predictions[*].score' \ + sagemaker_score +``` +### Option 2: TOML configuration file + +Put all parameters into a `.toml` file, then pass only `config_file_path` in the trigger arguments: + +```bash +influxdb3 create trigger \ + --database mydb \ + --path "sagemaker.py" \ + --trigger-spec "every:1m" \ + --trigger-arguments 'config_file_path=sagemaker_config.toml' \ + sagemaker_score +``` +The `config_file_path` value is resolved as follows: +- **Absolute path** (e.g. `/etc/plugins/sagemaker_config.toml`) — used as-is. +- **Relative path** (e.g. `sagemaker_config.toml`) — resolved relative to the plugin directory taken from the `INFLUXDB3_PLUGIN_DIR` environment variable (set by the Processing Engine), falling back to `PLUGIN_DIR`. + +When `config_file_path` is provided all other trigger arguments are ignored; all parameters must be in the file. + +#### TOML format + +The TOML file uses the same parameter names as trigger arguments. The key format differences are: + +| Parameter | Trigger args format | TOML format | +|-------------------|-----------------------------------------------|------------------------------------------------------------------| +| `feature_order` | `{col1}\|{col2}\|0.0` (pipe-separated string) | `feature_order = ["{col1}", "{col2}", "0.0"]` (array) | +| `output_fields` | `score=pred[*].score\|label=pred[*].label` | `output_fields = ["score=pred[*].score", "label=pred[*].label"]` | +| `extra_body` | `parameters.top_p=0.9\|parameters.n=3` | `extra_body = ["parameters.top_p=0.9", "parameters.n=3"]` | +| `batch_inference` | `"true"` or `"false"` (string) | `batch_inference = true` (native bool) | +| `limit` | `"10"` (string) | `limit = 10` (integer) | +| `tag_values` | `sensor_id:A1@A2.env:prod` (encoded string) | `[tag_values]` section (see below) | + +Tag filters in TOML use a dedicated section where each key maps to a list of allowed values: + +```toml +[tag_values] +sensor_id = ["A1", "A2"] +env = ["prod"] +``` +#### Minimal TOML example + +```toml +endpoint_name = "my-endpoint-2025-01-15" +source_measurement = "motor_data" +region = "eu-central-1" +interval = "5min" +limit = 10 +content_type = "application/json" +accept = "application/json" +json_shape = "instances_array" +batch_inference = true + +feature_order = ["{motor_speed}", "{ambient_temperature}", "0.0"] +output_fields = ["score=predictions[*].score"] + +target_measurement = "motor_predictions" + +[tag_values] +sensor_id = ["A1", "A2"] +``` +A complete reference with examples for all supported body shapes is provided in `sagemaker_config_example.toml`. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines the supported trigger type and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Required parameters + +| Parameter | Type | Default | Description | +|----------------------|----------------|------------|-------------------------------------------------------------------------------------------------------------------------------| +| `endpoint_name` | string | required | Name of the deployed SageMaker real-time endpoint | +| `source_measurement` | string | required | InfluxDB measurement (table) the plugin reads rows from | +| `feature_order` | string / array | required | Tokens describing how to build each row's request body. Pipe-separated string in args; TOML array. See *feature_order syntax* | +| `output_fields` | string / array | required | `name=path` pairs describing how to extract predictions from the response. Pipe-separated in args; TOML array | + +### Request format parameters + +| Parameter | Type | Default | Description | +|----------------|----------------|---------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| `content_type` | string | `application/json` | Request body Content-Type. One of `application/json`, `text/csv` | +| `accept` | string | same as `content_type` | Expected response Content-Type. One of `application/json`, `application/jsonlines`, `text/csv`, `text/plain` | +| `json_shape` | string | `instances_array` | JSON body shape (only when `content_type=application/json`). See *json_shape values* below | +| `extra_body` | string / array | none | `path=value` pairs merged into the JSON body (e.g. nested LLM `parameters`). Pipe-separated in args; TOML array. Only with JSON content types | + +### Inference parameters + +| Parameter | Type | Default | Description | +|-------------------|---------|----------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `interval` | string | `60min` | Lookback window for the source query. Format: `` where unit is `s`, `min`, `h`, `d`, `w`, `m` (months≈30d), `q` (quarters≈91d), `y` (years=365d). Examples: `5min`, `1h`, `30d` | +| `limit` | integer | `1` | Maximum number of rows to read per scheduled call. String or integer accepted | +| `batch_inference` | boolean | `true` | If `true`, send all selected rows in one request and parse a batch response. If `false`, one request per row. String `"true"`/`"false"` (args) or native bool (TOML) | +| `forecast_output` | boolean | `false` | If `true`, `output_fields` paths may return arrays; each position becomes a separate output row. Scalar values are broadcast to all rows. Only supported with `accept=application/json`. Use with `json_shape=inputs_timeseries` for time-series forecasting models | +| `region` | string | `eu-central-1` | AWS region of the SageMaker endpoint | +| `target_model` | string | none | Optional model identifier for **multi-model endpoints** — sets `X-Amzn-SageMaker-Target-Model` header | + +### Output parameters + +| Parameter | Type | Default | Description | +|----------------------|--------|--------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `target_measurement` | string | `_predictions` | Measurement to write predictions into | +| `target_database` | string | trigger's database | Database to write predictions into | +| `timestamp_path` | string | none | Optional path to extract per-row timestamps from the response. JMESPath for JSON/JSONLines, integer column index for CSV. If empty, the plugin uses `time.time_ns()`. If set but a row's timestamp is missing or unparseable, that prediction is skipped with an error (no wall-clock fallback) | + +### Filtering parameters + +| Parameter | Type | Default | Description | +|--------------|----------------|----------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `tag_values` | string / table | none | Tag filter applied to the source query. In args: `tag1:val1@val2.tag2:val3` — dot separates tag pairs, colon separates tag name from values, `@` separates multiple values. In TOML: `[tag_values]` section with `tag = ["v1", "v2"]`. Tags with a single value are also written to the output line | + +## `feature_order` syntax + +`feature_order` is a list of tokens. In trigger arguments it is pipe-separated; in TOML it is a native array. Each token is either a column reference or a literal value. Aliases (`:alias`) are mainly used in object-shape JSON bodies as JSON keys. + +| Token | Meaning | +|-----------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| `{col}` | Take the value of column `col` from the row. In object shapes, the JSON key equals `col` | +| `{col:alias}` | Take the value of column `col`. In object shapes, the JSON key equals `alias`; in array/CSV shapes the alias is ignored (one warning at init) | +| `0`, `0.0`, `-3.14`, `1e3` | Numeric literal (int or float). In CSV the literal is written as text | +| `true`, `false` | Boolean literal (in JSON bodies) | +| `null` | JSON `null` | +| `"text"`, `'text'` | String literal | +| `"text":alias`, `0.0:alias`, etc. | Literal **with key alias**. Required for literals in object shapes (`instances_object`, `raw_object`); ignored elsewhere with a warning | + +To use a literal containing `:` or `|`, wrap it in quotes: `"1:30":time`. + +## `output_fields` syntax + +`output_fields = name=path|name=path|...` (args) or `output_fields = ["name=path", ...]` (TOML) + +Each entry produces one field on the output line. The path syntax depends on `accept`: + +| `accept` | Path syntax | Example | +|---------------------------|------------------------------------------------------------------|-----------------------------------------------------------------------------| +| `application/json` | JMESPath — yields an array in batch mode, scalar in per-row mode | `forecasting=predictions[*].score` (batch) or `forecasting=score` (per-row) | +| `application/jsonlines` | JMESPath, applied per line | `forecasting=score` | +| `text/csv` | integer column index (0-based) | `forecasting=0` | +| `text/plain` | empty (response body becomes value); exactly one entry | `result=` | + +In batch mode, `M` (the number of written rows) equals `min(length)` across all `output_fields` paths and `timestamp_path`. If lengths differ, the plugin truncates and logs a warning. + +In **forecast mode** (`forecast_output=true`), each `output_fields` path may return an array — every array element becomes a separate output row. Paths that return a scalar are broadcast to all rows. All arrays must have equal length; if they differ, the min-length rule applies with a warning. + +The Python type of each extracted value determines the InfluxDB field type: +- `int` → `int64_field` +- `float` → `float64_field` +- `bool` → `bool_field` +- `str` → `string_field` +- `list`/`dict` → row error (use a more specific JMESPath; in forecast mode, nested arrays inside a forecast array are not supported) + +## `json_shape` values + +| Shape | Body produced | Used by | +|-----------------------|-----------------------------------------------------------------------------------------------|------------------------------------------------------------------------| +| `instances_array` | `{"instances": [[v1,v2,...], ...]}` | TF Serving REST (positional) | +| `instances_object` | `{"instances": [{"col": v, ...}, ...]}` | TF Serving REST (named columns) | +| `instances_features` | `{"instances": [{"features": [v1,v2,...]}, ...]}` | Built-in AWS algorithms: KMeans, k-NN, RCF, Linear Learner, NTM, PCA | +| `inputs` | `{"inputs": [v1,v2,...]}` — single row | TF Serving simplified columnar (single tensor) | +| `inputs_array` | `{"inputs": [[v1,v2], ...]}` | TF Serving / PyTorch batch | +| `inputs_flat` | `{"inputs": [v1,v2,...]}` — one value per row, requires exactly one `{col}` token | Hugging Face NLP batch (`{"inputs": ["text1", "text2"]}`) | +| `inputs_timeseries` | `{"inputs": [{"target": [v1,v2,...,vN]}]}` — all rows collected into one target array; requires exactly one `{col}` token | Time-series forecasting models (Amazon Chronos-Bolt, etc.) | +| `raw_array` | `[[v1,v2,...], ...]` | Custom containers, simplified TF Serving | +| `raw_object` | `{"col": v, ...}` — single row | Hugging Face simple, custom containers (e.g. `{"inputs": "text"}`) | + +**Batch compatibility:** `inputs` and `raw_object` produce a single-row body and require either `batch_inference=false` or `limit=1`. All other shapes are batch-friendly. + +**Time-series forecasting:** use `inputs_timeseries` together with `forecast_output=true`. With `limit=N`, all N rows from the source query are packed into a single `target` array and sent in one request. The response's array fields are then expanded back into N output rows — one per forecast step. + +**Object shapes** (`instances_object`, `raw_object`) accept literal tokens **only with an alias** (`literal:alias`); the alias becomes the JSON key. + +## `extra_body` syntax + +`extra_body = path=value|path=value|...` (args) or `extra_body = ["path=value", ...]` (TOML) + +Each entry adds a static value at a JSON path. Dotted paths produce nested objects; values are coerced (numbers, booleans, `null`, quoted strings). **Array values** use semicolons as element separators inside square brackets: `[v1;v2;v3]`. + +```bash +# args — scalar values +extra_body = parameters.max_new_tokens=50|parameters.top_p=0.95|parameters.do_sample=true + +# args — array value (semicolons, no spaces inside brackets) +extra_body = parameters.quantile_levels=[0.1;0.5;0.9]|parameters.prediction_length=10 + +# TOML +extra_body = [ + "parameters.max_new_tokens=50", + "parameters.top_p=0.95", + "parameters.do_sample=true", + "parameters.quantile_levels=[0.1;0.5;0.9]", +] +``` +→ merged into the body as: +```json +{"parameters":{"max_new_tokens":50,"top_p":0.95,"do_sample":true,"quantile_levels":[0.1,0.5,0.9]}} +``` +Restrictions: +- Cannot be used with `content_type=text/csv`. +- Cannot be used with `json_shape=raw_array` (top-level list — nothing to merge into). +- Conflicts with feature_order keys at any depth fail at run time. + +## Auto-tags + +Every output line carries the following tags: + +| Tag | Value | +|--------------------------------|--------------------------------------| +| `sagemaker_endpoint` | `endpoint_name` | +| `sagemaker_source_measurement` | `source_measurement` | +| `sagemaker_region` | `region` | +| `sagemaker_model` | `target_model` *(only when set)* | + +Single-valued tags from `tag_values` are also written to the output line. + +## Software Requirements + +- **{{% product-name %}}** with the Processing Engine enabled +- **AWS credentials** available to the {{% product-name %}} host process (via environment variables, IAM role, or AWS credentials file) with permission to call `sagemaker:InvokeEndpoint` +- **Python packages**: + - `boto3` (AWS SDK) + - `pandas` (timestamp parsing) + - `jmespath` is included as a transitive dependency of `boto3`/`botocore` + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled: + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install required Python packages: + + ```bash + influxdb3 install package boto3 + influxdb3 install package pandas + ``` +3. Make AWS credentials available to {{% product-name %}} (one of): + - Set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, optionally `AWS_SESSION_TOKEN` in the host environment + - Run {{% product-name %}} on an EC2 instance / EKS pod with an IAM role that allows `sagemaker:InvokeEndpoint` + - Mount `~/.aws/credentials` and set `AWS_PROFILE` + +## Trigger setup + +### With trigger arguments + +```bash +influxdb3 create trigger \ + --database mydb \ + --path "sagemaker.py" \ + --trigger-spec "every:1m" \ + --trigger-arguments 'endpoint_name=my-endpoint,source_measurement=sensor_data,feature_order={motor_speed}|{ambient_temperature},output_fields=score=predictions[*].score' \ + sagemaker_score +``` +### With a TOML configuration file + +```bash +influxdb3 create trigger \ + --database mydb \ + --path "sagemaker.py" \ + --trigger-spec "every:1m" \ + --trigger-arguments 'config_file_path=sagemaker_config.toml' \ + sagemaker_score +``` +Place `sagemaker_config.toml` in your plugin directory — the one set via `--plugin-dir` and exposed as `INFLUXDB3_PLUGIN_DIR` (or `PLUGIN_DIR`) — or provide an absolute path. See `sagemaker_config_example.toml` for ready-to-use templates covering all supported endpoint types. + +## Example usage + +### Example 1: SageMaker Canvas tabular model (CSV) + +Score the latest row of sensor data against a Canvas model that expects CSV input and returns a single float per row. + +```bash +influxdb3 create trigger \ + --database factory \ + --path "sagemaker.py" \ + --trigger-spec "every:1m" \ + --trigger-arguments 'endpoint_name=canvas-deployment-2026-01-10,source_measurement=press,feature_order={motor_speed}|{ambient_temperature}|{vibration},content_type=text/csv,accept=text/csv,output_fields=quality_score=0,limit=1,batch_inference=false,region=us-east-1' \ + press_quality +``` +Request body sent to SageMaker: +``` +1500,25.5,0.3 +``` +Response: +``` +0.87 +``` +Result: a row in measurement `press_predictions` with field `quality_score=0.87` (float). + +### Example 2: AWS built-in algorithm — Random Cut Forest anomaly score + +RCF expects `{"instances":[{"features":[...]}]}`. + +```bash +influxdb3 create trigger \ + --database iot \ + --path "sagemaker.py" \ + --trigger-spec "every:30s" \ + --trigger-arguments 'endpoint_name=rcf-anomaly,source_measurement=metrics,feature_order={cpu}|{mem}|{rps}|{p99_ms},json_shape=instances_features,output_fields=anomaly_score=scores[*].score,interval=2min,limit=20' \ + rcf_score +``` +Request body for 20 rows (batch): +```json {lint="false"} +{"instances":[{"features":[42.1,71.0,1834,8.4]}, ..., {"features":[39.9,72.2,1903,9.0]}]} +``` +Response: +```json {lint="false"} +{"scores":[{"score":0.83},{"score":0.71}, ..., {"score":0.92}]} +``` +Each score is written as `anomaly_score` field. Use `timestamp_path` if your model returns timestamps. + +### Example 3: TensorFlow Serving with renamed JSON keys + +Rename InfluxDB columns to keys the model expects (`Speed`, `Temp`, `Vib`). + +```bash +influxdb3 create trigger \ + --database factory \ + --path "sagemaker.py" \ + --trigger-spec "every:30s" \ + --trigger-arguments 'endpoint_name=tf-serving,source_measurement=press,feature_order={motor_speed:Speed}|{ambient_temperature:Temp}|{vibration:Vib},json_shape=instances_object,output_fields=p_fail=predictions[*].p_fail|p_ok=predictions[*].p_ok,limit=10' \ + press_failure +``` +Request body: +```json {lint="false"} +{"instances":[{"Speed":1500,"Temp":25.5,"Vib":0.3}, ...]} +``` +Two prediction fields per row are written: `p_fail` and `p_ok`. + +### Example 4: Hugging Face LLM with parameters (TOML config) + +Score a free-text sensor log against a Hugging Face text classification endpoint, with generation parameters. Using a TOML file makes the `extra_body` list easy to maintain. + +`hf_classifier.toml`: +```toml +endpoint_name = "hf-classifier" +source_measurement = "tickets" +region = "eu-west-1" +interval = "5min" +limit = 1 +content_type = "application/json" +accept = "application/json" +json_shape = "raw_object" +batch_inference = false + +feature_order = ["{text:inputs}"] +output_fields = ["label=label", "score=score"] + +extra_body = [ + "parameters.top_k=5", + "parameters.temperature=0.3", +] + +target_measurement = "ticket_classifications" +``` +```bash +influxdb3 create trigger \ + --database support \ + --path "sagemaker.py" \ + --trigger-spec "every:5min" \ + --trigger-arguments 'config_file_path=hf_classifier.toml' \ + ticket_classifier +``` +Request body: +```json {lint="false"} +{"inputs":"Bearing 7 reports rising temperature for 20 minutes...","parameters":{"top_k":5,"temperature":0.3}} +``` +### Example 5: Forecasting with response timestamps + +DeepAR-style forecast where the model returns predicted timestamps along with values. The plugin writes each prediction at its own future timestamp. + +```bash +influxdb3 create trigger \ + --database forecasts \ + --path "sagemaker.py" \ + --trigger-spec "every:15min" \ + --trigger-arguments 'endpoint_name=deepar-fcast,source_measurement=demand,feature_order={value},json_shape=instances_features,output_fields=forecasting=predictions[*].mean,timestamp_path=predictions[*].t,limit=24,interval=24h,target_measurement=demand_forecasts' \ + demand_forecast +``` +Each forecasted point is written with timestamp from `predictions[*].t` — autodetected as ISO string, Unix seconds, or nanoseconds. + +### Example 6: Multi-model endpoint with tag filtering + +Score only rows from a specific sensor against a specific model deployed on a multi-model endpoint, with the `region` tag carried through to the output. + +```bash +influxdb3 create trigger \ + --database iot \ + --path "sagemaker.py" \ + --trigger-spec "every:1min" \ + --trigger-arguments 'endpoint_name=mme-models,target_model=motor-v3.tar.gz,source_measurement=sensors,feature_order={rpm}|{torque},output_fields=health=predictions[*].health,tag_values=region:eu-central-1.sensor_id:motor-7,limit=5' \ + motor_health +``` +Output lines carry tags `sagemaker_endpoint=mme-models`, `sagemaker_source_measurement=sensors`, `sagemaker_region=eu-central-1` (default region), `sagemaker_model=motor-v3.tar.gz`, `region=eu-central-1`, `sensor_id=motor-7`. + +### Example 7: Time-series forecasting with Amazon Chronos-Bolt (TOML config) + +Collect 20 historical sensor readings and send them to a Chronos-Bolt endpoint as one time series. The model returns mean and quantile forecasts for 10 future steps — each step is written as a separate row. + +`chronos.toml`: +```toml +endpoint_name = "chronos-bolt-small-endpoint" +source_measurement = "sensor_readings" +region = "eu-central-1" +interval = "1h" +limit = 20 +content_type = "application/json" +accept = "application/json" +json_shape = "inputs_timeseries" +batch_inference = true +forecast_output = true + +# Single {col} token required for inputs_timeseries +feature_order = ["{value}"] + +# Each path returns a 10-element array → 10 output rows +# Quantile key names start with a digit, so quote them in JMESPath +output_fields = [ + "mean=predictions[0].mean", + "q10=predictions[0].\"0.1\"", + "q90=predictions[0].\"0.9\"", +] + +# Array values use semicolons as element separator +extra_body = [ + "parameters.prediction_length=10", + "parameters.quantile_levels=[0.1;0.5;0.9]", +] + +target_measurement = "sensor_forecasts" +``` +```bash +influxdb3 create trigger \ + --database iot \ + --path "sagemaker.py" \ + --trigger-spec "every:1h" \ + --trigger-arguments 'config_file_path=chronos.toml' \ + chronos_forecast +``` +Request body sent to SageMaker (20 historical values packed into `target`): +```json {lint="false"} +{"inputs": [{"target": [1.1, 2.3, 1.8, ...]}], "parameters": {"prediction_length": 10, "quantile_levels": [0.1, 0.5, 0.9]}} +``` +Response: +```json {lint="false"} +{"predictions": [{"mean": [-0.0, 3.5, ...], "0.1": [-2.4, 1.5, ...], "0.9": [1.7, 5.6, ...]}]} +``` +Result: 10 rows written to `sensor_forecasts`, each with fields `mean`, `q10`, `q90`. + +## Code overview + +### Files + +- `sagemaker.py` — the main plugin file. Contains the JSON metadata header, configuration parsing, request body builders, response extractors, and the `process_scheduled_call` entry point. +- `sagemaker_config_example.toml` — annotated TOML templates for all supported endpoint types (instances_array, instances_object, CSV, LLM/HuggingFace, multi-model, Chronos-Bolt time-series forecasting). + +### Logging + +Logs are stored in the trigger's database in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database YOUR_DATABASE \ + "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name' ORDER BY event_time DESC LIMIT 50" +``` +Each log line includes a unique `task_id` (UUID) so messages from a single scheduled call can be correlated. Levels: +- **INFO** — initialization, source query, row count, write completion +- **WARN** — Content-Type mismatch, length mismatch in batch response, missing path values, ignored aliases +- **ERROR** — config validation failure, batch inference failure, write failure + +### Main functions + +#### `process_scheduled_call(influxdb3_local, call_time, args)` + +Entry point invoked by the Processing Engine on each tick. + +1. Loads cached config or builds it from `args` (validates schema, parses `feature_order`, `output_fields`, `tag_values`, `extra_body`) +2. Builds and runs the source SQL query (with optional tag filter) +3. Either sends one batched `InvokeEndpoint` call or iterates per row, depending on `batch_inference` +4. Parses the response according to `accept` and `output_fields` +5. Builds typed `LineBuilder` per prediction row with auto-tags, single-valued filter tags, and timestamp +6. Writes all lines in one batched call to InfluxDB + +#### `build_request_body(rows, tokens, content_type, json_shape, extra_body=None)` + +Builds the request body as a Python object first, optionally deep-merges `extra_body`, then serialises to JSON. Returns a CSV string for `content_type=text/csv`. + +#### `extract_outputs(response_body, response_ct, accept, output_fields, timestamp_compiled, ..., forecast_output)` + +Returns a list of `(timestamp_or_None, {field: value})` tuples. Dispatches to one of: +- `_extract_from_json_forecast` — when `forecast_output=True`: fields may be arrays (indexed per row) or scalars (broadcast) +- `_extract_from_json` — single JSON document, JMESPath projections for batch or scalar paths for per-row +- `_extract_from_jsonlines` — one record per line +- `_extract_from_csv` — `csv.reader` per line, integer column index +- `_extract_from_plain` — single scalar from the body + +If the actual response Content-Type differs from the requested `accept`, the plugin warns and tries to parse with the actual one (if supported). + +## Troubleshooting + +### Common issues + +#### Issue: `feature_order references columns that don't exist in ''` + +**Solution:** Check the column names in `feature_order` and confirm the source measurement has data. + +The plugin validates feature_order columns against `information_schema.columns` at start-up. The error message lists missing columns and all available columns — fix the typo in `feature_order` or write some data into the measurement first. + +#### Issue: `Got M prediction(s) for N input row(s)` + +**Solution:** Verify that the SageMaker model returns one prediction per submitted input row, or lower the batch size while debugging. + +The number of predictions returned by the model differs from the number of source rows sent in the batch. The plugin writes whatever predictions came back and warns about the delta. If consistent, check whether your model auto-aggregates or rejects malformed rows. + +#### Issue: `text/plain accepts exactly one output field` + +**Solution:** Use a single output field with an empty path. + +Plain-text responses are scalar — define one entry in `output_fields` with an empty path: `output_fields=result=`. + +#### Issue: `json_shape 'inputs' is not batch-compatible` + +**Solution:** Disable batch inference, limit the query to one row, or choose a batch-compatible JSON shape. + +`inputs` and `raw_object` produce single-row bodies. Either set `batch_inference=false`, set `limit=1`, or switch to a batch-compatible shape (`inputs_array`, `instances_array`, etc.). + +#### Issue: `extra_body conflicts with main body at ''` + +**Solution:** Move the static value to a different path or remove the duplicate key. + +The static path you set in `extra_body` collides with a key produced by `feature_order`. Pick a different alias or remove the conflicting path. + +#### Issue: `Invalid interval format` + +**Solution:** Use a duration in `` form with a supported unit. + +Interval must be ``. Unit must be one of `s`, `min`, `h`, `d`, `w`, `m`, `q`, `y`. Note that `m` means **months** (≈30 days), not minutes — use `min` for minutes. + +#### Issue: AWS authentication errors (`UnrecognizedClientException`, `AccessDeniedException`) + +**Solution:** Provide AWS credentials or an IAM role that can invoke the target SageMaker endpoint. + +The {{% product-name %}} host process needs AWS credentials with permission to call `sagemaker:InvokeEndpoint` on your endpoint. Set `AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY`, configure an IAM role on the instance, or set `AWS_PROFILE` to a profile in `~/.aws/credentials`. + +#### Issue: `Empty response body from SageMaker` + +**Solution:** Check the SageMaker endpoint and model logs in CloudWatch. + +The model returned an empty body. Check the model's CloudWatch logs in AWS — usually it indicates a model exception or an endpoint that is `OutOfService`. + +### Debugging tips + +1. **Inspect the source query** — every scheduled call logs `Source query: SELECT ...`. Run it manually: + + ```bash + influxdb3 query --database mydb "SELECT time, motor_speed, ambient_temperature FROM sensors WHERE time > now() - INTERVAL '5 minutes' ORDER BY time DESC LIMIT 1" + ``` +2. **Test the endpoint outside the plugin** — confirm Content-Type and Accept on a known-good payload: + + ```bash + aws sagemaker-runtime invoke-endpoint \ + --endpoint-name my-endpoint \ + --content-type application/json \ + --accept application/json \ + --body '{"instances":[[1500,25.5,0.3]]}' \ + /tmp/out.json && cat /tmp/out.json + ``` +3. **Switch to per-row mode** — set `batch_inference=false` to isolate which row fails when batch parsing is unclear. + +4. **Force a config rebuild** — config is cached for 1 hour. After updating trigger arguments or the TOML file, delete and recreate the trigger or wait for the cache to expire. + +### Performance considerations + +- **Batch inference** is dramatically cheaper. With 60 rows at 200ms latency, batch sends one HTTP request (~250ms total); per-row sends 60 requests (~12s total). Use `batch_inference=false` only when the model truly does not support batched bodies. +- **Limit your batch size** to stay under SageMaker's 6 MB body limit. Roughly 50–500 rows per batch for tabular data is a safe range. +- **Schema cache** — measurement columns are cached for 1 hour after the first lookup. +- **Config cache** — parsed config is cached for 1 hour; the boto3 client is reused for the duration of one scheduled call. + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/schema-validator.md b/content/shared/influxdb3-plugins/plugins-library/official/schema-validator.md new file mode 100644 index 0000000000..626dcf7337 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/schema-validator.md @@ -0,0 +1,279 @@ + + +> **Note:** This plugin requires {{% product-name %}}.8.2 or later. + +An {{% product-name %}} Processing Engine plugin that validates incoming line protocol data against a user-defined JSON schema. Only data that conforms to the schema is written to a target database or table, enabling a clean data pipeline pattern. + +## Use Case + +You have data coming into a "raw" database (for example, `raw_db`) from various sources. You want to ensure only properly-structured, validated data makes it into your "clean" database (for example, `clean_db`). This plugin sits on the WAL flush trigger and validates every incoming row against your schema definition before writing it to the target. + +**Common patterns:** +- `raw_db` -> validate -> `clean_db` (cross-database) +- `raw_table` -> validate -> `validated_table` (same database, different table) +- `source_table` -> validate -> `source_table_clean` (same database, with suffix) + +This is a **single-file plugin** (`schema_validator.py`) and can be loaded from GitHub via `gh:` trigger paths or created in InfluxDB Explorer. + +> **Note:** The JSON schema configuration file (`schema_validator_config.json`) must be manually uploaded to the plugin directory on the server. There is currently no API for uploading non-plugin files, so Explorer cannot upload it for you. You can use `scp`, `rsync`, or any other file transfer method to place the schema file in the plugin directory alongside the plugin. + +## Features + +- **Measurement validation**: Define a whitelist of allowed measurement/table names +- **Tag validation**: Required/optional tags, allowed tag values +- **Field validation**: Required/optional fields, type checking (float, integer, string, boolean, uint64), allowed field values +- **Field stripping**: Extra tags/fields not defined in the schema are automatically stripped from the output +- **Flexible targeting**: Write to a different database, different table name, or add prefix/suffix +- **Per-table schemas**: Define different validation rules for each measurement +- **Rejection logging**: Optionally log rejected rows and/or write rejection details to a measurement +- **Cached config**: Schema file is cached for 5 minutes to avoid repeated file reads + +## Quick Start + +### 1. Deploy the plugin files + +The plugin code can be deployed via the InfluxDB CLI, Explorer, or GitHub (`gh:`) trigger paths. However, the **schema JSON configuration file must be manually placed** in the plugin directory on the server since there is no API for uploading non-plugin files. + +- `schema_validator.py` - the plugin code (can be uploaded via CLI/Explorer/GitHub) +- `schema_validator_config.json` - your schema definition (must be manually copied to the plugin directory) + +### 2. Create a trigger + +**Cross-database validation (raw_db -> clean_db):** +```bash +influxdb3 create trigger \ + --database raw_db \ + --plugin-filename schema_validator.py \ + --trigger-spec "all_tables" \ + --trigger-arguments schema_file=schema_validator_config.json,target_database=clean_db \ + schema_validator_trigger +``` +**Same database, different table (with suffix):** +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename schema_validator.py \ + --trigger-spec "table:weather" \ + --trigger-arguments schema_file=schema_validator_config.json,target_table_suffix=_clean \ + schema_validator_weather +``` +**Using a TOML config file:** +```bash +influxdb3 create trigger \ + --database raw_db \ + --plugin-filename schema_validator.py \ + --trigger-spec "all_tables" \ + --trigger-arguments config_file_path=schema_validator_trigger_config.toml \ + schema_validator_trigger +``` +### 3. Write data normally + +Write to your raw database as usual. The plugin will automatically validate and forward conforming data. + +```bash +# This row has all required fields -> will be written to clean_db +influxdb3 write --database raw_db \ + "weather,location=us-east,station_id=ST001 temperature=72.5,humidity=45.2" + +# This row is missing required tag 'station_id' -> will be rejected +influxdb3 write --database raw_db \ + "weather,location=us-east temperature=72.5,humidity=45.2" +``` +## Schema Configuration (JSON) + +The schema is defined in a JSON file. Here is the full structure: + +```json +{ + "allowed_measurements": ["weather", "cpu"], + + "tables": { + "weather": { + "target_table": "weather_clean", + "tags": { + "location": { + "required": true, + "allowed_values": ["us-east", "us-west", "eu-west"] + }, + "station_id": { + "required": true + }, + "region": { + "required": false + } + }, + "fields": { + "temperature": { + "required": true, + "type": "float" + }, + "humidity": { + "required": true, + "type": "float" + }, + "condition": { + "required": false, + "type": "string", + "allowed_values": ["sunny", "cloudy", "rain", "snow"] + } + } + } + } +} +``` +### Schema Fields Reference + +#### Top-level + +| Field | Type | Description | +|---|---|---| +| `allowed_measurements` | `list[str]` (optional) | Whitelist of allowed measurement/table names. If omitted or empty (`[]`), no measurement filter is applied — all measurements fall through to the `tables` rules. | +| `tables` | `dict` (required) | Map of measurement name -> table schema definition. Must contain at least one entry. Only measurements with an entry here will be validated and written to the target. | + +#### Table Schema + +| Field | Type | Description | +|---|---|---| +| `target_table` | `str` (optional) | Override the target measurement name. Takes precedence over prefix/suffix args. | +| `tags` | `dict` | Map of tag name -> tag definition. | +| `fields` | `dict` | Map of field name -> field definition. | +#### Tag Definition + +| Field | Type | Description | +|---|---|---| +| `required` | `bool` | If `true`, the tag must be present on every row. | +| `allowed_values` | `list` (optional) | Whitelist of allowed values for this tag. | + +#### Field Definition + +| Field | Type | Description | +|---|---|---| +| `required` | `bool` | If `true`, the field must be present on every row. | +| `type` | `str` (optional) | Expected data type: `"float"`, `"integer"`, `"string"`, `"boolean"`, `"uint64"`. | +| `allowed_values` | `list` (optional) | Whitelist of allowed values for this field. | + +## Trigger Arguments + +These can be passed via `--trigger-arguments` or in a TOML config file. + +| Argument | Required | Default | Description | +|---|---|---|---| +| `schema_file` | Yes | - | Path to the JSON schema file (relative to PLUGIN_DIR). | +| `target_database` | No | (same db) | Database to write validated data to. | +| `target_table_prefix` | No | `""` | Prefix added to measurement names in the target. | +| `target_table_suffix` | No | `""` | Suffix added to measurement names in the target. | +| `log_rejected` | No | `"true"` | Log info about rejected rows. | +| `log_accepted` | No | `"false"` | Log info about accepted rows. | +| `write_rejection_log` | No | `"false"` | Write rejection details to `_schema_rejections` measurement. | +| `config_file_path` | No | - | Path to TOML config file to override these arguments. | + +## Validation Logic + +For each incoming row, the plugin checks (in order): + +1. **Measurement name**: Is the table name in `allowed_measurements`? (if defined) +2. **Table schema exists**: Is there a schema definition for this table in `tables`? If not, the table is skipped. +3. **Required tags**: Are all required tags present? +4. **Tag values**: Are tag values in the `allowed_values` list? (if defined) +5. **Required fields**: Are all required fields present? +6. **Field types**: Do field values match the expected type? (if defined) +7. **Field values**: Are field values in the `allowed_values` list? (if defined) +8. **Field stripping**: Any extra tags/fields not defined in the schema are stripped from the output. + +If **any** check fails, the row is rejected and not written to the target. + +## Examples + +### IoT Sensor Validation + +Ensure sensor readings always have a device_id, valid sensor type, and a numeric value: + +```json +{ + "allowed_measurements": ["sensor_readings"], + "tables": { + "sensor_readings": { + "target_table": "sensors_validated", + "tags": { + "device_id": { "required": true }, + "sensor_type": { + "required": true, + "allowed_values": ["temperature", "pressure", "humidity"] + } + }, + "fields": { + "value": { "required": true, "type": "float" }, + "status": { + "required": false, + "type": "string", + "allowed_values": ["ok", "warning", "critical"] + } + } + } + } +} +``` +### Multi-table with Cross-database + +Validate weather and cpu data from raw_db, writing clean data to clean_db: + +```bash +influxdb3 create trigger \ + --database raw_db \ + --plugin-filename schema_validator.py \ + --trigger-spec "all_tables" \ + --trigger-arguments schema_file=schema_validator_config.json,target_database=clean_db,write_rejection_log=true \ + schema_validator_all +``` +### Monitoring Rejections + +Query the rejection log to see what data is being rejected and why: + +```sql +SELECT * FROM _schema_rejections +WHERE time > now() - INTERVAL '1 hour' +ORDER BY time DESC +``` +## File Structure + +``` +schema_validator/ + schema_validator.py # Main plugin code + schema_validator_config.json # Example schema definition + schema_validator_trigger_config.toml # Example TOML trigger config + README.md # This file +``` +## Notes + +- The schema JSON file is cached for 5 minutes. To force a reload, restart the trigger or wait for the cache to expire. +- Extra tags/fields not defined in the schema are silently stripped from the output (not written to the target). +- Measurements without an entry in `tables` are skipped entirely (no data written). +- The `target_table` property in a table schema takes precedence over `target_table_prefix`/`target_table_suffix`. +- The `_schema_rejections` table (if `write_rejection_log=true`) is written to the target database. +- Uses `write_sync` / `write_sync_to_db` with `no_sync=True` for optimal memory performance. +- Valid rows are batched per table and written in a single call for efficiency. + +## Logging + +Logs are stored in the `_internal` database (or the database where the trigger is created) in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" +``` + +Log columns: +- **event_time**: Timestamp of the log event +- **trigger_name**: Name of the trigger that generated the log +- **log_level**: Severity level (INFO, WARN, ERROR) +- **log_text**: Message describing the action or error + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/signal-filter.md b/content/shared/influxdb3-plugins/plugins-library/official/signal-filter.md new file mode 100644 index 0000000000..38946b7905 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/signal-filter.md @@ -0,0 +1,401 @@ + + +The Signal Filter Plugin applies streaming digital IIR filters (Butterworth, +Chebyshev I, Bessel — lowpass/highpass/bandpass/bandstop — or manually supplied +second-order sections) to numeric time-series fields as they are written to +{{% product-name %}}. Filtering is causal and per series: each tag combination gets its own +filter whose delay-line state persists in the trigger cache across WAL commits, so +output over a stream of small writes is identical to filtering the whole signal at +once. It pairs naturally with the +[`signal_generator`](https://github.com/influxdata/influxdb3_plugins/tree/main/influxdata/signal_generator) +plugin but works with any measurement carrying numeric fields. + +- **Streaming, stateful**: per-series delay-line state persists in the trigger + cache, so chunked writes filter identically to a single batch +- **Preset or manual design**: SciPy-designed Butterworth, Chebyshev I, and Bessel + prototypes, or raw second-order sections you supply +- **Automatic sample-rate inference**: infers `fs` per series from the data when + not configured explicitly +- **Multi-field, multi-series**: filters each configured field and each tag + combination independently +- **Flexible output routing**: rename, prefix/suffix, and redirect the filtered + field to another measurement or database + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` +flag (`influxdb3 create trigger`) or in the `trigger_arguments` field of the API. +Values are strings; the plugin coerces them. Alternatively, supply every parameter +from a TOML file via `config_file_path` — see [TOML configuration](#toml-configuration). + +> **CLI limitation:** the `sos` argument is a JSON array containing commas, and +> `influxdb3 create trigger --trigger-arguments` splits on every comma, so the value +> is fragmented before it reaches the plugin. Configure `sos` (i.e. `design_type=manual`) +> through the InfluxDB 3 Explorer UI, the `/api/v3/configure/processing_engine_trigger` +> API, or a TOML file — not the CLI. Space-separated arguments such as `input_fields` +> and `tag_keys` are unaffected. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported +trigger types and configuration parameters. This metadata enables the +[InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display +and configure the plugin. + +### Input parameters + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `input_measurement` | string | *(all tables)* | Table to filter. If omitted, every table the trigger fires for is filtered. | +| `input_fields` | string | `value` | Space-separated numeric fields to filter; each is filtered independently. | +| `tag_keys` | string | *(auto)* | Space-separated tag columns that define a series. Defaults to all string-valued columns except `time` and the input fields. Set explicitly if a string *field* would otherwise be mistaken for a tag. | + +### Design parameters + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `design_type` | string | `preset` | `preset` (SciPy-designed IIR) or `manual` (SOS coefficients supplied via `sos`). | +| `prototype` | string | `butter` | Preset prototype: `butter`, `cheby1`, or `bessel`. | +| `order` | integer | `4` | Filter order, 1–12. Band filters yield effective order 2N. | +| `ripple` | float | — | Passband ripple in dB, 0.01–80 (0.1–3 typical). Required for `cheby1`; invalid otherwise. | +| `filter_type` | string | `lowpass` | `lowpass`, `highpass`, `bandpass`, or `bandstop`. | +| `fc` | float | — | Convenience alias for the single cutoff (Hz) of lowpass/highpass. Invalid for band types or together with the parameter it maps to. | +| `fc1` | float | — | Lower cutoff (Hz). Required for highpass and band filters. | +| `fc2` | float | — | Upper cutoff (Hz). Required for lowpass and band filters. Band filters require `fc1 < fc2`. | +| `bessel_norm` | string | `phase` | Bessel normalization: `phase`, `delay`, or `mag`. Bessel only. | +| `sos` | JSON string | — | Manual second-order sections as JSON `[[b0,b1,b2,a0,a1,a2], ...]`. Required for `design_type` `manual`; invalid otherwise. Rows are normalized by `a0`; unstable filters (any pole magnitude ≥ 1) are rejected. | +| `sample_rate` | float | *(inferred)* | Sample rate in Hz for preset design. If omitted, inferred per series from the median inter-sample interval and frozen once enough samples are seen (see [Sample-rate inference](#sample-rate-inference)). | +| `init_from_first_sample` | boolean | `true` | Initialize filter state from the first sample to suppress the startup transient. | + +### Output parameters + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `output_target_database` | string | *(trigger db)* | Database to write filtered output to. | +| `output_measurement` | string | *(source table)* | Measurement to write filtered output to. | +| `output_field` | string | *(source field)* | Base name override for the output field. Only valid when a single input field is configured. | +| `field_prefix` | string | *(empty)* | Prefix for the output field name. | +| `field_suffix` | string | `_filtered` | Suffix for the output field name. | +| `config_file_path` | string | — | Path to a TOML file supplying all parameters; mutually exclusive with inline arguments. Relative paths resolve against `PLUGIN_DIR`. | + +The final output field name is `{field_prefix}{output_field or source_field}{field_suffix}` +— by default, `value` becomes `value_filtered`. The raw input field is never copied +to the output. + +### TOML configuration + +To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and +reference the file with the `config_file_path` trigger argument (relative paths +resolve against `PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent of +`VIRTUAL_ENV`). The TOML file then supplies **all** parameters — it is mutually +exclusive with inline trigger arguments, so passing both is rejected. See +[`signal_filter_config_data_writes.toml`](signal_filter_config_data_writes.toml) +for an annotated template. + +## Data requirements + +- Input fields must be numeric (float or int). Null, boolean, and string values + contribute no samples; NaN/±Inf samples are dropped (they would permanently + poison IIR filter state). +- Duplicate timestamps within a commit keep the last occurrence, matching the + database's last-write-wins semantics. +- **Samples must arrive in time order across commits.** The plugin records the + last filtered timestamp per series and drops late/backfilled samples with a + warning — replaying old samples through a stateful causal filter would corrupt + the output. To re-filter history, delete the trigger's cache state (or recreate + the trigger) and replay the data in order. + +## Software Requirements + +- **{{% product-name %}}**: with the Processing Engine enabled + (`--plugin-dir` configured). +- **Python packages**: `numpy`, `scipy`, and `influxdata-plugin-utils`. + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install the Python dependencies into the plugin environment: + + ```bash + influxdb3 install package numpy scipy influxdata-plugin-utils + ``` + or via the HTTP API: + + ```bash + curl -X POST "http://localhost:8181/api/v3/configure/plugin_environment/install_packages" \ + -H "Authorization: Bearer $INFLUXDB3_AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"packages": ["numpy", "scipy", "influxdata-plugin-utils"]}' + ``` +3. Copy `signal_filter.py` into your plugin directory (or upload it with + `POST /api/v3/plugins/files`). + +## Trigger setup + +Create the trigger with `run_async: false` (the plugin's per-series state assumes +one invocation in flight at a time) and `error_behavior: log` or `retry` (state is +saved only after all writes are buffered, so retries re-filter and re-emit the same +points idempotently). Note the lowercase values — the live API rejects `Log`. + +### Data write trigger + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename signal_filter.py \ + --trigger-spec "table:signal" \ + --trigger-arguments 'input_fields=value,filter_type=lowpass,fc=5.0,order=4' \ + signal_lowpass +``` +or via the HTTP API: + +```bash +curl -X POST "http://localhost:8181/api/v3/configure/processing_engine_trigger" \ + -H "Authorization: Bearer $INFLUXDB3_AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "db": "mydb", + "plugin_filename": "signal_filter.py", + "trigger_name": "signal_lowpass", + "trigger_specification": "table:signal", + "trigger_settings": {"run_async": false, "error_behavior": "log"}, + "trigger_arguments": { + "input_fields": "value", + "filter_type": "lowpass", + "fc": "5.0", + "order": "4" + }, + "disabled": false + }' +``` +### Enable the trigger + +```bash +influxdb3 enable trigger --database mydb signal_lowpass +``` +### Testing without a trigger + +Use the WAL plugin test endpoint. Passing a fixed `cache_name` across calls +exercises cross-commit state continuity: + +```bash +curl -X POST "http://localhost:8181/api/v3/plugin_test/wal" \ + -H "Authorization: Bearer $INFLUXDB3_AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "filename": "signal_filter.py", + "database": "mydb", + "input_lp": "signal,host=a value=1.0 1717000000000000000\nsignal,host=a value=2.0 1717000000100000000", + "cache_name": "sigfilt_test", + "input_arguments": {"filter_type": "lowpass", "fc": "5.0", "sample_rate": "10.0"} + }' +``` +Note: test caches expire after 30 minutes by default; production trigger caches +persist indefinitely. + +## Example usage + +### Example 1: Lowpass smoothing (defaults) + +Attenuate high-frequency noise on the `value` field of the `signal` table with a +4th-order Butterworth lowpass at 5 Hz: + +```bash +# Create and enable the trigger +influxdb3 create trigger \ + --database mydb \ + --plugin-filename signal_filter.py \ + --trigger-spec "table:signal" \ + --trigger-arguments 'input_fields=value,filter_type=lowpass,fc=5.0,order=4,sample_rate=100.0' \ + signal_lowpass +influxdb3 enable trigger --database mydb signal_lowpass + +# Write some data, then query the filtered field +influxdb3 query \ + --database mydb \ + "SELECT time, value, value_filtered FROM signal ORDER BY time DESC LIMIT 10" +``` +**Expected output**: for every written `value`, a `value_filtered` field appears at +the same timestamp, with high-frequency content removed. The raw `value` is left +untouched. + +### Example 2: Bandpass with an explicit sample rate + +Isolate a 1–5 Hz band on a `vibration` field sampled at 50 Hz: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename signal_filter.py \ + --trigger-spec "table:sensors" \ + --trigger-arguments 'input_measurement=sensors,input_fields=vibration,filter_type=bandpass,fc1=1.0,fc2=5.0,order=4,sample_rate=50.0' \ + vibration_bandpass +influxdb3 enable trigger --database mydb vibration_bandpass +``` +**Expected output**: a `vibration_filtered` field carrying only the 1–5 Hz band, +per tag combination present in `sensors`. + +### Example 3: Manual second-order sections + +Apply coefficients you designed elsewhere, routed to a separate measurement: + +```bash +curl -X POST "http://localhost:8181/api/v3/configure/processing_engine_trigger" \ + -H "Authorization: Bearer $INFLUXDB3_AUTH_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "db": "mydb", + "plugin_filename": "signal_filter.py", + "trigger_name": "manual_sos", + "trigger_specification": "table:signal", + "trigger_settings": {"run_async": false, "error_behavior": "log"}, + "trigger_arguments": { + "design_type": "manual", + "sos": "[[0.2, 0.4, 0.2, 1.0, -0.4, 0.2]]", + "output_measurement": "signal_filtered" + }, + "disabled": false + }' +``` +**Expected output**: filtered points written to `signal_filtered` (not the source +`signal` table). Manual mode needs no sample rate — the coefficients are already +digital. + +## Sample-rate inference + +Preset design needs the sample rate. Resolution order per series: + +1. Explicit `sample_rate` argument — always wins, so config corrections take + effect immediately (a changed rate re-designs the filter and resets state). +2. Frozen per-series value from a previous successful inference. +3. Inference: `fs = 1e9 / median(inter-sample interval)` over accumulated + timestamps. An estimate is frozen only after ≥ 8 inter-sample intervals have + been observed; smaller commits accumulate timestamps across invocations and + are skipped (with an info log, samples not emitted) until the threshold is + met. At typical `signal_generator` rates a single commit clears it + immediately. + +Manual mode needs no sample rate — the coefficients are already digital. + +## Write-loop behavior + +Writing the output into the source measurement re-fires this trigger. This is +safe by default: re-fired rows carry only the output field, the input field is +null on them, and null values produce no samples. **However**, if your overrides +resolve the output field to the *same name* as the input field in the same +measurement and database (for example `field_suffix=""` with no `output_field`), +the output feeds the filter again and grows without bound. The plugin logs a +prominent warning in that configuration — change `field_suffix`, `output_field`, +`output_measurement`, or `output_target_database` to break the cycle. + +## Code overview + +### Files + +- `signal_filter.py`: The main plugin code — config parsing/validation, filter + design (preset + manual), the streaming runtime, per-series state helpers, and + the `process_writes` WAL entry point. +- `signal_filter_config_data_writes.toml`: Annotated example TOML configuration. +- `test_signal_filter.py`: Unit and integration tests (mock-based; no running + engine required). + +### Logging + +Every log line is prefixed with a per-invocation task id: `[] signal_filter: ...`. +Per invocation the plugin logs an info summary — tables and series processed, +samples in, points written, and counts of dropped non-finite, out-of-order, and +warm-up-skipped samples. Warnings cover out-of-order drops and the write-loop +hazard; errors cover missing dependencies, invalid configuration, and filter +design failures (for example a cutoff at or above the Nyquist frequency). Logs are +available in the `system.processing_engine_logs` table. + +### Main functions + +#### `process_writes(influxdb3_local, table_batches, args)` + +Entry point for data write triggers. Validates configuration, then for each table +batch: extracts per-(field, series) samples, resolves the sample rate, designs (or +reuses a memoized) filter, applies it with the cached delay-line state, writes the +filtered points, and finally saves the advanced per-series state. + +Key operations: + +1. Guards that `numpy`, `scipy`, and `influxdata-plugin-utils` are installed; logs an install command otherwise +2. Parses and validates trigger arguments (inline, or entirely from a TOML file) +3. Warns on any write-loop hazard configuration +4. Groups rows into per-(field, series) samples, dropping null/non-numeric/non-finite values +5. Resolves the sample rate (explicit → frozen → inferred with warm-up) +6. Designs the IIR filter (memoized by parameters + `fs`) and checks stability +7. Applies the filter using cached `zi` state, writing each point with `write_sync(..., no_sync=True)` +8. Saves the advanced state (delay line, `fs`, coeff hash, last timestamp) to the cache + +## Troubleshooting + +### Common issues + +#### Issue: "required packages are not installed" + +**Cause**: The plugin environment lacks `numpy`, `scipy`, or `influxdata-plugin-utils`. + +**Solution**: Run `influxdb3 install package numpy scipy influxdata-plugin-utils` and re-enable the trigger. + +#### Issue: No output points + +**Cause**: Warm-up (sample-rate inference not yet frozen), a single-sample sparse +series, or a non-numeric/null input field. + +**Solution**: + +- Check the info logs for warm-up skips: with no `sample_rate` argument the + plugin waits for ≥ 8 inter-sample intervals per series before filtering. +- A single-sample series with no `sample_rate` cannot infer a rate; set the + argument explicitly for very sparse data. +- Verify the input field is numeric and non-null in the written rows. + +#### Issue: "filter design failed: cutoff ... must be within (0, fs/2)" + +**Cause**: The (possibly inferred) sample rate puts your cutoff at or beyond Nyquist. + +**Solution**: Set `sample_rate` explicitly or lower the cutoff. + +#### Issue: Output has a transient after a server restart + +**Cause**: The cache is cleared on restart, so filters re-initialize and inferred +sample rates re-freeze (the cutoff may shift very slightly if the new estimate +differs). + +**Solution**: This is expected; set `sample_rate` explicitly to pin the design. + +#### Issue: Series explosion / unexpected series + +**Cause**: The automatic tag heuristic treats every string-valued column as a tag, +including string *fields*. + +**Solution**: Set `tag_keys` explicitly to bound the series set. + +### Viewing logs + +```bash +influxdb3 query \ + --database YOUR_DATABASE \ + "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'signal_lowpass' ORDER BY event_time DESC LIMIT 20" +``` + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/signal-generator.md b/content/shared/influxdb3-plugins/plugins-library/official/signal-generator.md new file mode 100644 index 0000000000..eb9d1ec9c4 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/signal-generator.md @@ -0,0 +1,559 @@ + + +The Signal Generator Plugin lets new users produce realistic time-series data for testing {{% product-name %}} functionality and downstream plugins without needing an external data source. Generate configurable waveform signals on a schedule to test alerts, anomaly detection, threshold checks, and dashboards from the moment you start. + +- **Zero dependencies**: Uses Python standard library only — no packages to install +- **Composable waveforms**: Mix sine, square, triangle, sawtooth, noise, and spike signals by stacking them +- **Realistic signals**: Default preset produces a signal centered at 30 with a slow sine trend, Gaussian noise, and occasional large spikes — immediately useful for alert testing +- **Timestamp jitter**: Offsets point timestamps by default to simulate sensors that do not sample at perfectly uniform intervals +- **Simulated gaps**: Simulates temporary data-source outages by default — bounded windows with no written points, useful for testing deadman alerts and gap handling. Set `gap_enabled=false` for continuous output +- **Catch-up generation**: Generates data for the full span between executions regardless of trigger schedule — the only missing spans are the simulated gaps +- **Flexible output**: Configure measurement name, field name, and custom tags per trigger + +## Important CLI limitation + +Trigger configurations that pass JSON values, including `waveforms` and `tags`, currently cannot be created with the `influxdb3 create trigger --trigger-arguments` CLI flag. +The CLI splits `--trigger-arguments` on every comma, including commas inside JSON arrays and objects, so values such as `waveforms=[{"type":"spike","value":7}]` are split into invalid fragments before they reach the plugin. +Use the InfluxDB 3 Explorer UI or the `/api/v3/configure/processing_engine_trigger` API to create configured Signal Generator triggers. + +The default no-argument trigger can still be created with the CLI because it does not pass JSON or comma-containing values. +Configured examples below use the API until the CLI parsing fix is available. + +## Configuration + +Plugin parameters should be specified in the `trigger_arguments` field when creating a trigger with the API or InfluxDB 3 Explorer. +Avoid the CLI for configured Signal Generator triggers until comma-aware parsing is available for JSON arrays and objects. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Optional parameters + +This plugin has no required parameters. + +| Parameter | Type | Default | Description | +|--------------------|--------------|------------------|----------------------------------------------------------------------------------------------| +| `waveforms` | JSON string | *(default preset)* | JSON array of waveform config objects. If omitted, uses the built-in default preset. | +| `measurement` | string | `signal` | Output measurement name. | +| `field` | string | `value` | Output field name. | +| `tags` | JSON string | *(none)* | Optional JSON object of tags to add to each data point. Example: `{"host": "server01"}`. | +| `points_per_second`| float | `1.0` | Data point resolution in points per second. Controls how many points are generated per second of elapsed time. | +| `jitter_amplitude_seconds` | float | 10% of point interval | Maximum timestamp offset in seconds. Set to `0` to disable timestamp jitter. | +| `jitter_seed` | integer | *(none)* | Optional seed for reproducible per-timestamp jitter, mainly useful for tests. | +| `gap_enabled` | boolean | `true` | Enables simulated data-source outage gaps. Set to `false` for continuous output. | +| `gap_min_duration_seconds` | float | `10.0` | Shortest simulated outage duration, in seconds. | +| `gap_max_duration_seconds` | float | `30.0` | Longest simulated outage duration, in seconds. | +| `gap_min_interval_seconds` | float | `120.0` | Shortest time between consecutive outage starts, in seconds. Must be at least `gap_max_duration_seconds`. | +| `gap_max_interval_seconds` | float | `240.0` | Longest time between consecutive outage starts, in seconds. | +| `gap_seed` | integer | *(none)* | Optional seed for reproducible gap timing. | +| `target_database` | string | *(none)* | Optional target database. If omitted, writes to the trigger's own database. | + +### Timestamp jitter + +Timestamp jitter is enabled by default. If `jitter_amplitude_seconds` is omitted, the plugin uses 10% of the point interval: + +```text +point_interval_seconds = 1 / points_per_second +default_jitter_amplitude_seconds = 0.10 * point_interval_seconds +``` +Each generated timestamp receives an independent random offset from `[-jitter_amplitude_seconds, +jitter_amplitude_seconds]`. When `jitter_seed` is set, the offset is derived from the seed and the nominal timestamp, so each timestamp still gets its own offset even if scheduled executions generate one point at a time. Waveforms are still evaluated at the nominal cadence timestamps, then jitter is applied only to the timestamp that is written. With the same waveform seeds, jittered and non-jittered runs produce the same field values. + +Set `jitter_amplitude_seconds=0` to preserve perfectly regular timestamps. The plugin rejects jitter settings that could produce duplicate or reordered timestamps. + +### Simulated gaps + +Simulated gaps are enabled by default. The plugin periodically simulates a data-source outage: points whose final (jittered) timestamps fall inside an outage window are not written, leaving realistic missing spans in the output. + +With the default configuration, each outage lasts 10–30 seconds, consecutive outages start 2–4 minutes apart, and about 11% of generated points are removed. The min/max bounds are hard guarantees: + +- Every outage duration lies in `[gap_min_duration_seconds, gap_max_duration_seconds]`. +- The time between consecutive outage starts lies in `[gap_min_interval_seconds, gap_max_interval_seconds]`. +- At least `gap_min_interval_seconds - gap_max_duration_seconds` seconds of continuous data separate consecutive outages (90 seconds at defaults). + +Gap windows are anchored to absolute time, so the same outage pattern is produced whether each scheduled execution writes one point or thousands, and catch-up after downtime contains the same gaps an uninterrupted run would have. When `gap_seed` is set, gap timing is fully reproducible. When omitted, the plugin generates a seed during first-run initialization and caches it, so the schedule stays coherent across executions; a restart that clears the trigger cache starts a new schedule. + +Missing spans are never backfilled: the generation boundary advances even when every point in a window was removed. Runs that removed points log a summary, for example: + +```text +Wrote 585 points to signal.value (600 generated, 15 removed by 1 gap windows) +``` +Set `gap_enabled=false` to disable simulated gaps entirely. + +**Upgrade note:** existing triggers pick up default gaps on upgrade with no configuration change. Downstream demos that use deadman or no-data alerts will begin firing on the simulated outages — useful for exercising those alerts, but set `gap_enabled=false` if you need continuous data. + +### Waveform types + +Waveform configuration is supplied as a JSON array. Each object requires a `type` key; all other parameters are optional and fall back to defaults. + +```json +[ + {"type": "sine", "frequency": 0.1, "amplitude": 5.0}, + {"type": "noise", "stddev": 0.3} +] +``` +Multiple waveforms are summed together to produce the final signal value. + +#### Deterministic waveforms + +| Type | Parameters | Defaults | Notes | +|------------|--------------------------------------------------|-------------------------------------------------------|----------------------------------------------------| +| `constant` | `value` | `0.0` | Fixed y-offset; shifts the entire combined signal. | +| `sine` | `frequency`, `amplitude`, `offset`, `phase` | `0.05 Hz`, `1.0`, `0.0`, `0.0` | 0.05 Hz = ~20 s period. Anchored to absolute time. | +| `square` | `frequency`, `amplitude`, `offset`, `phase`, `duty_cycle` | `0.05 Hz`, `1.0`, `0.0`, `0.0`, `0.5` | `duty_cycle` is 0–1, fraction of period spent high.| +| `triangle` | `frequency`, `amplitude`, `offset`, `phase` | `0.05 Hz`, `1.0`, `0.0`, `0.0` | Linear ramp up then down. | +| `sawtooth` | `frequency`, `amplitude`, `offset`, `phase` | `0.05 Hz`, `1.0`, `0.0`, `0.0` | Linear ramp up, instant drop. | + +#### Stochastic waveforms + +| Type | Parameters | Defaults | Notes | +|---------|----------------------------------------------------|---------------------------------------|-----------------------------------------------------------------------------------| +| `noise` | `stddev`, `mean`, `seed` | `0.1`, `0.0`, `None` | Gaussian noise. `seed` enables reproducible sequences. | +| `spike` | `probability`, `min_amplitude`, `max_amplitude`, `seed` | `0.01`, `5.0`, `10.0`, `None` | 1% chance of a spike per point. Magnitude drawn uniformly from `[min, max]` with random sign. | + +### Default preset + +When no `waveforms` argument is provided, the plugin uses this preset: + +```json +[ + {"type": "constant", "value": 30.0}, + {"type": "sine", "frequency": 0.005, "amplitude": 10.0}, + {"type": "noise", "stddev": 0.5}, + {"type": "spike", "probability": 0.005, "min_amplitude": 8.0, "max_amplitude": 15.0} +] +``` +This produces a signal centered around 30 with a slow-moving sine wave (period ~200 s / ~3.3 minutes), light Gaussian noise (stddev 0.5), and occasional large spikes (0.5% chance per point, magnitude 8–15). Designed to be immediately useful for testing alerts and anomaly detection without any configuration. + +## Software Requirements + +- **{{% product-name %}}**: with the Processing Engine enabled +- **Python packages**: No additional packages required (uses Python standard library only) + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. No additional Python packages are required for this plugin. + +## Trigger setup + +### Scheduled trigger with API configuration + +Use the API or InfluxDB 3 Explorer for configured triggers, especially when passing JSON values in `waveforms` or `tags`. + +```bash +curl -X POST "http://localhost:8181/api/v3/configure/processing_engine_trigger" \ + -H "Content-Type: application/json" \ + -d '{ + "db": "signals", + "plugin_filename": "gh:influxdata/signal_generator/signal_generator.py", + "trigger_name": "signal_generator_trigger", + "trigger_specification": "every:10s", + "trigger_arguments": { + "waveforms": "[{\"type\":\"constant\",\"value\":30.0},{\"type\":\"sine\",\"frequency\":0.005,\"amplitude\":10.0},{\"type\":\"noise\",\"stddev\":0.5}]", + "measurement": "signal", + "field": "value", + "jitter_amplitude_seconds": "0.2" + }, + "trigger_settings": { + "run_async": false, + "error_behavior": "log" + }, + "disabled": false + }' +``` +### Default CLI trigger + +The CLI can create a default no-argument trigger because no JSON value is passed. +Do not use the CLI for `waveforms` or `tags` until the CLI parsing issue is fixed. + +```bash +influxdb3 create trigger \ + --database signals \ + --path "gh:influxdata/signal_generator/signal_generator.py" \ + --trigger-spec "every:10s" \ + signal_generator_default +``` +**Note:** The first execution initializes the plugin (stores the current time in cache) and does not write any data. Data begins flowing on the second execution. + +## Example usage + +### Example 1: Basic (defaults) + +Use the default preset with no configuration. Creates a signal centered around 30 with sine trend, noise, and spikes: + +```bash +# Create the trigger +influxdb3 create trigger \ + --database signals \ + --path "gh:influxdata/signal_generator/signal_generator.py" \ + --trigger-spec "every:10s" \ + signal_basic + +# Enable the trigger +influxdb3 enable trigger --database signals signal_basic + +# Query signal data (after the second execution) +influxdb3 query \ + --database signals \ + "SELECT time, value FROM signal ORDER BY time DESC LIMIT 10" +``` +### Example 2: Custom waveforms (square + noise) + +Generate a square wave with added noise — useful for simulating on/off processes with sensor jitter: + +```bash +curl -X POST "http://localhost:8181/api/v3/configure/processing_engine_trigger" \ + -H "Content-Type: application/json" \ + -d '{ + "db": "signals", + "plugin_filename": "gh:influxdata/signal_generator/signal_generator.py", + "trigger_name": "signal_square_noise", + "trigger_specification": "every:10s", + "trigger_arguments": { + "waveforms": "[{\"type\":\"square\",\"frequency\":0.02,\"amplitude\":5.0,\"duty_cycle\":0.3},{\"type\":\"noise\",\"stddev\":0.2}]" + }, + "trigger_settings": { + "run_async": false, + "error_behavior": "log" + }, + "disabled": false + }' +``` +### Example 3: Custom output (measurement, field, tags) + +Write signal data to a specific measurement with custom field name and tags for multi-series dashboards: + +```bash +curl -X POST "http://localhost:8181/api/v3/configure/processing_engine_trigger" \ + -H "Content-Type: application/json" \ + -d '{ + "db": "signals", + "plugin_filename": "gh:influxdata/signal_generator/signal_generator.py", + "trigger_name": "signal_custom_output", + "trigger_specification": "every:10s", + "trigger_arguments": { + "measurement": "cpu_temperature", + "field": "temperature", + "tags": "{\"host\":\"server01\",\"region\":\"us-west\"}" + }, + "trigger_settings": { + "run_async": false, + "error_behavior": "log" + }, + "disabled": false + }' +``` +### Example 4: Multiple independent signals + +Run two triggers to generate multiple independent signals in the same database. Each trigger has its own cache and waveform configuration: + +```bash +# Signal A: slow sine wave (temperature-like) +curl -X POST "http://localhost:8181/api/v3/configure/processing_engine_trigger" \ + -H "Content-Type: application/json" \ + -d '{ + "db": "signals", + "plugin_filename": "gh:influxdata/signal_generator/signal_generator.py", + "trigger_name": "signal_temperature", + "trigger_specification": "every:10s", + "trigger_arguments": { + "waveforms": "[{\"type\":\"constant\",\"value\":22.0},{\"type\":\"sine\",\"frequency\":0.002,\"amplitude\":3.0},{\"type\":\"noise\",\"stddev\":0.1}]", + "measurement": "environment", + "field": "temperature", + "tags": "{\"sensor\":\"A\"}" + }, + "trigger_settings": { + "run_async": false, + "error_behavior": "log" + }, + "disabled": false + }' + +# Signal B: faster oscillation with spikes (pressure-like) +curl -X POST "http://localhost:8181/api/v3/configure/processing_engine_trigger" \ + -H "Content-Type: application/json" \ + -d '{ + "db": "signals", + "plugin_filename": "gh:influxdata/signal_generator/signal_generator.py", + "trigger_name": "signal_pressure", + "trigger_specification": "every:10s", + "trigger_arguments": { + "waveforms": "[{\"type\":\"constant\",\"value\":101.3},{\"type\":\"sine\",\"frequency\":0.01,\"amplitude\":0.8},{\"type\":\"spike\",\"probability\":0.01,\"min_amplitude\":2.0,\"max_amplitude\":5.0}]", + "measurement": "environment", + "field": "pressure", + "tags": "{\"sensor\":\"B\"}" + }, + "trigger_settings": { + "run_async": false, + "error_behavior": "log" + }, + "disabled": false + }' +``` +### Waveform JSON examples + +#### Single sine wave (all defaults) + +```json +[{"type": "sine"}] +``` +Produces a sine at 0.05 Hz (20 s period), amplitude 1.0, centered at 0. + +#### Combined sine and noise + +```json +[ + {"type": "constant", "value": 50.0}, + {"type": "sine", "frequency": 0.01, "amplitude": 15.0}, + {"type": "noise", "stddev": 1.0} +] +``` +Produces a signal centered at 50 with a 100 s period sine (±15) and moderate noise. + +#### Square wave with custom duty cycle + +```json +[ + {"type": "square", "frequency": 0.05, "amplitude": 10.0, "duty_cycle": 0.25} +] +``` +Produces a square wave spending 25% of each period at +10 and 75% at -10. + +#### Full custom signal + +```json +[ + {"type": "constant", "value": 100.0}, + {"type": "triangle", "frequency": 0.005, "amplitude": 20.0}, + {"type": "noise", "stddev": 2.0}, + {"type": "spike", "probability": 0.02, "min_amplitude": 30.0, "max_amplitude": 60.0} +] +``` +Produces a triangle wave centered at 100, with 2% spike probability and magnitude 30–60. + +## Output schema + +### Measurement: `signal` (default, configurable) + +**Tags:** + +Tags are optional and have no defaults. Tags are only present when specified via the `tags` trigger argument. + +Example with `tags={"host": "server01", "region": "us-west"}`: +- `host`: `server01` +- `region`: `us-west` + +**Fields:** + +- `value` (float64): The computed signal value at each timestamp. Field name is configurable via the `field` argument. + +**Timestamp:** + +- Nanosecond precision Unix epoch timestamps. Each point's timestamp reflects simulated sample time, not the wall-clock time of the write. +- Timestamp jitter is enabled by default, so adjacent timestamps are usually close to, but not exactly on, the nominal cadence grid. Set `jitter_amplitude_seconds=0` for regular intervals. +- Simulated gaps are enabled by default, so output contains periodic missing spans of 10–30 seconds. Set `gap_enabled=false` for continuous data. + +### Line protocol examples + +Without tags (default): + +``` +signal value=32.47 1712678399918245021 +signal value=31.89 1712678401084217139 +signal value=33.21 1712678401962364102 +``` +With custom measurement, field, and tags: + +``` +cpu_temperature,host=server01,region=us-west temperature=72.3 1712678400000000000 +cpu_temperature,host=server01,region=us-west temperature=71.8 1712678401000000000 +``` +## Example queries + +### View the most recent signal values + +```sql +SELECT time, value +FROM signal +WHERE time > now() - INTERVAL '5 minutes' +ORDER BY time DESC +LIMIT 20; +``` +### Compute rolling statistics + +```sql +SELECT + time_bucket(time, INTERVAL '1 minute') AS minute, + AVG(value) AS avg_value, + MIN(value) AS min_value, + MAX(value) AS max_value +FROM signal +WHERE time > now() - INTERVAL '1 hour' +GROUP BY minute +ORDER BY minute DESC; +``` +### Find spikes (values far from the mean) + +```sql +SELECT time, value +FROM signal +WHERE time > now() - INTERVAL '1 hour' + AND ABS(value - 30.0) > 10.0 +ORDER BY time DESC; +``` +### Compare multiple signals + +```sql +SELECT time, temperature, pressure +FROM environment +WHERE time > now() - INTERVAL '30 minutes' +ORDER BY time DESC +LIMIT 50; +``` +## Code overview + +### Files + +- `signal_generator.py`: The main plugin code containing all waveform factories, config parsing, time series generation, and the scheduled trigger entry point. + +### Main functions + +#### `process_scheduled_call(influxdb3_local, call_time, args)` + +Entry point for scheduled triggers. Orchestrates the full execution: parses config, reads the cache, generates timestamps and signal values, writes points, and updates the cache. + +Key operations: + +1. Parses waveform, output, resolution, timestamp jitter, and gap configuration from trigger arguments +2. Reads `last_time` from the trigger-local cache +3. On first run: stores current time (and the auto-generated gap seed when gaps are enabled and unseeded) and returns without writing (initialization) +4. Generates timestamps in the half-open interval `(last_time, now]` +5. Evaluates the combined waveform at each nominal timestamp +6. Applies timestamp jitter without changing field values +7. Removes points whose final timestamps fall inside simulated gap windows (see [Simulated gaps](#simulated-gaps)) +8. Writes the remaining points using `write_sync` with `no_sync=True` +9. Updates `last_time` in the cache + +#### Waveform factories + +`make_constant`, `make_sine`, `make_square`, `make_triangle`, `make_sawtooth`, `make_noise`, `make_spike` — each returns a function `f(t: float) -> float` where `t` is Unix epoch seconds. Deterministic waveforms are anchored to absolute time (same `t` always produces the same value). Stochastic waveforms draw from a `random.Random` instance per factory call. + +#### `combine(waveform_fns)` + +Composes a list of waveform functions by summing their outputs: `combined(t) = sum(fn(t) for fn in waveform_fns)`. + +#### `generate_timestamps(start, end, points_per_second)` + +Generates timestamps in the half-open interval `(start, end]`. The interval is exclusive of `start` to prevent duplicate points across consecutive executions. + +#### `apply_timestamp_jitter(points, amplitude_seconds, seed)` + +Offsets generated point timestamps after signal evaluation. Values are unchanged; only timestamps are moved. + +#### `gap_window(gap_config, n)` / `in_gap(gap_config, t)` / `apply_gaps(points, gap_config)` + +Implements the simulated gap schedule: absolute time is divided into fixed strata, and stratum `n` contains one gap whose start offset and duration are derived from a `blake2b` hash of `(seed, n)`. Every window is a pure function of seed, configuration, and gap index, so results are identical regardless of how many points each scheduled call generates. `apply_gaps` filters final (jittered) timestamps and reports how many points were removed. See `GAP_DESIGN.md` for the full design. + +## Troubleshooting + +### Common issues + +#### Issue: No data appearing after enabling the trigger + +**Cause**: The first execution initializes the plugin (stores the current time) and does not write any data. This is by design. + +**Solution**: Wait for the second execution. Check the logs to confirm initialization succeeded: + +```bash +influxdb3 query \ + --database signals \ + "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'signal_basic' ORDER BY event_time DESC LIMIT 5" +``` +Look for a log entry containing `"Signal generator initializing"` — this confirms the first run completed successfully and data will appear on the next execution. + +#### Issue: Config parsing error in logs + +**Cause**: Malformed JSON in `waveforms` or `tags` arguments, or an unknown waveform type. + +**Solution**: Validate your JSON before passing it as a trigger argument. Supported waveform types are: `constant`, `sine`, `square`, `triangle`, `sawtooth`, `noise`, `spike`. Check for typos and ensure the JSON array is valid: + +```bash +# Validate JSON locally +echo '[{"type": "sine"}, {"type": "noise"}]' | python3 -m json.tool +``` +#### Issue: Missing spans in query results + +**Cause**: Simulated gaps are enabled by default. The plugin periodically simulates a data-source outage and writes no points inside the outage window. + +**Solution**: This is expected behavior. To confirm a missing span is a simulated gap rather than a real failure, check the logs — runs that removed points report `... (N generated, M removed by K gap windows)`. Set `gap_enabled=false` in the trigger arguments for continuous output. + +#### Issue: No points generated (interval too short) + +**Cause**: The trigger fires more frequently than `1 / points_per_second` seconds, so no timestamps fall in the interval. + +**Solution**: Increase `points_per_second` to match your trigger frequency, or reduce trigger frequency. For example, if firing every 1 s with `points_per_second=1.0`, at least one point is generated per execution. If firing every 100ms, increase to `points_per_second=10.0`. + +#### Issue: Invalid jitter config in logs + +**Cause**: `jitter_amplitude_seconds` is negative, non-numeric, or too large for the configured `points_per_second`, or `jitter_seed` is not an integer. + +**Solution**: Reduce `jitter_amplitude_seconds`, reduce `points_per_second`, set `jitter_amplitude_seconds=0`, or use an integer `jitter_seed`. The plugin requires at least a 1 microsecond minimum gap between any two possible jittered timestamps. + +#### Issue: Write failures for individual points + +**Cause**: Intermittent write errors. The plugin logs each failed point and continues. + +**Solution**: Check logs for write error details: + +```bash +influxdb3 query \ + --database signals \ + "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'signal_basic' AND log_text LIKE '%Write failed%' ORDER BY event_time DESC LIMIT 10" +``` +#### Issue: Signal is not phase-continuous after restart + +**Cause**: Deterministic waveforms (sine, square, triangle, sawtooth) are anchored to absolute Unix time, so they are always at the correct phase for a given wall-clock time. If the signal appears discontinuous, check that the `frequency` parameter is the same before and after the restart. + +**Cause of gaps in data**: Short gaps (10–30 seconds by default) are simulated outages; see [Simulated gaps](#simulated-gaps). For downtime-related gaps: if the trigger was disabled or InfluxDB was stopped, the cache retains `last_time` from the last successful execution, and on restart the plugin generates all missing points from `last_time` to the current time — the caught-up span contains the same simulated gap windows an uninterrupted run would have. + +### Viewing logs + +```bash +influxdb3 query \ + --database YOUR_DATABASE \ + "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'signal_basic' ORDER BY event_time DESC LIMIT 20" +``` + +## Logging + +Logs are stored in the `_internal` database (or the database where the trigger is created) in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" +``` + +Log columns: +- **event_time**: Timestamp of the log event +- **trigger_name**: Name of the trigger that generated the log +- **log_level**: Severity level (INFO, WARN, ERROR) +- **log_text**: Message describing the action or error + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/simple-data-replicator.md b/content/shared/influxdb3-plugins/plugins-library/official/simple-data-replicator.md new file mode 100644 index 0000000000..9337cd7b11 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/simple-data-replicator.md @@ -0,0 +1,257 @@ + + +The Simple Data Replicator Plugin replicates data from a local {{% product-name %}} instance to a remote {{% product-name %}} instance over HTTP. It supports table and field filtering, table and field renaming, and runs either on a schedule (pulling a time window from a source measurement) or on every write (replicating committed WAL batches). Data is buffered in a compressed JSONL queue file and delivered with automatic retries, so it survives transient remote failures. + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. This plugin supports TOML configuration files, which can be specified using the `config_file_path` parameter; values in the file override the arguments passed to the trigger. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Scheduler mode parameters + +Multi-value arguments (`excluded_fields`, `field_renames`) use delimited strings on the CLI and native TOML types (lists/tables) in a config file. + +| Argument | Description | Required | Example (CLI) | Example (TOML) | Default | +|--------------------------|------------------------------------------------------------------------------------------|----------|--------------------|------------------------------------------|----------------------| +| `host` | Remote InfluxDB host URL. Default scheme `http`, port `8181`. | `true` | `example.com` | `host = "example.com"` | `None` | +| `remote_token` | Remote InfluxDB API token. Falls back to `REMOTE_INFLUXDB_TOKEN` env var. | `false` | `apiv3_AuHk...` | `remote_token = "apiv3_AuHk..."` | `None` | +| `database` | Remote database name. | `true` | `remote_db` | `database = "remote_db"` | `None` | +| `source_measurement` | Measurement to replicate. | `true` | `home` | `source_measurement = "home"` | `None` | +| `window` | Time window per job. Format `` (`s`, `min`, `h`, `d`, `w`). | `true` | `1h` | `window = "1h"` | `None` | +| `unique_file_suffix` | Unique suffix for the queue file to avoid conflicts. | `true` | `abcd1234` | `unique_file_suffix = "abcd1234"` | `None` | +| `max_size` | Maximum size for the queue file in MB. Integer ≥ 1. | `false` | `1024` | `max_size = 1024` | `512` | +| `verify_ssl` | Whether to verify SSL certificates when connecting via HTTPS. | `false` | `false` | `verify_ssl = false` | `true` | +| `max_retries` | Maximum number of retries for write operations. Integer ≥ 1. | `false` | `5` | `max_retries = 5` | `3` | +| `queue_flush_chunk_size` | Line-protocol entries sent per remote flush request. Integer ≥ 1. | `false` | `1000` | `queue_flush_chunk_size = 1000` | `500` | +| `excluded_fields` | Fields and tags to exclude. CLI: space-separated. TOML: list. | `false` | `co location` | `excluded_fields = ["co", "location"]` | `None` | +| `target_table` | New name for the measurement in the remote database. | `false` | `home2` | `target_table = "home2"` | `source_measurement` | +| `field_renames` | Field renames. CLI: `old:new` pairs separated by space. TOML: table. | `false` | `temp:temperature` | `field_renames = {temp = "temperature"}` | `None` | +| `offset` | Time offset to apply to the window. Format `` (`s`, `min`, `h`, `d`, `w`). | `false` | `10min` | `offset = "10min"` | `0` | +| `config_file_path` | Path to the TOML config file. Absolute, or relative to `PLUGIN_DIR`. | `false` | `config.toml` | — | `None` | + +### Data write mode parameters + +Multi-value arguments (`tables`, `excluded_fields`, `tables_rename`, `field_renames`) use delimited strings on the CLI and native TOML types (lists/tables) in a config file. + +| Argument | Description | Required | Example (CLI) | Example (TOML) | Default | +|--------------------------|--------------------------------------------------------------------------------------------------------|----------|---------------------------------------|------------------------------------------------------------------------|---------| +| `host` | Remote InfluxDB host URL. Default scheme `http`, port `8181`. | `true` | `example.com` | `host = "example.com"` | `None` | +| `remote_token` | Remote InfluxDB API token. Falls back to `REMOTE_INFLUXDB_TOKEN` env var. | `false` | `apiv3_AuHk...` | `remote_token = "apiv3_AuHk..."` | `None` | +| `database` | Remote database name. | `true` | `remote_db` | `database = "remote_db"` | `None` | +| `unique_file_suffix` | Unique suffix for the queue file to avoid conflicts. | `true` | `wxyz5678` | `unique_file_suffix = "wxyz5678"` | `None` | +| `tables` | Tables to replicate; all tables if omitted. CLI: space-separated. TOML: list. | `false` | `home home2` | `tables = ["home", "home2"]` | `None` | +| `verify_ssl` | Whether to verify SSL certificates when connecting via HTTPS. | `false` | `false` | `verify_ssl = false` | `true` | +| `max_size` | Maximum size for the queue file in MB. Integer ≥ 1. | `false` | `1024` | `max_size = 1024` | `512` | +| `queue_flush_chunk_size` | Line-protocol entries sent per remote flush request. Integer ≥ 1. | `false` | `1000` | `queue_flush_chunk_size = 1000` | `500` | +| `excluded_fields` | Fields/tags to exclude per table. CLI: `table:f1\|f2` blocks separated by space. TOML: table of lists. | `false` | `home:co\|location home2:temp` | `excluded_fields = {home = ["co", "location"], home2 = ["temp"]}` | `None` | +| `tables_rename` | Table renames. CLI: `old:new` pairs separated by space. TOML: table. | `false` | `home:home2 office:office2` | `tables_rename = {home = "home2", office = "office2"}` | `None` | +| `field_renames` | Field renames per table. CLI: `table:old=new\|old=new` blocks separated by space. TOML: nested table. | `false` | `home:temp=temperature\|hum=humidity` | `[field_renames]`
`home = {temp = "temperature", hum = "humidity"}` | `None` | +| `config_file_path` | Path to the TOML config file. Absolute, or relative to `PLUGIN_DIR`. | `false` | `config.toml` | — | `None` | + +### Authentication + +The remote API token is read from the `remote_token` argument first; if it is not set, the plugin falls back to the `REMOTE_INFLUXDB_TOKEN` environment variable. If neither is provided, the plugin returns an error. + +### File path resolution + +The `config_file_path` and the queue directory follow the same resolution logic: + +- **Absolute paths** are used as-is. +- **Relative paths** are resolved from the `PLUGIN_DIR` environment variable. If `PLUGIN_DIR` is not set, the plugin falls back to `INFLUXDB3_PLUGIN_DIR` and then the parent of `VIRTUAL_ENV`. + +If none of these are available, the plugin returns an error. + +#### Example TOML configuration + +- [simple_data_replicator_config_scheduler.toml](https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/simple_data_replicator/simple_data_replicator_config_scheduler.toml) — scheduler mode +- [simple_data_replicator_config_data_writes.toml](https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/simple_data_replicator/simple_data_replicator_config_data_writes.toml) — data write mode + +## Data requirements + +The plugin assumes the table schema is already defined in the local database; it relies on this schema to retrieve the field and tag names required for processing. The remote target database must already exist. + +## Software Requirements + +- **{{% product-name %}}**: with the Processing Engine enabled +- **Python packages**: + - `influxdb3-python` ({{% product-name %}} client library used to write to the remote instance) + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/plugins + ``` +2. Install required Python packages: + + ```bash + influxdb3 install package influxdb3-python + ``` +## Trigger setup + +### Scheduler mode + +Periodically pulls data from a local measurement within a time window and replicates it to the remote instance: + +```bash +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/simple_data_replicator/simple_data_replicator.py \ + --trigger-spec "every:10s" \ + --trigger-arguments host=example.com,remote_token=apiv3_token,database=remote_db,source_measurement=home,window=10s,unique_file_suffix=abcd1234 \ + simple_data_replicator_trigger +``` +Enable the trigger to start periodic replication: + +```bash +influxdb3 enable trigger --database mydb simple_data_replicator_trigger +``` +### Data write mode + +Replicates data automatically whenever the Write-Ahead Log (WAL) is flushed after a write: + +```bash +influxdb3 create trigger \ + --database mydb \ + --trigger-spec "all_tables" \ + --plugin-filename gh:influxdata/simple_data_replicator/simple_data_replicator.py \ + --trigger-arguments host=example.com,remote_token=apiv3_token,database=remote_db,tables="home home2",field_renames="home:hum=humidity|temp=temperature",unique_file_suffix=wxyz5678 \ + simple_data_replicator_trigger +``` +Enable the trigger to start replication on WAL flush events: + +```bash +influxdb3 enable trigger --database mydb simple_data_replicator_trigger +``` +> **Note:** +> - The plugin is triggered whenever the WAL is flushed, which happens after data is written. +> - It processes the batches of data written to the database, using the arguments specified in `--trigger-arguments`. +> - The plugin caches the list of measurements in the database and the tag names for each measurement for one hour to avoid repeated queries. +> - In data write mode the remote flush is attempted once per invocation (no in-call retries); any data that fails to deliver stays in the queue and is retried on the next WAL flush. +> - Run the trigger in the default synchronous mode. Do not use `--run-asynchronous`: concurrent invocations would access the same queue file in parallel and can corrupt it or lose data. Synchronous execution serializes invocations and keeps queue access safe. + +### Replication with TOML configuration + +Recommended for complex filtering and renaming. Store the arguments in a TOML file and reference it with the `config_file_path` argument (absolute, or relative to `PLUGIN_DIR`): + +```bash +export PLUGIN_DIR=~/plugins +cp simple_data_replicator_config_scheduler.toml $PLUGIN_DIR/ + +influxdb3 create trigger \ + --database mydb \ + --plugin-filename gh:influxdata/simple_data_replicator/simple_data_replicator.py \ + --trigger-spec "every:10s" \ + --trigger-arguments config_file_path=simple_data_replicator_config_scheduler.toml \ + simple_data_replicator_trigger +``` +## Queue management + +The plugin buffers data in a compressed JSONL queue file before delivery, ensuring reliable replication. The file is unique per trigger configuration to avoid conflicts. + +- **Location**: the `sdr_queues` subdirectory of the resolved plugin directory (see [File path resolution](#file-path-resolution)). For example, with `PLUGIN_DIR=/opt/plugins` the queue lives in `/opt/plugins/sdr_queues/`. +- **File name**: + - Data write mode: `sdr_queue_writes_.jsonl.gz` + - Scheduler mode: `sdr_queue_schedule_.jsonl.gz` +- **Format**: gzip-compressed JSON Lines, one JSON object per line. +- **Lifecycle**: data is appended before replication and removed after it is delivered successfully. +- **Maximum size**: controlled by `max_size` (default 512 MB); exceeding it raises an error. + +Use a distinct `unique_file_suffix` for every trigger, and monitor the file size to avoid disk overflow. + +## Logging + +Logs are stored in the `_internal` database (or the database where the trigger is created) in the `system.processing_engine_logs` table: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs" +``` +Example output: + +```text ++-------------------------------+-----------------+--------------+----------------------------------------------------------------------------------------------------------+ +| event_time | trigger_name | log_level | log_text | ++-------------------------------+-----------------+--------------+----------------------------------------------------------------------------------------------------------+ +| 2025-05-14T16:31:10.033295886 | my_scheduler | INFO | [4726cf36-3b15-442e-bd9d-f9b768ad8781] Finished execution in 31ms 449us 193ns | +| 2025-05-14T16:31:10.033275724 | my_scheduler | INFO | [4726cf36-3b15-442e-bd9d-f9b768ad8781] Replicated 10 lines to remote instance | +| 2025-05-14T16:31:10.033232792 | my_scheduler | INFO | [4726cf36-3b15-442e-bd9d-f9b768ad8781] Queued 10 lines from table1,table2 | +| 2025-05-14T16:31:00.046579787 | some_scheduler | ERROR | [528a316e-b28c-4bd2-8c05-07ad50bc1de2] Error during replication: Connection failed | +| 2025-05-14T16:31:00.025482558 | my_scheduler | INFO | [cd5caa89-47db-443c-9621-5f90a129a0cc] Starting data replication process | ++-------------------------------+-----------------+--------------+----------------------------------------------------------------------------------------------------------+ +``` +Each entry carries the task ID in `log_text`, so logs from a single execution can be correlated. + +## Troubleshooting + +### Check plugin logs + +```bash +influxdb3 query --database _internal \ + "SELECT * FROM system.processing_engine_logs + WHERE trigger_name = 'simple_data_replicator_trigger' + ORDER BY time DESC LIMIT 20" +``` +### Common issues + +#### "Missing remote token" + +Set the `remote_token` argument or the `REMOTE_INFLUXDB_TOKEN` environment variable. + +#### Relative `config_file_path` not found + +For relative paths, ensure `PLUGIN_DIR` (or `INFLUXDB3_PLUGIN_DIR`) is set. For absolute paths, verify the file exists. + +#### Queue file exceeds `max_size` + +Replication is failing or falling behind. Check the logs for delivery errors, fix the remote connection, or raise `max_size`. + +#### Corrupted or invalid queue file + +If the plugin reports persistent errors caused by a corrupted queue file, delete it to reset the queue: + +```bash +# Data write triggers: +rm $PLUGIN_DIR/sdr_queues/sdr_queue_writes_.jsonl.gz + +# Scheduled triggers: +rm $PLUGIN_DIR/sdr_queues/sdr_queue_schedule_.jsonl.gz +``` +**Warning:** deleting the queue file permanently loses any entries that have not yet been replicated. Only delete it if you are certain the data is corrupted or you accept the loss. + +## Architecture + +### How It Works + +1. **Trigger fires**: on a schedule (`every:`) or on each WAL flush (`all_tables`). +2. **Read rows**: scheduler mode queries the time window from `source_measurement`; data write mode reads the committed table batches. +3. **Filter and rename**: excluded tables and fields are dropped, then table and field renames are applied. +4. **Queue**: rows are converted to points and appended to the compressed JSONL queue file. If the queue already reached `max_size`, new rows are skipped (with a warning) but the existing queue is still flushed, so the buffer can drain once the remote recovers. +5. **Replicate**: the queue is flushed to the remote instance over HTTP in chunks of `queue_flush_chunk_size` lines (default 500). Scheduler mode retries failures and rate limits; data write mode attempts each flush once. +6. **Cleanup**: entries are removed from the queue only after they are delivered successfully. + +### Performance Optimization + +- **Client caching**: the remote client is cached and reused across invocations instead of reconnecting each cycle. It is rebuilt when the connection settings change, and evicted after retries are exhausted on a connection failure so the next cycle reconnects cleanly. +- **Chunked flush**: the queue is written to the remote instance in chunks of `queue_flush_chunk_size` lines (default 500), bounding request size and memory for large backlogs. Each successful chunk is removed from the queue, so a mid-flush failure does not re-send already delivered data. +- **Schema caching**: the measurement list and per-measurement tag names are cached for one hour to avoid repeated queries. +- **Compressed queue**: gzip-compressed JSONL keeps the on-disk buffer small. +- **Retry logic**: in scheduler mode, transient failures and rate limits are retried up to `max_retries` times; in data write mode failed data stays queued for the next WAL flush. + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/state-change.md b/content/shared/influxdb3-plugins/plugins-library/official/state-change.md index 86ad30e70d..a6fac84d31 100644 --- a/content/shared/influxdb3-plugins/plugins-library/official/state-change.md +++ b/content/shared/influxdb3-plugins/plugins-library/official/state-change.md @@ -1,4 +1,5 @@ - + + The State Change Plugin provides comprehensive field monitoring and threshold detection for {{% product-name %}} data streams. Detect field value changes, monitor threshold conditions, and trigger notifications when specified criteria are met. Supports both scheduled batch monitoring and real-time data write monitoring with configurable stability checks and multi-channel alerts. ## Configuration @@ -13,20 +14,20 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor ### Required parameters -| Parameter | Type | Default | Description | -|----------------------|--------|----------|----------------------------------------------------------------------------------------------| -| `measurement` | string | required | Measurement to monitor for field changes | -| `field_change_count` | string | required | Dot-separated field thresholds (for example, "temp:3.load:2"). Supports count-based conditions | -| `senders` | string | required | Dot-separated notification channels with multi-channel alert support (Slack, Discord, etc.) | -| `window` | string | required | Time window for analysis. Format: `` (for example, "10m", "1h") | +| Parameter | Type | Default | Description | +|----------------------|--------|----------|------------------------------------------------------------------------------------------------------------------------| +| `measurement` | string | required | Measurement to monitor for field changes | +| `field_change_count` | string | required | Dot-separated field thresholds (for example, "temp:3.load:2" or "temp:3.disk.used:2"). Each count must be 1 or greater | +| `senders` | string | required | Dot-separated notification channels with multi-channel alert support (Slack, Discord, etc.) | +| `window` | string | required | Time window for analysis. Format: ``, units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`. Must be positive | ### Data write trigger parameters -| Parameter | Type | Default | Description | -|--------------------|--------|----------|----------------------------------------------------------------------------------------------------------------| -| `measurement` | string | required | Measurement to monitor for threshold conditions | -| `field_thresholds` | string | required | Flexible threshold conditions with count-based and duration-based support (for example, "temp:30:10@status:ok:1h") | -| `senders` | string | required | Dot-separated notification channels with multi-channel alert support (Slack, Discord, HTTP, SMS, WhatsApp) | +| Parameter | Type | Default | Description | +|--------------------|--------|----------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `measurement` | string | required | Measurement to monitor for threshold conditions | +| `field_thresholds` | string | required | Threshold conditions with count-based and duration-based support (for example, "temp:30:10@status:ok:1h"). Counts must be 1 or greater; durations must be positive | +| `senders` | string | required | Dot-separated notification channels with multi-channel alert support (Slack, Discord, HTTP, SMS, WhatsApp) | ### Notification parameters @@ -43,8 +44,10 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor | Parameter | Type | Default | Description | |-----------------------|--------|---------|-------------------------------------------------------------------------------------------| -| `state_change_window` | number | 1 | Recent values to check for stability (configurable state change detection to reduce noise) | -| `state_change_count` | number | 1 | Max changes allowed within stability window (configurable state change detection) | +| `state_change_window` | number | 1 | Recent values to check for stability (reduces noise from flapping fields) | +| `state_change_count` | number | 1 | Changes within the stability window at which notifications start being suppressed | + +The stability check applies only when `state_change_window` is 2 or greater; the default of 1 leaves it off. Notifications are suppressed once the window contains `state_change_count` changes, so `state_change_count=3` is the setting that tolerates two flips. ### TOML configuration @@ -52,7 +55,11 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor |--------------------|--------|---------|----------------------------------------------------------------------------------| | `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (required for TOML configuration) | -*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. Relative paths are resolved against the first directory that is set: `PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent of `VIRTUAL_ENV`. Only that directory is used — the file is not looked up in the remaining ones. + +When `config_file_path` is set, the TOML file provides the whole configuration and inline trigger arguments are ignored. `INFLUXDB3_AUTH_TOKEN` from the environment still applies when `influxdb3_auth_token` is not set in the file. In TOML, `senders`, `field_thresholds`, and `field_change_count` use native structures (list, list of entries, table) instead of the inline string formats, though the inline strings are also accepted. + +Data write triggers cache the loaded configuration for 10 minutes to keep the write path fast, so configuration changes take effect within that window. Example TOML configuration files provided: @@ -74,7 +81,8 @@ The plugin assumes that the table schema is already defined in the database, as - **{{% product-name %}}**: with the Processing Engine enabled. - **Notification Sender Plugin for {{% product-name %}}**: Required for sending notifications. See the [influxdata/notifier plugin](/influxdb3/version/plugins/library/official/notifier/). - **Python packages**: - - `requests` (for HTTP notifications) + - `influxdata-plugin-utils>=0.3.0` (configuration loading, parsing, and schema introspection) + - `requests` (for HTTP notifications) ### Installation steps @@ -90,6 +98,7 @@ The plugin assumes that the table schema is already defined in the database, as 2. Install required Python packages: ```bash + influxdb3 install package "influxdata-plugin-utils>=0.3.0" influxdb3 install package requests ``` 3. *Optional*: For notifications, install and configure the [influxdata/notifier plugin](/influxdb3/version/plugins/library/official/notifier/) @@ -105,7 +114,7 @@ influxdb3 create trigger \ --database mydb \ --path "gh:influxdata/state_change/state_change_check_plugin.py" \ --trigger-spec "every:10m" \ - --trigger-arguments "measurement=cpu,field_change_count=temp:3.load:2,window=10m,senders=slack,slack_webhook_url=$SLACK_WEBHOOK_URL" \ + --trigger-arguments "measurement=cpu,field_change_count=temp:3.load:2,window=10min,senders=slack,slack_webhook_url=$SLACK_WEBHOOK_URL" \ state_change_scheduler ``` Set `SLACK_WEBHOOK_URL` to your Slack incoming webhook URL. @@ -174,7 +183,7 @@ Set `SLACK_WEBHOOK_URL` to your Slack incoming webhook URL. **Expected output** -When the field changes more than 5 times within 1 hour, a notification is sent: "Temperature sensor value changed 6 times in 1h for tags location=office" +When the field changes 5 or more times within 1 hour, a notification is sent: "Field value in table temperature changed 6 times in window 1:00:00 for tags location=office" ### Example 2: Advanced scheduled field change monitoring @@ -225,6 +234,9 @@ Set `SLACK_WEBHOOK_URL` to your Slack incoming webhook URL. - `state_change_check_plugin.py`: The main plugin code containing handlers for scheduled and data write triggers - `state_change_config_scheduler.toml`: Example TOML configuration for scheduled triggers - `state_change_config_data_writes.toml`: Example TOML configuration for data write triggers +- `test_state_change.py`: Pytest suite, runs without a live {{% product-name %}} server +- `requirements.txt`: Runtime dependencies (`influxdata-plugin-utils>=0.3.0`, `requests`) +- `requirements-dev.txt`: Development dependencies (`pytest`) ### Logging @@ -264,17 +276,21 @@ Handles real-time threshold monitoring on data writes. Evaluates incoming data a **Count-based thresholds** - Format: `field_name:"value":count` -- Example: `temp:"30.5":10` (10 occurrences of temperature = 30.5) +- Example: `temp:"30.5":10` (10 consecutive occurrences of temperature = 30.5) +- The count must be an integer of 1 or greater **Time-based thresholds** - Format: `field_name:"value":duration` - Example: `status:"error":5min` (status = error for 5 minutes) -- Supported units: `s`, `min`, `h`, `d`, `w` +- Supported units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`; the duration must be positive **Multiple conditions** - Separate with `@`: `temp:"30":5@humidity:"high":10min` +- Segments that fail to parse are skipped with a warning; if none remain, the run stops with an error + +In TOML, the same thresholds are written as entries: `field_thresholds = [["temp", 30.5, 10], ["status", "error", "5min"]]`. ### Message template variables @@ -301,4 +317,6 @@ For plugin issues, see the Plugins repository [issues page](https://github.com/i ## Find support for {{% product-name %}} The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. -For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. \ No newline at end of file +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/stateless-adtk-detector.md b/content/shared/influxdb3-plugins/plugins-library/official/stateless-adtk-detector.md index f93d74790d..c8fa96f598 100644 --- a/content/shared/influxdb3-plugins/plugins-library/official/stateless-adtk-detector.md +++ b/content/shared/influxdb3-plugins/plugins-library/official/stateless-adtk-detector.md @@ -1,4 +1,5 @@ - + + The ADTK Anomaly Detector Plugin provides advanced time series anomaly detection for {{% product-name %}} using the ADTK (Anomaly Detection Toolkit) library. Apply statistical and machine learning-based detection methods to identify outliers, level shifts, volatility changes, and seasonal anomalies in your data. Features consensus-based detection requiring multiple detectors to agree before triggering alerts, reducing false positives. ## Configuration @@ -19,15 +20,33 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor | `field` | string | required | Numeric field to evaluate | | `detectors` | string | required | Dot-separated list of advanced ADTK detectors for different anomaly types | | `detector_params` | string | required | Base64-encoded JSON parameters for each detector | -| `window` | string | required | Data analysis window with flexible scheduling. Format: `` (for example, "1h", "30m") | +| `window` | string | required | Data analysis window. Format: `` (for example, "1h", "30min"). Must be positive | | `senders` | string | required | Dot-separated notification channels with multi-channel notification support | +Duration units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`. + ### Advanced parameters -| Parameter | Type | Default | Description | -|--------------------------|--------|---------|----------------------------------------------------------------------------------------------| -| `min_consensus` | number | 1 | Minimum detectors required to agree for consensus-based filtering to reduce false positives | -| `min_condition_duration` | string | "0s" | Minimum duration for configurable anomaly persistence before alerting | +| Parameter | Type | Default | Description | +|-----------------------------|---------|----------|------------------------------------------------------------------------------------------------------------| +| `min_consensus` | number | 1 | Minimum detectors required to agree for consensus-based filtering to reduce false positives (1 or greater) | +| `min_condition_duration` | string | "0s" | Minimum duration for configurable anomaly persistence before alerting | +| `group_by_tags` | bool | false | Analyze every tag combination as its own time series | +| `max_notifications_per_run` | number | 20 | Maximum number of notifications a single run may send | + +#### Analyzing tagged measurements + +By default the whole window forms a single time series. When a measurement holds several tag combinations (for example `host=server1` and `host=server2`), those rows share timestamps, and ADTK keeps only the first value of each timestamp — so one arbitrary series is analyzed and the rest of the data is ignored. + +Set `group_by_tags=true` to analyze each tag combination separately. Every series then gets its own detector run, its own consensus evaluation, and its own debounce state, so a measurement with N tag combinations can produce up to N notifications per run. + +#### Notification behavior + +The timestamp of the last alerted point is remembered per series, so a `window` longer than the trigger interval does not resend anomalies that earlier runs already reported. + +A single run sends at most `max_notifications_per_run` notifications. Anomalies beyond the limit are counted in a warning and are not resent by later runs — raise the limit if a run legitimately produces more alerts. + +Points that have no value for `field` are dropped before detection and reported in the log; without this a single NULL makes every detector fail. ### Notification parameters @@ -46,6 +65,8 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor *To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. +When a config file is given, it replaces the inline trigger arguments entirely. In TOML, `detectors` and `senders` accept either a list (`["QuantileAD", "PersistAD"]`) or a dot-separated string, and detector parameters may be given as a `[detector_params]` table or as a base64-encoded JSON string. + #### Example TOML configuration [adtk_anomaly_config_scheduler.toml](https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/stateless_adtk_detector/adtk_anomaly_config_scheduler.toml) @@ -68,10 +89,17 @@ For more information on using TOML configuration files, see the Using TOML Confi ## Software Requirements - **{{% product-name %}}**: with the Processing Engine enabled. +- **Python 3.11+** - **Python packages**: + - `influxdata-plugin-utils>=0.3.0` (for configuration loading, parsing, and schema introspection) - `adtk` (for anomaly detection) - - `pandas` (for data manipulation) + - `pandas<3` (for data manipulation) - `requests` (for HTTP notifications) + +`pandas` must stay below 3.0. Window-based detectors (`LevelShiftAD`, `VolatilityShiftAD`, `PersistAD`) +return `NaN` for the first `window` points, which pandas 3 refuses to store in a boolean result. With +pandas 3 installed those three detectors fail with `Invalid value 'nan' for dtype 'bool'`, are skipped +with a warning, and stop contributing to the consensus. - **Notification Sender Plugin** *(optional)*: Required if using the `senders` parameter. See the [influxdata/notifier plugin](/influxdb3/version/plugins/library/official/notifier/). ### Installation steps @@ -88,9 +116,10 @@ For more information on using TOML configuration files, see the Using TOML Confi 2. Install required Python packages: ```bash + influxdb3 install package "influxdata-plugin-utils>=0.3.0" influxdb3 install package requests influxdb3 install package adtk - influxdb3 install package pandas + influxdb3 install package "pandas<3" ``` 3. *(Optional)* For notifications, install the [influxdata/notifier plugin](/influxdb3/version/plugins/library/official/notifier/) and create an HTTP trigger for it. @@ -105,7 +134,7 @@ influxdb3 create trigger \ --database mydb \ --path "gh:influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py" \ --trigger-spec "every:10m" \ - --trigger-arguments "measurement=cpu,field=usage,detectors=QuantileAD.LevelShiftAD,detector_params=eyJRdWFudGlsZUFKIjogeyJsb3ciOiAwLjA1LCAiaGlnaCI6IDAuOTV9LCAiTGV2ZWxTaGlmdEFKIjogeyJ3aW5kb3ciOiA1fX0=,window=10m,senders=slack,slack_webhook_url=$SLACK_WEBHOOK_URL" \ + --trigger-arguments "measurement=cpu,field=usage,detectors=QuantileAD.LevelShiftAD,detector_params=eyJRdWFudGlsZUFKIjogeyJsb3ciOiAwLjA1LCAiaGlnaCI6IDAuOTV9LCAiTGV2ZWxTaGlmdEFKIjogeyJ3aW5kb3ciOiA1fX0=,window=10min,senders=slack,slack_webhook_url=$SLACK_WEBHOOK_URL" \ anomaly_detector ``` Set `SLACK_WEBHOOK_URL` to your Slack incoming webhook URL. @@ -146,7 +175,7 @@ influxdb3 create trigger \ --database monitoring \ --path "gh:influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py" \ --trigger-spec "every:15m" \ - --trigger-arguments "measurement=cpu_metrics,field=utilization,detectors=QuantileAD.LevelShiftAD,detector_params=eyJRdWFudGlsZUFEIjogeyJsb3ciOiAwLjEsICJoaWdoIjogMC45fSwgIkxldmVsU2hpZnRBRCI6IHsid2luZG93IjogMTB9fQ==,min_consensus=2,window=30m,senders=discord,discord_webhook_url=$DISCORD_WEBHOOK_URL" \ + --trigger-arguments "measurement=cpu_metrics,field=utilization,detectors=QuantileAD.LevelShiftAD,detector_params=eyJRdWFudGlsZUFEIjogeyJsb3ciOiAwLjEsICJoaWdoIjogMC45fSwgIkxldmVsU2hpZnRBRCI6IHsid2luZG93IjogMTB9fQ==,min_consensus=2,window=30min,senders=discord,discord_webhook_url=$DISCORD_WEBHOOK_URL" \ cpu_consensus_detector ``` Set `DISCORD_WEBHOOK_URL` to your Discord incoming webhook URL. @@ -163,7 +192,7 @@ influxdb3 create trigger \ --database trading \ --path "gh:influxdata/stateless_adtk_detector/adtk_anomaly_detection_plugin.py" \ --trigger-spec "every:1m" \ - --trigger-arguments "measurement=stock_prices,field=price,detectors=VolatilityShiftAD,detector_params=eyJWb2xhdGlsaXR5U2hpZnRBRCI6IHsid2luZG93IjogMjB9fQ==,window=1h,min_condition_duration=5m,senders=sms,twilio_from_number=+1234567890,twilio_to_number=+0987654321" \ + --trigger-arguments "measurement=stock_prices,field=price,detectors=VolatilityShiftAD,detector_params=eyJWb2xhdGlsaXR5U2hpZnRBRCI6IHsid2luZG93IjogMjB9fQ==,window=1h,min_condition_duration=5min,senders=sms,twilio_from_number=+1234567890,twilio_to_number=+0987654321" \ volatility_detector ``` @@ -173,6 +202,9 @@ influxdb3 create trigger \ - `adtk_anomaly_detection_plugin.py`: The main plugin code containing the scheduled handler for anomaly detection - `adtk_anomaly_config_scheduler.toml`: Example TOML configuration file +- `test_adtk_anomaly_detection.py`: Pytest suite (49 tests, runs without a live {{% product-name %}} server) +- `requirements.txt`: Runtime dependencies (`influxdata-plugin-utils>=0.3.0`, `requests`, `adtk`, `pandas<3`) +- `requirements-dev.txt`: Development dependencies (`pytest`) ### Logging @@ -195,6 +227,14 @@ Key operations: 4. Evaluates consensus across detectors 5. Sends notifications when anomalies are confirmed +#### `parse_detectors(influxdb3_local, config, task_id)` + +Resolves the detectors to apply together with their parameters. Detectors that are unknown, have no entry in `detector_params`, or miss a parameter required to construct them are skipped with a warning and do not count toward `min_consensus`. + +#### `split_by_tags(df, tags, group_by_tags)` + +Splits query results into one frame per tag combination when `group_by_tags` is enabled, so detectors never mix values written under different tag sets. + ## Troubleshooting ### Common issues @@ -207,9 +247,25 @@ Key operations: **Solution**: Increase `min_consensus` to require more detectors to agree. Add `min_condition_duration` to require anomalies to persist. Adjust detector-specific thresholds in `detector_params`. +#### Issue: A newly created measurement is reported as not found + +**Solution**: Table and tag names are cached for one hour per trigger. Wait for the cache to expire, or recreate the trigger to clear it. + +#### Issue: Anomalies of some tag combinations are never detected + +**Solution**: Set `group_by_tags=true`. Without it, rows of different tag combinations share timestamps and only the first series survives. + +#### Issue: A detector is skipped with a warning + +**Solution**: The warning names the reason: the detector is not in the supported list (check the spelling), has no entry in `detector_params`, or misses a required parameter (`window` for `LevelShiftAD` and `VolatilityShiftAD`). `Invalid value 'nan' for dtype 'bool'` means pandas 3 is installed — downgrade to `pandas<3`. + +#### Issue: No anomalies are ever reported + +**Solution**: Check the warnings. `min_consensus` must not exceed the number of detectors that were actually applied — skipped detectors reduce that count. `min_condition_duration` must be shorter than `window`, otherwise no anomaly can persist long enough within a single query window. + #### Issue: Missing dependencies -**Solution**: Install required packages: `adtk`, `pandas`, `requests`. Ensure the Notifier Plugin is installed for notifications. +**Solution**: Install required packages: `influxdata-plugin-utils`, `adtk`, `pandas`, `requests`. Ensure the Notifier Plugin is installed for notifications. #### Issue: Data quality issues @@ -251,4 +307,6 @@ For plugin issues, see the Plugins repository [issues page](https://github.com/i ## Find support for {{% product-name %}} The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. -For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. \ No newline at end of file +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/stock-plugin.md b/content/shared/influxdb3-plugins/plugins-library/official/stock-plugin.md new file mode 100644 index 0000000000..d66b2dbb09 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/stock-plugin.md @@ -0,0 +1,282 @@ + + +The Stock Portfolio Tracker plugin periodically fetches current stock, ETF, and mutual fund prices from Yahoo Finance and writes per-holding rows plus per-portfolio and per-category roll-up totals to {{% product-name %}}. Configure your holdings inline as trigger arguments or in a TOML file, group them into named portfolios (e.g. `401k`, `brokerage`) and user-defined categories (e.g. `Retirement`, `Investment`), and the plugin will track the value of each holding and each grouping over time. Built-in gating skips fetches during market closures (with a clear log line so dashboard gaps have a known cause), polls mutual funds only once per day after NAV close, and carries forward the last known price for skipped symbols so portfolio totals stay accurate. + +### Scope + +Yahoo Finance supports tickers from many global exchanges (e.g. `.L` for London, `.T` for Tokyo, `.PA` for Paris), so the price-fetch path works for international holdings out of the box. The **market-hours gating** is also internationalizable: `market_calendar` and `market_timezone` configure which exchange's calendar to use. Defaults are US-centric (`NYSE` / `America/New_York`) because that's the most common case, but any exchange name accepted by [pandas_market_calendars](https://pandas-market-calendars.readthedocs.io/) works (LSE, TSX, JPX, XETR, ASX, HKEX, etc.). Currency conversion is **not** performed — all values are stored in whatever currency yfinance returns per symbol; this is recorded on the `currency` field of `stock_holdings` rows. If your portfolio mixes currencies, do conversion at query time. + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. The plugin also supports a TOML configuration file for the full portfolio shape; the trigger-argument form is best for one-off testing. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines the supported trigger type and configuration parameters. This metadata enables the InfluxDB 3 Explorer UI to display and configure the plugin. + +### Optional parameters + +| Parameter | Type | Default | Description | +|-----------------------------|---------|--------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `database` | string | `stocks` | Target database for writes. | +| `portfolio` | string | `AAPL:1\|MSFT:1\|GOOG:1` | Inline holdings: pipe-separated `SYMBOL:QUANTITY[:PORTFOLIO_NAME]` entries (e.g. `AAPL:10:401k\|MSFT:5:401k\|GOOG:2.5:brokerage`). Portfolio defaults to `main`. When omitted and no TOML config is found, falls back to the default shown. | +| `categories` | string | none | Inline category map: pipe-separated `PORTFOLIO:CATEGORY` entries (e.g. `401k:Retirement\|brokerage:Investment`). | +| `config_path` | string | `stock_plugin.toml` | Path to the TOML config file, relative to the InfluxDB plugin directory (or absolute). The default file is loaded when it exists; an explicit `config_path` that does not exist is an error. | +| `write_during_closed_hours` | boolean | `true` | See the TOML table below. Also settable as a trigger argument. | +| `mutual_fund_check_time` | string | `18:00` | See the TOML table below. Also settable as a trigger argument. | +| `market_calendar` | string | `NYSE` | See the TOML table below. Also settable as a trigger argument. | +| `market_timezone` | string | `America/New_York` | See the TOML table below. Also settable as a trigger argument. | + +*If neither `portfolio` nor a TOML file with `[holdings.]` is provided, the plugin runs with the default holdings `AAPL:1|MSFT:1|GOOG:1` in the `main` portfolio.* + +### TOML configuration + +The TOML file is the recommended way to configure anything more than a handful of holdings. The plugin reads it from `config_path` (default: `/stock_plugin.toml`). + +Trigger arguments take precedence over TOML keys of the same name, so a TOML file can hold the full portfolio shape while a trigger argument overrides a single setting. + +Holdings and categories are the exception, because each is spelled differently per source: + +| Setting | Trigger argument | TOML | +|------------|---------------------------------------|--------------------------------| +| Holdings | `portfolio=AAPL:10:401k\|MSFT:5:401k` | `[holdings.401k]` tables | +| Categories | `categories=401k:Retirement` | `[portfolio_categories]` table | + +Each spelling is read only from its own source: a top-level `portfolio` or `categories` key in the TOML file is ignored, as is a trigger argument named `holdings` or `portfolio_categories`. When both sources are present, the trigger argument replaces the TOML tables entirely rather than merging with them. + +| Key | Type | Default | Description | +|-----------------------------|---------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `database` | string | `stocks` | Target database for writes. | +| `write_during_closed_hours` | boolean | `true` | When `false`, stocks/ETFs are skipped outside the configured exchange's regular session (the calendar handles holidays and early closes). Mutual funds always follow their own daily check schedule. | +| `mutual_fund_check_time` | string | `"18:00"` | Time of day in `market_timezone` after which the plugin fetches mutual fund NAV. Mutual funds are fetched at most once per local calendar day, at the first tick at or after this time. Bootstrap exception: a mutual fund with no cached asset type is fetched on its first tick regardless of time. | +| `market_calendar` | string | `"NYSE"` | Exchange calendar used for the market-hours check. Any name accepted by [pandas_market_calendars](https://pandas-market-calendars.readthedocs.io/) (e.g. `NYSE`, `LSE`, `TSX`, `JPX`, `XETR`, `ASX`, `HKEX`). | +| `market_timezone` | string | `"America/New_York"` | IANA timezone for the exchange's local time. Used for `mutual_fund_check_time` comparisons and for resolving the "today" date the calendar consults. | +| `[portfolio_categories]` | table | empty | Maps portfolio name to category name. Portfolios not listed are uncategorized (omitted from `category_totals`). | +| `[holdings.]` | table | default holdings | Holdings for each portfolio. Each entry is `SYMBOL = quantity`. Fractional quantities supported; the quantity must be a finite number (`inf` and `nan` are rejected). Quote symbols containing dots, for example `"VOD.L" = 10`. Duplicate same-symbol entries in one portfolio are aggregated. The portfolio name `_total` is reserved. When no `[holdings.*]` section is present, the plugin falls back to `AAPL:1\|MSFT:1\|GOOG:1`. | + +The trigger spec is the source of truth for cadence. For example, `--trigger-spec "every:15m"` runs the plugin every 15 minutes. + +#### Example TOML configuration + +[stock_plugin.toml.example](stock_plugin.toml.example) + +## Software requirements + +- **{{% product-name %}}**: with the Processing Engine enabled. +- **Python 3.11 or higher** +- **Python packages** (installed into the plugin venv): + - `yfinance` — Yahoo Finance scraper for price data + - `pandas_market_calendars` — exchange calendars for accurate market-hours and holiday gating + - `influxdata-plugin-utils>=0.3.0` — shared configuration, parsing, and write helpers + +### Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install required Python packages: + + ```bash + influxdb3 install package yfinance pandas_market_calendars influxdata-plugin-utils + ``` +3. Copy `stock_plugin.toml.example` to `/stock_plugin.toml` and edit it with your holdings and categories. + +## Trigger setup + +### Basic scheduled trigger (TOML config) + +```bash +influxdb3 create trigger \ + --database stocks \ + --path "stock_plugin.py" \ + --trigger-spec "every:15m" \ + --trigger-arguments config_path=stock_plugin.toml \ + --error-behavior log \ + stock_portfolio_tracker +``` +### Inline holdings without TOML + +```bash +influxdb3 create trigger \ + --database stocks \ + --path "stock_plugin.py" \ + --trigger-spec "every:15m" \ + --trigger-arguments 'portfolio=AAPL:10:401k|MSFT:5:401k|GOOG:2:brokerage,categories=401k:Retirement|brokerage:Investment' \ + --error-behavior log \ + stock_portfolio_tracker +``` +### Skip writes when the configured market is closed + +Set `write_during_closed_hours = false` in your TOML config and create the trigger as normal. With the default NYSE calendar, stocks and ETFs will skip outside 9:30–16:00 ET on weekdays and on NYSE holidays. Their portfolio total values are still maintained via carry-forward. + +On a cold cache, such as after an InfluxDB restart, a symbol that would otherwise be skipped can be fetched once so the plugin has a last-known price to carry forward on later skipped ticks. The summary log calls these out as cold-cache bootstrap fetches. + +## Example usage + +### Latest portfolio value by category + +```bash +influxdb3 query \ + --database stocks \ + "SELECT category, value, portfolio_count, symbol_count + FROM category_totals + WHERE time = (SELECT MAX(time) FROM category_totals)" +``` +**Expected output** + +``` ++------------+-----------+-----------------+--------------+ +| category | value | portfolio_count | symbol_count | ++------------+-----------+-----------------+--------------+ +| Investment | 257908.71 | 2 | 17 | +| Retirement | 1990604.5 | 3 | 5 | ++------------+-----------+-----------------+--------------+ +``` +### Total portfolio value over time + +```bash +influxdb3 query \ + --database stocks \ + "SELECT time, value FROM portfolio_totals + WHERE portfolio = '_total' ORDER BY time DESC LIMIT 10" +``` +### Daily P&L per holding (using previous_close) + +```bash +influxdb3 query \ + --database stocks \ + "SELECT symbol, portfolio, + (price - previous_close) * quantity AS daily_change + FROM stock_holdings + WHERE time = (SELECT MAX(time) FROM stock_holdings) + AND previous_close IS NOT NULL + ORDER BY daily_change DESC" +``` +### Filter for fully fresh rows + +Rows in `portfolio_totals` and `category_totals` carry `missing_symbols`, `skipped_symbols`, and `carried_symbols` counts so dashboards can distinguish fully fresh data from carry-forward or partial fetches: + +```sql +-- "All symbols fetched fresh this tick" +SELECT * FROM portfolio_totals +WHERE missing_symbols = 0 AND skipped_symbols = 0; + +-- "All values present (fresh + carry-forward), no failures" +SELECT * FROM portfolio_totals +WHERE missing_symbols = 0; +``` +## Code overview + +### Main functions + +#### `process_scheduled_call(influxdb3_local, call_time, args)` + +Entry point for the scheduled trigger. Stamps one UTC timestamp for the whole run and delegates to `_main` with the runtime-injected `LineBuilder` and the live `influxdb3_local`. All side-effecting work lives in `_main` so its logic can be reasoned about with injected dependencies. + +#### `_main(local, args, fetcher, line_builder_cls, now_ns, task_id)` + +Drives the full plugin flow: + +1. Resolve config from trigger args + TOML. +2. Determine configured market state via `pandas_market_calendars` and parse the mutual-fund check time. +3. For each configured holding, decide whether to fetch (gated by asset type, cached state, and config), fetch via `yfinance.fast_info`, and build a `HoldingRow`. +4. Build carry-forward `HoldingRow`s for intentionally-skipped symbols whose last known price is cached. +5. Aggregate per-portfolio totals + a grand `_total` row. +6. Aggregate per-category totals across portfolios. +7. Write `stock_holdings`, `portfolio_totals`, and `category_totals` as a single batched payload. +8. Log a single summary line. + +#### `resolve_config(args)` + +Merges the TOML file with the trigger arguments and validates the result. The TOML path comes from `config_path`; relative paths resolve against the plugin directory (`PLUGIN_DIR`, `INFLUXDB3_PLUGIN_DIR`, or the `VIRTUAL_ENV` parent). Returns a `ResolvedConfig`, raising `ValueError` on any invalid value. + +### Measurements and fields + +#### `stock_holdings` + +One row per symbol per successful fetch. Carry-forward symbols are NOT written here (their last fresh row remains the most recent record in this measurement). + +- **Tags**: `symbol`, `portfolio`, `asset_type` (`equity`, `etf`, `mutualfund`, `other`), `category` (omitted if portfolio is uncategorized) +- **Fields**: `price`, `quantity`, `value`, `currency`, `previous_close`, `day_open`, `day_high`, `day_low` + +#### `portfolio_totals` + +One row per configured portfolio plus a `_total` grand-total row, written every tick. + +- **Tags**: `portfolio`, `category` (omitted if uncategorized; always omitted on `_total`) +- **Fields**: `value`, `symbol_count`, `missing_symbols`, `skipped_symbols`, `carried_symbols` + +#### `category_totals` + +One row per defined category, rolled up across portfolios in that category. Uncategorized portfolios and the `_total` row are excluded to avoid double-counting. + +- **Tags**: `category` +- **Fields**: `value`, `symbol_count`, `portfolio_count`, `missing_symbols`, `skipped_symbols`, `carried_symbols` + +### Internal cache keys + +The plugin uses `influxdb3_local.cache` (no TTL) for state across runs: + +| Key | Value | Purpose | +|------------------------------|-----------------|---------------------------------------------------------------------------------------------------------------| +| `asset_type:` | string | Cached `yfinance.fast_info.quote_type` so we don't re-derive it each run. | +| `last_mf_date:` | `YYYY-MM-DD` (ET) | Last calendar day a mutual fund was fetched. Used to enforce once-per-day NAV polling. | +| `last_price:` | float | Last known price. Used to carry forward portfolio value when a symbol is skipped. | + +Cache is cleared on server restart. The plugin self-bootstraps: any symbol with a missing cache key gets a fresh fetch on its next tick regardless of skip rules. + +## Troubleshooting + +### Common issues + +#### Issue: No holdings are configured + +**Solution:** With no `portfolio` argument and no TOML file, the plugin uses the default holdings `AAPL:1|MSFT:1|GOOG:1`. To track your own holdings, provide `portfolio` trigger arguments or a TOML file with at least one `[holdings.]` table. If you use a TOML file, confirm that `config_path` points to the file in the {{% product-name %}} plugin directory. + +#### Issue: Market-hours checks skip expected writes + +**Solution:** Set `write_during_closed_hours = true` or choose the correct `market_calendar` and `market_timezone` for your exchange. + +When `write_during_closed_hours` is false, the plugin uses `pandas_market_calendars` to skip equity and ETF fetches outside the configured exchange session. Mutual funds are still gated by `mutual_fund_check_time`. + +#### Issue: Yahoo Finance returns missing prices + +**Solution:** Verify the ticker symbol and check whether Yahoo Finance exposes current price data for that instrument. + +A symbol whose price is missing or not a finite number is counted as a fetch failure and reported in the summary log; the optional `previous_close`, `day_open`, `day_high`, and `day_low` fields are simply omitted when unusable. The plugin carries forward the last known price for intentionally skipped symbols, but it cannot value a new holding until the first successful fetch. + +### Debugging tips + +Check `system.processing_engine_logs` for the trigger summary line. It reports fetched, skipped, carried, and missing symbol counts for each scheduled run. + + +## Logging + +Logs are stored in the `_internal` database (or the database where the trigger is created) in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" +``` + +Log columns: +- **event_time**: Timestamp of the log event +- **trigger_name**: Name of the trigger that generated the log +- **log_level**: Severity level (INFO, WARN, ERROR) +- **log_text**: Message describing the action or error + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/synthefy-forecasting.md b/content/shared/influxdb3-plugins/plugins-library/official/synthefy-forecasting.md new file mode 100644 index 0000000000..aa89ae56c1 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/synthefy-forecasting.md @@ -0,0 +1,402 @@ + + +The Synthefy Forecasting Plugin integrates the Synthefy Forecasting API with {{% product-name %}} to enable on-demand time series forecasting via HTTP requests. It reads time series data from InfluxDB, generates forecasts using Synthefy's foundation models, and writes the results back to InfluxDB for visualization and alerting. + +**Key Features:** + +- **On-Demand Forecasting**: Generate forecasts on-demand via HTTP requests +- **Multiple Models**: Support for various Synthefy models +- **Metadata Support**: Use additional fields as covariates for improved accuracy +- **Tag Filtering**: Filter input data by tags (for example, location, device); supports multiple values per tag (`IN (...)`) +- **Line Protocol Writes**: Reliable, batched writing via `write_sync` / `write_sync_to_db` + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger, and/or in the JSON body of each HTTP request. Body values override trigger arguments. A `null` in the body counts as unset, so the trigger argument applies; send `{}` for `tags` or `[]` for `metadata_fields` to clear them. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Authentication for the Synthefy API + +The Synthefy API key is **never** read from trigger arguments or the request body. It must be provided via either of: + +- HTTP request header: `X-Synthefy-Api-Key: ` +- Environment variable: `SYNTHEFY_API_KEY` + +If both are set, the header takes precedence. + +### Request body parameters + +| Parameter | Type | Default | Description | +|----------------------|----------------|----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `measurement` | string | required | Source measurement (table) containing historical data | +| `field` | string | `"value"` | Field name to forecast | +| `tags` | string \| dict | `""` | Tag filters. Trigger args: dot-separated string `key:val1@val2.key2:val3`. Request body: JSON object mapping tag name to a string or list of strings. See [Tag filter format](#tag-filter-format). | +| `time_range` | string | `"30d"` | Historical window. Format: ``. Units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`, `m`, `q`, `y` (`m`/`q`/`y` are approximate). | +| `forecast_horizon` | string | `"7d"` | Forecast duration. Format: `` (same units as `time_range`) or ` points`. | +| `model` | string | `"sfm-tabular"` | Synthefy model identifier (for example, `sfm-tabular`, `Migas-latest`) | +| `output_measurement` | string | `"{measurement}_forecast"` | Destination measurement for forecast results | +| `metadata_fields` | string \| list | `""` | Trigger args: space-separated list of field names (`"humidity pressure"`). Request body: JSON list of strings. Used as covariates. | +| `max_forecast_points`| integer | `10000` | Upper bound on the forecast points one request may produce. A time-based `forecast_horizon` is divided by the series' own step, so a dense series with a long horizon builds a very large payload. | +| `database` | string | `""` | Optional override database for **writes only**. If unset, forecasts are written to the trigger's database. Reads always go to the trigger's database. | + +#### Tag filter format + +The plugin supports multi-value tag filters that are translated to `tag IN ('a', 'b', ...)` in SQL. + +**Trigger arguments (string form)** — based on the downsampler convention: + +- `.` separates `key:value` pairs +- `:` separates the tag name from its value(s) +- `@` separates multiple values for the same tag +- Quote a value with `'...'` or `"..."` if it contains `:`, `@` or `.`. A quote inside a value, as in `Bob's`, needs no escaping. An unclosed quote is rejected. + +Examples: +``` +tags="room:Bedroom" +tags="room:Bedroom@Kitchen.location:Hall" +tags="room:'Some other room'@Bedroom.device:sensor1" +tags="owner:Bob's.room:Bedroom" +``` +**Request body (JSON form)**: +```json +{ "tags": { "room": ["Bedroom", "Kitchen"], "location": "Hall" } } +``` +The body also accepts the string form above. Send `{}` to clear the tag filters configured on the trigger. + +#### Forecast points and tags + +When a tag filter has a single value, that value is added as a tag on every forecast point. When it has multiple values (an `IN (...)` filter), no value is written for that tag — the response covers several tag values at once. + +## Requirements + +### Dependencies + +- Python 3.11 or higher +- `pandas` — Data manipulation +- `requests` — HTTP client for the Synthefy API +- `influxdata-plugin-utils` — Shared configuration, schema and write helpers + +### Installation steps + +Using the {{% product-name %}} package manager: + +```bash +influxdb3 install package pandas +influxdb3 install package requests +influxdb3 install package influxdata-plugin-utils +``` +### Prerequisites + +- {{% product-name %}} Core or Enterprise installed and running +- Synthefy API key — create one at [https://console.synthefy.com/api-keys](https://console.synthefy.com/api-keys) + +## Quick Start / Testing Setup + +Create a database and write some sample data: + +```bash +influxdb3 create database mydb + +NOW=$(date +%s) +for i in {0..168}; do + TIMESTAMP=$((NOW - (168 - i) * 3600))000000000 + influxdb3 write --database mydb "temperature,location=NYC value=$((70 + RANDOM % 10)),humidity=$((60 + RANDOM % 15)),pressure=$((1000 + RANDOM % 20)) ${TIMESTAMP}" +done +``` +**Note**: In {{% product-name %}}, tables/measurements are created automatically when you first write data to them. + +**Quick check that data exists:** +```bash +influxdb3 query --database mydb "SELECT COUNT(*) FROM temperature" +influxdb3 query --database mydb "SELECT * FROM temperature ORDER BY time DESC LIMIT 5" +``` +## Trigger setup + +### HTTP trigger + +Create and enable the trigger: + +```bash +influxdb3 create trigger \ + --database mydb \ + --path "gh:influxdata/synthefy_forecasting/synthefy_forecasting.py" \ + --trigger-spec "request:forecast" \ + --trigger-arguments measurement=temperature,field=value \ + temperature_forecast_http + +influxdb3 enable trigger --database mydb temperature_forecast_http +``` +Trigger arguments act as defaults; any field can be overridden in the request body. + +Then call via HTTP. **Always** pass the Synthefy API key as a header (or set the `SYNTHEFY_API_KEY` env var on the InfluxDB process): + +```bash +TOKEN=$(influxdb3 create token --admin --offline | grep token | cut -d'=' -f2) + +curl -X POST http://localhost:8181/api/v3/engine/forecast \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -H "X-Synthefy-Api-Key: YOUR_SYNTHEFY_API_KEY" \ + -d '{ + "measurement": "temperature", + "field": "value", + "time_range": "30d", + "forecast_horizon": "7d", + "model": "sfm-tabular" + }' +``` +**Important Notes:** +- Endpoint is `/api/v3/engine/forecast` (matches the `request:forecast` trigger spec). +- Authentication for InfluxDB itself is handled by the framework via the `Authorization` header. +- The Synthefy API key must be in the `X-Synthefy-Api-Key` header or in the `SYNTHEFY_API_KEY` env var; it cannot be passed via trigger arguments or request body. + +## Example usage + +### Basic forecast + +```bash +TOKEN=$(influxdb3 create token --admin --offline | grep token | cut -d'=' -f2) + +curl -X POST http://localhost:8181/api/v3/engine/forecast \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -H "X-Synthefy-Api-Key: YOUR_SYNTHEFY_API_KEY" \ + -d '{ + "measurement": "temperature", + "field": "value" + }' +``` +### Forecast with single-tag filter + +```bash +curl -X POST http://localhost:8181/api/v3/engine/forecast \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -H "X-Synthefy-Api-Key: YOUR_SYNTHEFY_API_KEY" \ + -d '{ + "measurement": "temperature", + "field": "value", + "tags": { "location": "NYC" }, + "time_range": "30d" + }' +``` +### Forecast with multi-value tag filter + +```bash +curl -X POST http://localhost:8181/api/v3/engine/forecast \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -H "X-Synthefy-Api-Key: YOUR_SYNTHEFY_API_KEY" \ + -d '{ + "measurement": "temperature", + "field": "value", + "tags": { "location": ["NYC", "SF"], "device": "sensor1" }, + "time_range": "30d" + }' +``` +### Forecast with metadata covariates + +```bash +curl -X POST http://localhost:8181/api/v3/engine/forecast \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -H "X-Synthefy-Api-Key: YOUR_SYNTHEFY_API_KEY" \ + -d '{ + "measurement": "temperature", + "field": "value", + "metadata_fields": ["humidity", "pressure"], + "time_range": "30d" + }' +``` +### Trigger arguments form + +The same parameters can also be set as trigger arguments (note the dot/colon/`@` syntax for `tags` and the space-separated list for `metadata_fields`): + +```bash +influxdb3 create trigger \ + --database mydb \ + --path "gh:influxdata/synthefy_forecasting/synthefy_forecasting.py" \ + --trigger-spec "request:forecast" \ + --trigger-arguments 'measurement=temperature,field=value,tags=location:NYC@SF.device:sensor1,metadata_fields=humidity pressure,time_range=30d,forecast_horizon=7d' \ + temperature_forecast_http +``` +### Advanced model + +```bash +curl -X POST http://localhost:8181/api/v3/engine/forecast \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -H "X-Synthefy-Api-Key: YOUR_SYNTHEFY_API_KEY" \ + -d '{ + "measurement": "sales", + "field": "revenue", + "model": "Migas-latest", + "forecast_horizon": "30d", + "time_range": "90d" + }' +``` +### Writing the forecast to a different database + +```bash +curl -X POST http://localhost:8181/api/v3/engine/forecast \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: application/json" \ + -H "X-Synthefy-Api-Key: YOUR_SYNTHEFY_API_KEY" \ + -d '{ + "measurement": "temperature", + "field": "value", + "database": "forecasts" + }' +``` +## Output Format + +Forecasts are written to a new measurement (default: `{measurement}_forecast`) using `write_sync` (or `write_sync_to_db` when `database` is set) with `no_sync=True`, batched into a single line-protocol payload. + +- **Measurement**: `{measurement}_forecast` (configurable via `output_measurement`) +- **Tags**: Single-value tag filters from the request + `model={model_name}` +- **Fields**: + - `{field_name}`: Forecasted values + - `value_{quantile}`: Quantile forecasts when available (for example, `value_0.1`, `value_0.9`) + +Example Line Protocol output: + +``` +temperature_forecast,location=NYC,model=sfm-tabular value=72.5,value_0.1=71.2,value_0.9=73.8 1704672000000000000 +``` +## Querying Forecasts + +Forecast points sit in the future, so query by an upcoming time window rather than a past one: + +```sql +SELECT * +FROM temperature_forecast +WHERE time >= now() AND time <= now() + INTERVAL '7 days' +ORDER BY time +``` +Historical data — past 30 days: + +```sql +SELECT time, value +FROM temperature +WHERE time >= now() - INTERVAL '30 days' +ORDER BY time +``` +## Supported Models + +The plugin supports models available through the Synthefy Forecasting API. Key supported models include: + +- `sfm-tabular`: Synthefy Foundation Model for tabular/multivariate time series +- `Migas-latest`: Latest Migas foundation model + +Check the [Synthefy documentation](https://docs.synthefy.com) for the most up-to-date model list and availability. + +## Code overview + +### Files + +- `synthefy_forecasting.py`: HTTP trigger entry point, argument parsing, InfluxDB query construction, Synthefy API calls, and forecast writes. +- `requirements.txt`: Python packages required by the plugin. +- `README.md`: Setup, usage, and troubleshooting documentation. + +### Key functions + +- `process_request(influxdb3_local, query_parameters, request_headers, request_body, args=None)`: handles HTTP requests, merges trigger arguments with request-body overrides, validates input, calls Synthefy, and writes forecast points. +- `build_history_query(measurement, field, metadata_fields, tag_filters, start_time)`: builds the parameterized SQL query for historical data. +- `dataframe_to_synthefy_request(influxdb3_local, df, field, forecast_horizon, metadata_fields, model, max_forecast_points, task_id)`: converts InfluxDB query rows into the Synthefy forecast request payload and enforces the point limit. +- `forecast_response_to_line_builders(influxdb3_local, forecast_response, output_measurement, tag_filters, model, field_name, task_id)`: converts Synthefy forecast results into InfluxDB line protocol builders. + +## Troubleshooting + +### Common issues + +### Missing API key + +`{"message":"Missing API key"}` — set `X-Synthefy-Api-Key` in the request headers, or `SYNTHEFY_API_KEY` in the environment of the InfluxDB process. + +### Measurement not found + +`{"message":"Measurement '' not found"}` — the plugin verifies the measurement exists by querying `information_schema.columns`. Make sure the measurement has been written to in the trigger's database. + +### Field does not exist + +`{"message":"Field '' does not exist in ''"}` — the requested `field` (or `metadata_fields` entries) was not found in the measurement schema. + +### No data found + +`{"message":"No data found"}` — the query returned zero rows. Widen `time_range`, relax `tags` filters, or verify timestamps fall inside the window. + +### Invalid interval format + +`Invalid interval format: ''. Expected ''.` — `time_range` and `forecast_horizon` must be `` (units: `us`, `ms`, `s`, `min`, `h`, `d`, `w`, `m`, `q`, `y`) or, for `forecast_horizon`, ` points`. + +### Forecast horizon too large + +`forecast_horizon '' resolves to N points … above the max_forecast_points limit` — the horizon is divided by the interval between the last two historical points, so a dense series produces many points. Shorten `forecast_horizon`, use the ` points` form, or raise `max_forecast_points`. + +### Timestamps collapse onto each other + +`N history timestamps differ by less than a microsecond …` or `The series' step of … is finer than a microsecond …` — timestamps are sent to Synthefy with microsecond precision, so a series stepping in nanoseconds cannot be represented. Resample it to a coarser step. + +### History holds repeated timestamps + +`History holds N repeated timestamps, so the window covers more than one series …` — the query matched several tag series and their values are interleaved in one input sequence. Add a `tags` filter that selects a single series. + +### Synthefy API errors + +If Synthefy API calls fail: + +- Verify the API key is correct +- Check API URL accessibility and rate limits +- Check network connectivity + +### Write errors + +If writes fail: + +- Ensure the database exists (the trigger database, or the override `database` if used) +- Ensure the plugin has write permissions +- Check the `[task_id] Failed to write forecasts after N attempts: …` error in the InfluxDB logs + +### Query file limit exceeded ({{% product-name %}} Core) + +If you see "Query would scan X Parquet files, exceeding the file limit" errors, narrow `time_range` (for example, `90d` instead of `730d`), or upgrade to {{% product-name %}} Enterprise (which compacts files and removes the limit). + +## Limitations + +- Currently supports a single time series per request (one `field` plus optional covariates). A request whose window matches several tag series is logged as a warning. +- Forecast horizon calculation assumes regular time intervals. +- Timestamps are exchanged with microsecond precision; a series whose step is finer is rejected. +- In the trigger-arguments string form, a tag value containing `:`, `@` or `.` must be quoted; the JSON request body needs no quoting. + +## License + +Apache 2.0 + + +## Logging + +Logs are stored in the `_internal` database (or the database where the trigger is created) in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database _internal "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" +``` + +Log columns: +- **event_time**: Timestamp of the log event +- **trigger_name**: Name of the trigger that generated the log +- **log_level**: Severity level (INFO, WARN, ERROR) +- **log_text**: Message describing the action or error + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/system-metrics.md b/content/shared/influxdb3-plugins/plugins-library/official/system-metrics.md index a8add5e3fc..820bb0786e 100644 --- a/content/shared/influxdb3-plugins/plugins-library/official/system-metrics.md +++ b/content/shared/influxdb3-plugins/plugins-library/official/system-metrics.md @@ -1,4 +1,5 @@ - + + The System Metrics Plugin provides comprehensive system monitoring capabilities for {{% product-name %}}, collecting CPU, memory, disk, and network metrics from the host system. Monitor detailed performance insights including per-core CPU statistics, memory usage breakdowns, disk I/O performance, and network interface statistics. Features configurable metric collection with robust error handling and retry logic for reliable monitoring. ## Configuration @@ -13,24 +14,28 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor ### Optional parameters -| Parameter | Type | Default | Description | -|-------------------|---------|-------------|--------------------------------------------------------------------------------| -| `hostname` | string | `localhost` | Hostname to tag all metrics with for system identification | -| `include_cpu` | boolean | `true` | Include comprehensive CPU metrics collection (overall and per-core statistics) | -| `include_memory` | boolean | `true` | Include memory metrics collection (RAM usage, swap statistics, page faults) | -| `include_disk` | boolean | `true` | Include disk metrics collection (partition usage, I/O statistics, performance) | -| `include_network` | boolean | `true` | Include network metrics collection (interface statistics and error counts) | -| `max_retries` | integer | `3` | Maximum retry attempts on failure with graceful error handling | +| Parameter | Type | Default | Description | +|-------------------|---------|-------------|--------------------------------------------------------------------------------------------------| +| `hostname` | string | `localhost` | Hostname to tag all metrics with for system identification | +| `include_cpu` | boolean | `true` | Include comprehensive CPU metrics collection (overall and per-core statistics) | +| `include_memory` | boolean | `true` | Include memory metrics collection (RAM usage, swap statistics, page faults) | +| `include_disk` | boolean | `true` | Include disk metrics collection (partition usage, I/O statistics, performance) | +| `include_network` | boolean | `true` | Include network metrics collection (interface statistics and error counts) | +| `max_retries` | integer | `3` | Retry attempts per metric type; the group is skipped and the run continues once they are used up | *Note: This plugin has no required parameters. All parameters have sensible defaults.* +Boolean parameters accept `true`/`false`, `1`/`0`, `yes`/`no`, and `on`/`off`. A value the plugin cannot interpret is reported in the logs and the run collects nothing, so fix the trigger arguments and the next run recovers. + ### TOML configuration | Parameter | Type | Default | Description | |--------------------|--------|---------|----------------------------------------------------------------------------------| | `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (required for TOML configuration) | -*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. Relative paths are resolved against the first directory that is set: `PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent of `VIRTUAL_ENV`. Only that directory is used — the file is not looked up in the remaining ones. + +Values in the TOML file override the inline trigger arguments. If the file cannot be read, the plugin logs an error and collects metrics using the inline arguments and defaults. #### Example TOML configuration @@ -41,8 +46,7 @@ For more information on using TOML configuration files, see the Using TOML Confi ## Software Requirements - **{{% product-name %}}**: with the Processing Engine enabled. -- **Python packages**: - - `psutil` (for system metrics collection) +- **Python packages**: `influxdata-plugin-utils>=0.3.0`, `psutil` ### Installation steps @@ -58,6 +62,7 @@ For more information on using TOML configuration files, see the Using TOML Confi 2. Install required Python packages: ```bash + influxdb3 install package "influxdata-plugin-utils>=0.3.0" influxdb3 install package psutil ``` ## Trigger setup @@ -153,21 +158,19 @@ influxdb3 query \ #### `process_scheduled_call()` -The main entry point for scheduled triggers. Collects system metrics based on configuration and writes them to InfluxDB. +The main entry point for scheduled triggers. Loads the configuration, then runs each enabled collector and writes the lines it built. A collector is retried up to `max_retries` times, and its lines are written only once it completes. ```python -def process_scheduled_call(influxdb3_local, call_time, args): - # Parse configuration - config = parse_config(args) - - # Collect metrics based on configuration - if config['include_cpu']: - collect_cpu_metrics(influxdb3_local, config['hostname']) - - if config['include_memory']: - collect_memory_metrics(influxdb3_local, config['hostname']) - - # ... additional metric collections +def process_scheduled_call(influxdb3_local, call_time, args=None): + config = _load_config(influxdb3_local, args, task_id) + + for config_key, metric_type, collect in _COLLECTORS: + if not config[config_key]: + continue + lines = _collect_with_retry( + influxdb3_local, collect, metric_type, hostname, max_retries, task_id + ) + write_data(influxdb3_local, lines, batch=False, retries=0) ``` ### Measurements and Fields @@ -178,6 +181,8 @@ Overall CPU statistics and metrics: - **Tags**: `host`, `cpu=total` - **Fields**: `user`, `system`, `idle`, `iowait`, `nice`, `irq`, `softirq`, `steal`, `guest`, `guest_nice`, `frequency_current`, `frequency_min`, `frequency_max`, `ctx_switches`, `interrupts`, `soft_interrupts`, `syscalls`, `load1`, `load5`, `load15` +The state shares (`user` through `guest_nice`) are derived from the change in the CPU time counters between two consecutive runs, so each value covers the interval between the previous run and the current one. They are absent on the first run after the trigger is created or restarted; the remaining fields are written from the first run on. + #### system_cpu_cores Per-core CPU statistics: @@ -185,6 +190,8 @@ Per-core CPU statistics: - **Tags**: `host`, `core` (core number) - **Fields**: `usage`, `user`, `system`, `idle`, `iowait`, `nice`, `irq`, `softirq`, `steal`, `guest`, `guest_nice`, `frequency_current`, `frequency_min`, `frequency_max` +Shares are derived the same way as in `system_cpu`; `usage` is the busy share of the core, everything except `idle` and `iowait`. + #### system_memory System memory statistics: @@ -222,11 +229,13 @@ Disk I/O statistics: #### system_disk_performance -Calculated disk performance metrics: +Disk performance rates, derived from the change in the I/O counters between two consecutive runs of the plugin: - **Tags**: `host`, `device` - **Fields**: `read_bytes_per_sec`, `write_bytes_per_sec`, `read_iops`, `write_iops`, `avg_read_latency_ms`, `avg_write_latency_ms`, `util_percent` +Each value covers the interval between the previous run and the current one, so the shorter the trigger interval, the finer the resolution. A device gets no line when there is nothing to compare against: on the first run after the trigger is created or restarted, and when its counters were reset (for example after the device was re-attached). + #### system_network Network interface statistics: @@ -242,13 +251,26 @@ Network interface statistics: **Solution**: The plugin will continue collecting other metrics even if some require elevated permissions. Run InfluxDB with appropriate permissions if disk I/O metrics are required. -#### Issue: Missing psutil library +#### Issue: Missing Python packages -**Solution**: Install the psutil package: +**Solution**: Install the required packages: ```bash +influxdb3 install package "influxdata-plugin-utils>=0.3.0" influxdb3 install package psutil ``` +#### Issue: No `system_disk_performance` data, or CPU shares are missing + +**Solution**: Both are derived from two consecutive runs. Wait for the second run of the trigger; if the values stay missing, check the logs for `No previous disk I/O sample` and `No previous CPU sample`, which repeat when the cached counters are lost on every run. + +#### Issue: One metric group is missing while the others are written + +**Solution**: A collector that keeps failing is skipped so the rest of the run survives. Look for `Failed to collect metrics after N retries` in the logs, followed by `skipped after repeated failures`, which names every group left out of that run. + +#### Issue: No metrics at all and a configuration error in the logs + +**Solution**: An invalid parameter value stops the run before any collection. Look for `Failed to load configuration` in the logs, which names the offending value, and fix the trigger arguments. A TOML file that cannot be read is a separate case: it is logged as `Failed to apply config file` and collection continues with the inline arguments. + #### Issue: High CPU usage from plugin **Solution**: Increase the trigger interval (for example, from `every:10s` to `every:30s`). Disable unnecessary metric types. Reduce the number of disk partitions monitored. @@ -299,4 +321,6 @@ For plugin issues, see the Plugins repository [issues page](https://github.com/i ## Find support for {{% product-name %}} The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. -For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. \ No newline at end of file +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/threshold-deadman-checks.md b/content/shared/influxdb3-plugins/plugins-library/official/threshold-deadman-checks.md index 1a59cabd75..a6a905651a 100644 --- a/content/shared/influxdb3-plugins/plugins-library/official/threshold-deadman-checks.md +++ b/content/shared/influxdb3-plugins/plugins-library/official/threshold-deadman-checks.md @@ -1,4 +1,5 @@ - + + The Threshold Deadman Checks Plugin provides comprehensive monitoring capabilities for time series data in {{% product-name %}}, combining real-time threshold detection with deadman monitoring. Monitor field values against configurable thresholds, detect data absence patterns, and trigger multi-level alerts based on aggregated metrics. Features both scheduled batch monitoring and real-time data write monitoring with configurable trigger counts and severity levels. ## Configuration @@ -11,11 +12,11 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor ### Required parameters -| Parameter | Type | Default | Description | -|---------------|--------|----------|---------------------------------------------------------------------------------------------------| -| `measurement` | string | required | Measurement to monitor for deadman alerts and aggregation-based conditions | -| `senders` | string | required | Dot-separated notification channels with multi-channel notification integration | -| `window` | string | required | Time window for periodic data presence checking | +| Parameter | Type | Default | Description | +|---------------|--------|----------|------------------------------------------------------------------------------------------------------------------------------------------| +| `measurement` | string | required | Measurement to monitor for deadman alerts and aggregation-based conditions | +| `senders` | string | required | Dot-separated notification channels with multi-channel notification integration | +| `window` | string | required | Time window for periodic data presence checking. Format: ``, units: `s`, `min`, `h`, `d`, `w`. Must be a positive duration | ### Data write trigger parameters @@ -27,12 +28,12 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor ### Threshold check parameters -| Parameter | Type | Default | Description | -|----------------------------|---------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `field_aggregation_values` | string | none | Multi-level aggregation conditions with aggregation support for avg, min, max, count, sum, median, stddev, first_value, last_value, var, and approx_median values | -| `deadman_check` | boolean | false | Enable deadman detection to monitor for data absence and missing data streams | -| `interval` | string | "5min" | Configurable aggregation time interval for batch processing with performance optimization | -| `trigger_count` | number | 1 | Configurable triggers requiring multiple consecutive failures before alerting | +| Parameter | Type | Default | Description | +|----------------------------|---------|---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `field_aggregation_values` | string | none | Multi-level aggregation conditions with aggregation support for avg, min, max, count, sum, median, stddev, first_value, last_value, var, and approx_median values | +| `deadman_check` | boolean | false | Enable deadman detection to monitor for data absence and missing data streams | +| `interval` | string | "5min" | Aggregation time interval used in `DATE_BIN`. Format: ``, units: `s`, `min`, `h`, `d`, `w` | +| `trigger_count` | number | 1 | Breaches required before alerting. Threshold checks count consecutive breaches per row identifier, including across the time bins of a single run; deadman checks count consecutive runs without data | ### Notification parameters @@ -51,7 +52,11 @@ This plugin includes a JSON metadata schema in its docstring that defines suppor |--------------------|--------|---------|----------------------------------------------------------------------------------| | `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (required for TOML configuration) | -*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. Relative paths are resolved against the first directory that is set: `PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent of `VIRTUAL_ENV`. Only that directory is used — the file is not looked up in the remaining ones. + +When `config_file_path` is set, the TOML file provides the whole configuration and inline trigger arguments are ignored. `INFLUXDB3_AUTH_TOKEN` from the environment still applies when `influxdb3_auth_token` is not set in the file. In TOML, `senders`, `field_conditions`, and `field_aggregation_values` use native structures instead of the inline string formats. + +Data write triggers cache the loaded configuration for 10 minutes to keep the write path fast, so configuration changes take effect within that window. Example TOML configuration files provided: @@ -71,6 +76,7 @@ The plugin assumes that the table schema is already defined in the database, as ## Software requirements - **InfluxDB v3 Core/Enterprise**: with the Processing Engine enabled. +- **Python packages**: `influxdata-plugin-utils>=0.3.0`, `requests` - **Notification Sender Plugin for {{% product-name %}}**: This plugin is required for sending notifications. See the [influxdata/notifier plugin](/influxdb3/version/plugins/library/official/notifier/). ## Installation steps @@ -87,6 +93,7 @@ The plugin assumes that the table schema is already defined in the database, as 2. **Install required Python packages**: ```bash + influxdb3 install package "influxdata-plugin-utils>=0.3.0" influxdb3 install package requests ``` 3. **Optional**: For notifications, install and configure the [influxdata/notifier plugin](/influxdb3/version/plugins/library/official/notifier/) @@ -102,7 +109,7 @@ influxdb3 create trigger \ --database mydb \ --path "gh:influxdata/threshold_deadman_checks/threshold_deadman_checks_plugin.py" \ --trigger-spec "every:10m" \ - --trigger-arguments "measurement=cpu,senders=slack,field_aggregation_values=temp:avg@>=30-ERROR,window=10m,trigger_count=3,deadman_check=true,slack_webhook_url=$SLACK_WEBHOOK_URL" \ + --trigger-arguments "measurement=cpu,senders=slack,field_aggregation_values=temp:avg@>=30-ERROR,window=10min,trigger_count=3,deadman_check=true,slack_webhook_url=$SLACK_WEBHOOK_URL" \ threshold_scheduler ``` Set `SLACK_WEBHOOK_URL` to your Slack incoming webhook URL. @@ -148,7 +155,7 @@ influxdb3 create trigger \ --database sensors \ --path "gh:influxdata/threshold_deadman_checks/threshold_deadman_checks_plugin.py" \ --trigger-spec "every:5m" \ - --trigger-arguments "measurement=heartbeat,senders=slack,window=5m,deadman_check=true,slack_webhook_url=$SLACK_WEBHOOK_URL" \ + --trigger-arguments "measurement=heartbeat,senders=slack,window=5min,deadman_check=true,slack_webhook_url=$SLACK_WEBHOOK_URL" \ heartbeat_monitor influxdb3 enable trigger --database sensors heartbeat_monitor @@ -173,7 +180,7 @@ influxdb3 create trigger \ --database sensors \ --path "gh:influxdata/threshold_deadman_checks/threshold_deadman_checks_plugin.py" \ --trigger-spec "every:15m" \ - --trigger-arguments "measurement=heartbeat,senders=sms,window=10m,deadman_check=true,trigger_count=2,twilio_from_number=+1234567890,twilio_to_number=+0987654321,notification_deadman_text=CRITICAL: No heartbeat data from \$table between \$time_from and \$time_to" \ + --trigger-arguments "measurement=heartbeat,senders=sms,window=10min,deadman_check=true,trigger_count=2,twilio_from_number=+1234567890,twilio_to_number=+0987654321,notification_deadman_text=CRITICAL: No heartbeat data from \$table between \$time_from and \$time_to" \ heartbeat_monitor ``` ### Multi-level threshold monitoring @@ -185,7 +192,7 @@ influxdb3 create trigger \ --database monitoring \ --path "gh:influxdata/threshold_deadman_checks/threshold_deadman_checks_plugin.py" \ --trigger-spec "every:5m" \ - --trigger-arguments "measurement=system_metrics,senders=slack.discord,field_aggregation_values='cpu_usage:avg@>=80-WARN cpu_usage:avg@>=95-ERROR memory_usage:max@>=90-WARN',window=5m,interval=1min,trigger_count=3,slack_webhook_url=$SLACK_WEBHOOK_URL,discord_webhook_url=$DISCORD_WEBHOOK_URL" \ + --trigger-arguments "measurement=system_metrics,senders=slack.discord,field_aggregation_values='cpu_usage:avg@>=80-WARN cpu_usage:avg@>=95-ERROR memory_usage:max@>=90-WARN',window=5min,interval=1min,trigger_count=3,slack_webhook_url=$SLACK_WEBHOOK_URL,discord_webhook_url=$DISCORD_WEBHOOK_URL" \ system_threshold_monitor ``` Set `SLACK_WEBHOOK_URL` and `DISCORD_WEBHOOK_URL` to your webhook URLs. @@ -213,7 +220,7 @@ influxdb3 create trigger \ --database comprehensive \ --path "gh:influxdata/threshold_deadman_checks/threshold_deadman_checks_plugin.py" \ --trigger-spec "every:10m" \ - --trigger-arguments "measurement=temperature_sensors,senders=whatsapp,field_aggregation_values='temperature:avg@>=35-WARN temperature:max@>=40-ERROR',window=15m,deadman_check=true,trigger_count=2,twilio_from_number=+1234567890,twilio_to_number=+0987654321" \ + --trigger-arguments "measurement=temperature_sensors,senders=whatsapp,field_aggregation_values='temperature:avg@>=35-WARN temperature:max@>=40-ERROR',window=15min,deadman_check=true,trigger_count=2,twilio_from_number=+1234567890,twilio_to_number=+0987654321" \ comprehensive_sensor_monitor ``` @@ -224,6 +231,9 @@ influxdb3 create trigger \ - `threshold_deadman_checks_plugin.py`: The main plugin code containing handlers for scheduled and data write triggers - `threshold_deadman_config_scheduler.toml`: Example TOML configuration for scheduled triggers - `threshold_deadman_config_data_writes.toml`: Example TOML configuration for data write triggers +- `test_threshold_deadman_checks.py`: Pytest suite (59 tests, runs without a live {{% product-name %}} server) +- `requirements.txt`: Runtime dependencies (`influxdata-plugin-utils>=0.3.0`, `requests`) +- `requirements-dev.txt`: Development dependencies (`pytest`) ### Logging @@ -252,7 +262,7 @@ Handles real-time threshold monitoring on data writes. Evaluates incoming data a #### Issue: False positive alerts -**Solution**: Increase `trigger_count` to require more consecutive failures. Adjust threshold values to be less sensitive. Consider longer aggregation intervals for noisy data. +**Solution**: Increase `trigger_count` to require more consecutive breaches. In scheduled mode every `DATE_BIN` bin of the window counts as a breach, so keep `window`, `interval`, and `trigger_count` aligned. Adjust threshold values to be less sensitive. Consider longer aggregation intervals for noisy data. #### Issue: Missing deadman alerts @@ -316,12 +326,14 @@ Handles real-time threshold monitoring on data writes. Evaluates incoming data a - `$op_sym`: Operator symbol - `$compare_val`: Threshold value - `$actual`: Actual field value +- `$trigger_count`: Consecutive matches required before alerting +- `$row`: Unique identifier ### Row identification -The `row` variable uniquely identifies alert contexts using format: `measurement:level:tag1=value1:tag2=value2` +The `row` variable uniquely identifies alert contexts using format: `measurement:field[:aggregation]:level:tag1=value1:tag2=value2` (`aggregation` is present for scheduled threshold checks only). Tags without a value are omitted. -This ensures trigger counts are maintained independently for each unique combination of measurement, severity level, and tag values. +Trigger counts are maintained independently for each unique combination of measurement, field, aggregation, severity level, tag values, **and the condition's operator and threshold** — two conditions that differ only by threshold never share a count. ## Report an issue @@ -330,4 +342,6 @@ For plugin issues, see the Plugins repository [issues page](https://github.com/i ## Find support for {{% product-name %}} The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. -For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. \ No newline at end of file +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/content/shared/influxdb3-plugins/plugins-library/official/valuecounter.md b/content/shared/influxdb3-plugins/plugins-library/official/valuecounter.md new file mode 100644 index 0000000000..250ccb75e8 --- /dev/null +++ b/content/shared/influxdb3-plugins/plugins-library/official/valuecounter.md @@ -0,0 +1,266 @@ + + +The Value Counter Plugin counts how many times each unique value of one or more watched fields appears in a source table, then writes the result as a rollup measurement with field keys of the form `_=i`. It is a direct port of the [Telegraf `valuecounter` aggregator](https://github.com/influxdata/telegraf/tree/master/plugins/aggregators/valuecounter) to the {{% product-name %}} Processing Engine. The plugin ships with two independent modes — a data-write (WAL) trigger that accumulates counts as rows arrive, and a scheduled trigger that queries the source table on every fire and aggregates the window since the previous successful fire. + +## Configuration + +Plugin parameters may be specified as key-value pairs in the `--trigger-arguments` flag (CLI) or in the `trigger_arguments` field (API) when creating a trigger. Some plugins support TOML configuration files, which can be specified using the plugin's `config_file_path` parameter. + +If a plugin supports multiple trigger specifications, some parameters may depend on the trigger specification that you use. + +### Plugin metadata + +This plugin includes a JSON metadata schema in its docstring that defines supported trigger types and configuration parameters. This metadata enables the [InfluxDB 3 Explorer](https://docs.influxdata.com/influxdb3/explorer/) UI to display and configure the plugin. + +### Required parameters + +| Parameter | Type | Default | Description | +|-----------|--------|---------------------------|--------------------------------------------------------------------------------------------------------------| +| `fields` | string | required | Space-separated list of field names whose unique values are counted (for example, `status method`) | +| `table` | string | required (scheduled only) | Source table to query. Mode A derives the source table from the trigger spec and rejects this key | + +### Data write trigger parameters + +| Parameter | Type | Default | Description | +|------------------|---------|------------------|----------------------------------------------------------------------------------------------------------------------------------| +| `period_seconds` | integer | `60` | Emission period in seconds, at least `1`. Cache TTL is set to `2 * period_seconds` | +| `period` | string | `60s` | Emission period as a duration, at least `1s`. Units: `s`, `min`, `h`, `d`, `w`. Overridden by `period_seconds` when both are set | +| `output_suffix` | string | `_valuecounts` | Suffix appended to the source measurement name for rollup output. Must be non-empty | +| `dest_database` | string | trigger's own DB | Optional database to write rollups to via `write_sync_to_db` | + +### Scheduled trigger parameters + +| Parameter | Type | Default | Description | +|-----------------|--------|------------------|--------------------------------------------------------------------------------------| +| `output_suffix` | string | `_valuecounts` | Suffix appended to the source measurement name for rollup output. Must be non-empty | +| `dest_database` | string | trigger's own DB | Optional database to write rollups to via `write_sync_to_db` | + +Mode B is drift-based: the trigger's `every:` spec is the only cadence knob. The plugin queries the window between the previous successful fire and the current `call_time`, so it self-aligns to whatever the scheduler delivers. Mode B rejects `period_seconds` and `period` keys. + +### TOML configuration + +| Parameter | Type | Default | Description | +|--------------------|--------|---------|----------------------------------------------------------------------------------| +| `config_file_path` | string | none | TOML config file path relative to `PLUGIN_DIR` (required for TOML configuration) | + +*To use a TOML configuration file, set the `PLUGIN_DIR` environment variable and specify the `config_file_path` in the trigger arguments.* This is in addition to the `--plugin-dir` flag when starting {{% product-name %}}. Relative paths are resolved against `PLUGIN_DIR`, then `INFLUXDB3_PLUGIN_DIR`, then the parent of `VIRTUAL_ENV`. + +If `config_file_path` is set, no other inline arguments may be set on the trigger. The TOML accepts the same keys as inline arguments. + +#### Example TOML configuration + +- [valuecounter_config.toml](https://github.com/influxdata/influxdb3_plugins/blob/master/influxdata/valuecounter/valuecounter_config.toml) — shows both Mode A (WAL) and Mode B (scheduled) shapes + +For more information on using TOML configuration files, see the Using TOML Configuration Files section in the [influxdb3_plugins/README.md](https://github.com/influxdata/influxdb3_plugins/blob/master/README.md). + +## Schema requirement + +The plugin reads tag column names from `information_schema.columns` for the source table; the source measurement must exist before the trigger fires. The watched fields named in `fields` must be of categorical type (string or low-cardinality numeric). The rollup measurement is created on first emission; do not pre-create it with a conflicting schema. + +The rollup measurement name is always ``. The default suffix (`_valuecounts`) keeps the rollup separate from the source table so the WAL trigger does not feed itself. An empty `output_suffix` is rejected at trigger creation. + +## Software Requirements + +- **{{% product-name %}}**: with the Processing Engine enabled +- **Python packages**: `influxdata-plugin-utils>=0.3.0` + +## Installation steps + +1. Start {{% product-name %}} with the Processing Engine enabled (`--plugin-dir /path/to/plugins`): + + ```bash + influxdb3 serve \ + --node-id node0 \ + --object-store file \ + --data-dir ~/.influxdb3 \ + --plugin-dir ~/.plugins + ``` +2. Install required Python packages: + + ```bash + influxdb3 install package "influxdata-plugin-utils>=0.3.0" + ``` +## Trigger setup + +> **Do not install both Mode A and Mode B triggers against the same source table.** The two modes share no state and will write duplicate rollup rows. Choose one mode per source table. + +### Data write trigger (Mode A) + +Count unique field values as rows are written. Emission is period-gated by the plugin's own bookkeeping — the trigger emits when at least `period_seconds` have elapsed since the last successful emission for a given series. + +```bash +influxdb3 create trigger \ + --database mydb \ + --path "gh:influxdata/valuecounter/valuecounter.py" \ + --trigger-spec "table:http_requests" \ + --trigger-arguments "fields=status method,period_seconds=60" \ + --error-behavior log \ + http_status_counter +``` +Mode A's `--trigger-spec` must name a single table (`table:`). `all_tables` is not supported because one `fields` list cannot apply to heterogeneous tables. + +### Scheduled trigger (Mode B) + +Query the source table on a fixed cadence and aggregate the window since the previous fire. Best when writes are bursty or sparse, or when a deterministic emission cadence matters more than per-write freshness. + +```bash +influxdb3 create trigger \ + --database mydb \ + --path "gh:influxdata/valuecounter/valuecounter.py" \ + --trigger-spec "every:60s" \ + --trigger-arguments "table=http_requests,fields=status method" \ + --error-behavior log \ + http_status_counter_scheduled +``` +The first fire after install (or after a server restart that wipes the cache) only establishes the cadence anchor and emits no rollup. The second fire emits a rollup covering the window from the first fire to the second. + +### Enable triggers + +```bash +influxdb3 enable trigger --database mydb http_status_counter +``` +## Example usage + +### Example 1: HTTP status code rollup (Mode A) + +Roll up HTTP status codes to per-minute counts per host and endpoint: + +```bash +# Create and enable the trigger +influxdb3 create trigger \ + --database web \ + --path "gh:influxdata/valuecounter/valuecounter.py" \ + --trigger-spec "table:http_requests" \ + --trigger-arguments "fields=status,period_seconds=60" \ + --error-behavior log \ + http_status_counter + +influxdb3 enable trigger --database web http_status_counter + +# Write some sample requests +influxdb3 write \ + --database web \ + "http_requests,host=web-1,endpoint=/api/users status=200i +http_requests,host=web-1,endpoint=/api/users status=200i +http_requests,host=web-1,endpoint=/api/users status=404i +http_requests,host=web-1,endpoint=/api/users status=500i" + +# After period_seconds elapses, write one more row to drive emission +sleep 60 +influxdb3 write \ + --database web \ + "http_requests,host=web-1,endpoint=/api/users status=200i" + +# Query the rollup +influxdb3 query \ + --database web \ + "SELECT * FROM http_requests_valuecounts WHERE time >= now() - 5m" +``` +**Expected output** + +```text +http_requests_valuecounts,host=web-1,endpoint=/api/users status_200=2i,status_404=1i,status_500=1i +``` +### Example 2: Log level counts (Mode B) + +Roll up application log levels per service over a fixed five-minute cadence: + +```bash +influxdb3 create trigger \ + --database observability \ + --path "gh:influxdata/valuecounter/valuecounter.py" \ + --trigger-spec "every:5m" \ + --trigger-arguments "table=app_logs,fields=level" \ + --error-behavior log \ + log_level_counter + +influxdb3 enable trigger --database observability log_level_counter +``` +Each fire produces one rollup row per `(service, env)` tag combination, with fields like `level_INFO=120i`, `level_WARN=8i`, `level_ERROR=2i`. + +### Example 3: Cross-database rollup (Mode B) + +Write rollups to a separate analytics database while leaving the source database untouched: + +```bash +influxdb3 create trigger \ + --database production \ + --path "gh:influxdata/valuecounter/valuecounter.py" \ + --trigger-spec "every:1m" \ + --trigger-arguments "table=payment_attempts,fields=result,dest_database=analytics" \ + --error-behavior log \ + payment_result_rollup +``` +The rollup measurement `payment_attempts_valuecounts` is written to the `analytics` database. + +## Code overview + +### Files + +- `valuecounter.py`: The main plugin code containing `process_writes` (Mode A) and `process_scheduled_call` (Mode B) +- `valuecounter_config.toml`: Example TOML configuration with both Mode A and Mode B shapes +- `test_valuecounter.py`: Pytest suite (79 tests, runs without a live {{% product-name %}} server) +- `requirements.txt`: Runtime dependencies (`influxdata-plugin-utils>=0.3.0`) +- `requirements-dev.txt`: Development dependencies (`pytest`) + +### Logging + +Logs are stored in the trigger's database in the `system.processing_engine_logs` table. To view logs: + +```bash +influxdb3 query --database YOUR_DATABASE "SELECT * FROM system.processing_engine_logs WHERE trigger_name = 'your_trigger_name'" +``` +Every log line is prefixed with a per-fire `task_id` (eight hex characters) so log records from a single trigger fire can be correlated. + +### Main functions + +#### `process_writes(influxdb3_local, table_batches, args)` + +Handles Mode A WAL-trigger invocations. Accumulates per-value counts into trigger-local cache entries, period-gates emission per series, and writes one rollup row per series via `write_sync` (or `write_sync_to_db` if `dest_database` is set). Uses a snapshot-subtract pattern on emit so concurrent increments arriving during a write are preserved across emissions. + +#### `process_scheduled_call(influxdb3_local, call_time, args)` + +Handles Mode B scheduled invocations. On the first fire, only the cadence anchor is established (no rollup emitted). On subsequent fires, the plugin queries the source table for the window since the previous successful fire, groups results by tag set and watched field, and writes one rollup row per series. The anchor advances only on a successful write or a genuinely empty window — write failures cause the next fire to cover a larger window and naturally retry. + +## Troubleshooting + +### Common issues + +#### Issue: No rollup rows appear + +**Solution (Mode A)**: Mode A emission is tied to write traffic. If no rows are written for at least `period_seconds`, emission stops. Confirm the source table is receiving writes and that `period_seconds` is shorter than the inter-write interval. Check `system.processing_engine_logs` for the trigger. + +**Solution (Mode B)**: The first fire after install (or server restart) establishes the cadence anchor and emits no rollup — look for a log line containing `vc-scheduled: first fire`. Subsequent fires emit normally. + +#### Issue: Rollup measurement schema explosion + +**Solution**: The plugin creates one rollup field per unique value of each watched field. Configuring `fields` against a high-cardinality column (for example, `user_id`) creates one column per distinct value and stresses the Parquet schema. Use only on bounded-cardinality categorical fields such as HTTP status, log level, or A/B variant. + +#### Issue: Field key contains an invalid character + +**Solution**: Watched-field values that contain characters other than `[A-Za-z0-9_]` are sanitized — every disallowed character is replaced with `_`. Two distinct raw values that sanitize to the same key (for example, `"not found"` and `"not_found"`) are summed into one rollup field. When this happens, the plugin emits a warning to `system.processing_engine_logs` listing the colliding raw values. + +#### Issue: Source table tag list changed and rollups are missing tags + +**Solution**: The plugin caches tag-column names from `information_schema.columns` for one hour per source table. Schema changes are picked up within an hour. To force a refresh, delete the cache key `shared:tags:`. + +#### Issue: Both modes installed against the same source table + +**Solution**: The two modes share no state and will both emit rollups, producing duplicate rows. Drop one trigger and use the other exclusively. + +### Operational notes + +- Do not enable `--run-asynchronous` on a Mode A trigger. The plugin assumes serialized per-trigger invocations; concurrent fires race on the per-series counts dict and the in-memory index. +- Re-running a fire (for example, via `influxdb3 test schedule_plugin`) overwrites the previous emission idempotently because every rollup row uses the same timestamp for a given fire (Mode A: `now_ns`; Mode B: `call_time_ns`). Combined with {{% product-name %}}'s last-write-wins semantics, replays do not produce duplicate rollup rows. +- Server restart wipes the trigger-local cache. Mode A loses any accumulated-but-not-yet-emitted counts; Mode B's next fire is treated as a "first fire" and skips one period of source data. Both behaviors are documented above. + +## Report an issue + +For plugin issues, see the Plugins repository [issues page](https://github.com/influxdata/influxdb3_plugins/issues). + +## Find support for {{% product-name %}} + +The [InfluxDB Discord server](https://discord.gg/9zaNCW2PRT) is the best place to find support for InfluxDB 3 Core and InfluxDB 3 Enterprise. +For other InfluxDB versions, see the [Support and feedback](#bug-reports-and-feedback) options. + + diff --git a/helper-scripts/influxdb3-plugins/README.md b/helper-scripts/influxdb3-plugins/README.md index 726ce77551..ae33862fd9 100644 --- a/helper-scripts/influxdb3-plugins/README.md +++ b/helper-scripts/influxdb3-plugins/README.md @@ -98,9 +98,9 @@ is the worked example. no pull request. 3. Scaffold Core and Enterprise stubs for any plugin that lacks them. An existing stub is never opened for writing. -4. Transform the README of each plugin in the `plugins:` map of - `docs_mapping.yaml` and merge it into the generated region of its shared - page. +4. Transform the README of every discovered plugin and merge it into the + generated region of its shared page. `docs_mapping.yaml` supplies only + exceptions to the standard upstream README and shared-page paths. 5. Report one row per plugin to the step summary, and set the `needs_attention` output. @@ -149,9 +149,9 @@ identical from the registry index, so resolving one is a human decision. `docs_mapping.yaml` carries four things the registry index cannot supply. -- `plugins`: the README source and shared-page target for each plugin whose - prose is transformed. A plugin absent here still gets a data file entry and - product stubs. +- `plugins`: exceptional README source and shared-page target paths. Plugins + absent here use the standard `influxdata//README.md` upstream path and + a hyphenated shared-page filename. - `overrides`: per-plugin product stub slugs, where the stub slug differs from the name-derived shared-page slug. `mad_check` is the only current case: its shared page is `mad-check.md` and its stubs are `mad-anomaly-detection.md`. @@ -167,7 +167,7 @@ The transform reads plugin READMEs from a checkout of `influxdb3_plugins` at ```bash git clone --depth 1 https://github.com/influxdata/influxdb3_plugins.git \ - .ext/influxdb3_plugins + ../.ext/influxdb3_plugins ``` Without that checkout, discovery, the data file, and stub scaffolding still @@ -243,6 +243,6 @@ Two things still need a human: 1. The scaffolded stubs carry baseline tags (`plugins`, `processing engine`, `python`, `official`). Editorial tags are not derivable from the registry. -2. To publish the plugin's README prose as a shared page, add a `plugins:` - entry to `docs_mapping.yaml` pointing at its upstream README and its shared - page target. +2. If the plugin's README or shared-page path differs from the standard + convention, add a `plugins:` entry to `docs_mapping.yaml` with the + exceptional paths. diff --git a/helper-scripts/influxdb3-plugins/port_to_docs.js b/helper-scripts/influxdb3-plugins/port_to_docs.js index 2eb9da59bd..3a84475f69 100644 --- a/helper-scripts/influxdb3-plugins/port_to_docs.js +++ b/helper-scripts/influxdb3-plugins/port_to_docs.js @@ -268,6 +268,36 @@ function fixCodeBlockFormatting(content) { return content; } +/** + * Upstream READMEs use ellipses in abbreviated JSON request and response + * examples. They are deliberately not parseable JSON, so exempt only those + * fences from the repository's code-block parser. + */ +function exemptAbbreviatedJsonExamples(content) { + return content.replace( + /```json([^\n]*)\n([\s\S]*?)```/g, + (match, attributes, body) => { + if (attributes.includes('lint=') || !body.includes('...')) return match; + return `\`\`\`json${attributes} {lint="false"}\n${body}\`\`\``; + } + ); +} + +/** + * Markdown list indentation uses spaces. Normalize tabs that appear only in + * a list prefix without changing tabs in code fences or prose. + */ +function normalizeListIndentation(content) { + return content.replace( + /^([ \t]+)([-*+] )/gm, + (match, indent, marker) => `${indent.replaceAll('\t', ' ')}${marker}` + ); +} + +function exemptGeneratedContentFromVale(content) { + return `\n${content.trim()}\n`; +} + const GENERATED_REGION_BEGIN = ''; const GENERATED_REGION_END = ''; @@ -328,6 +358,8 @@ function transformContent(content, pluginName) { content = enhanceOpeningParagraph(content); content = extractStyleAttributes(content); content = fixCodeBlockFormatting(content); + content = exemptAbbreviatedJsonExamples(content); + content = normalizeListIndentation(content); // Add logging section content = addLoggingSection(content); @@ -335,7 +367,7 @@ function transformContent(content, pluginName) { // Replace support section content = replaceSupportSection(content); - return content; + return exemptGeneratedContentFromVale(content); } /** @@ -430,6 +462,24 @@ function shouldRunDiscovery(pluginArg) { const SHARED_OFFICIAL_DIR = '../../content/shared/influxdb3-plugins/plugins-library/official'; +const UPSTREAM_OFFICIAL_DIR = '../../../.ext/influxdb3_plugins/influxdata'; + +/** + * Return the README and shared-page paths for a discovered official plugin. + * + * Most plugins follow this convention. `docs_mapping.yaml` remains the place + * for the exceptional source or target paths that need an explicit override. + */ +function mappingForDiscoveredPlugin(plugin, configPlugins) { + if (configPlugins[plugin.name]) { + return configPlugins[plugin.name]; + } + + return { + source: `${UPSTREAM_OFFICIAL_DIR}/${plugin.name}/README.md`, + target: `${SHARED_OFFICIAL_DIR}/${plugin.slug}.md`, + }; +} /** * Shared pages left behind by a plugin that is no longer in the registry. @@ -646,21 +696,20 @@ async function main() { process.exit(1); } - // Discover official plugins from the registry index. The transform loop - // below still walks docs_mapping.yaml's plugins map for README content - // until Task 5 lands stub scaffolding for plugins that aren't mapped yet, - // but data/influxdb3_plugins.yml is fully registry-driven and regenerated - // every run regardless of --plugin. + // Discover official plugins from the registry index. A full sync transforms + // every discovered plugin README. docs_mapping.yaml supplies exceptions to + // the conventional README and shared-page paths, rather than a roster that + // can omit newly published plugins. // Each artifact the run touches appends a `{ plugin, status, detail }` // entry. `main` collapses them to one row per plugin before reporting. const artifactResults = []; + let discovered = null; if (shouldRunDiscovery(options.plugin)) { console.log('Discovering official plugins from the registry index...'); // A registry fetch failure is a bad afternoon on the network, not drift. // It must not fail a nightly run, so it is reported as a skip. - let discovered = null; try { const indexJson = await fetchRegistryIndex(); const parsed = parseRegistryIndex(indexJson, { @@ -669,10 +718,6 @@ async function main() { }); discovered = parsed.plugins; - const { mapped, unmapped } = partitionDiscoveredPlugins( - discovered, - Object.keys(config.plugins) - ); console.log( `Discovered ${discovered.length} official plugin(s) in the registry.` ); @@ -681,11 +726,11 @@ async function main() { `Excluded by docs_mapping.yaml: ${parsed.excluded.join(', ')}` ); } - console.log(` Mapped (transformed below): ${mapped.length}`); - console.log(` Not yet mapped: ${unmapped.length}`); - if (unmapped.length > 0) { - console.log(` ${unmapped.map((plugin) => plugin.name).join(', ')}`); - } + const { mapped } = partitionDiscoveredPlugins( + discovered, + Object.keys(config.plugins) + ); + console.log(` Explicit path overrides: ${mapped.length}`); } catch (error) { console.warn(`⚠️ Could not read the registry index: ${error.message}`); artifactResults.push({ @@ -738,8 +783,11 @@ async function main() { console.log(''); } - // Process plugins - const { selected: pluginsToProcess, unknown } = selectPlugins( + // Process plugins. A successful full discovery is authoritative: every + // official plugin gets a conventional mapping unless configuration overrides + // it. If discovery was unavailable, retain the configured fallback so an + // upstream-network failure does not prevent known pages from refreshing. + const { selected: configuredPlugins, unknown } = selectPlugins( config.plugins, options.plugin ); @@ -749,6 +797,13 @@ async function main() { process.exit(1); } + const pluginsToProcess = discovered + ? discovered.map((plugin) => [ + plugin.name, + mappingForDiscoveredPlugin(plugin, config.plugins), + ]) + : configuredPlugins; + console.log( `${options.dryRun ? 'DRY RUN: ' : ''}Processing ${pluginsToProcess.length} plugin(s)...\n` ); @@ -797,4 +852,5 @@ export { mergeGeneratedRegion, selectPlugins, shouldRunDiscovery, + mappingForDiscoveredPlugin, }; diff --git a/helper-scripts/influxdb3-plugins/reporting.js b/helper-scripts/influxdb3-plugins/reporting.js index bcffea9c62..d8683829fa 100644 --- a/helper-scripts/influxdb3-plugins/reporting.js +++ b/helper-scripts/influxdb3-plugins/reporting.js @@ -77,7 +77,12 @@ function collapseByPlugin(artifactResults) { // Files that live alongside the generated plugin pages but do not describe a // plugin. Without this, every run reports them as removed plugins. -const NON_PLUGIN_PAGES = new Set(['_index.md', 'CLAUDE.md', 'README.md', 'AGENTS.md']); +const NON_PLUGIN_PAGES = new Set([ + '_index.md', + 'AGENTS.md', + 'CLAUDE.md', + 'README.md', +]); /** * Shared pages that no longer have a plugin in the registry index. diff --git a/helper-scripts/influxdb3-plugins/stub-template.js b/helper-scripts/influxdb3-plugins/stub-template.js index b3b14ae5c6..4a1ae946f7 100644 --- a/helper-scripts/influxdb3-plugins/stub-template.js +++ b/helper-scripts/influxdb3-plugins/stub-template.js @@ -11,10 +11,17 @@ const PRODUCTS = { }; const BASE_TAGS = ['plugins', 'processing engine', 'python', 'official']; +const INITIALISMS = new Map([['nws', 'NWS']]); function sentenceCase(name) { - const words = name.split('_').join(' '); - return words[0].toUpperCase() + words.slice(1); + return name + .split('_') + .map( + (word, index) => + INITIALISMS.get(word.toLowerCase()) ?? + (index === 0 ? word[0].toUpperCase() + word.slice(1) : word) + ) + .join(' '); } export function stubPath(plugin, product) { @@ -24,9 +31,10 @@ export function stubPath(plugin, product) { export function renderStub(plugin, product) { const { menuKey, tagPrefix } = PRODUCTS[product]; const label = sentenceCase(plugin.name); + const title = label.endsWith(' plugin') ? label : `${label} plugin`; return `--- -title: ${label} plugin +title: ${title} description: ${plugin.description} menu: ${menuKey}: @@ -35,7 +43,7 @@ menu: weight: 100 ${tagPrefix}/tags: [${BASE_TAGS.join(', ')}] related: - - ${plugin.repository}, ${label} plugin on GitHub + - ${plugin.repository}, ${title} on GitHub source: /shared/influxdb3-plugins/plugins-library/official/${plugin.slug}.md canonical: self --- diff --git a/helper-scripts/influxdb3-plugins/test/region-writer.test.js b/helper-scripts/influxdb3-plugins/test/region-writer.test.js index be5bd88ca6..999b8f4af8 100644 --- a/helper-scripts/influxdb3-plugins/test/region-writer.test.js +++ b/helper-scripts/influxdb3-plugins/test/region-writer.test.js @@ -1,6 +1,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { mergeGeneratedRegion } from '../port_to_docs.js'; +import { mergeGeneratedRegion, transformContent } from '../port_to_docs.js'; test('wraps content in markers when the target has none yet', () => { const result = mergeGeneratedRegion(null, 'Generated body text.'); @@ -53,3 +53,26 @@ test('preserves hand-owned content after the generated region', () => { ); assert.equal(result.error, undefined); }); + +test('exempts abbreviated JSON examples from code-block parsing', () => { + const transformed = transformContent( + '# Example\n\n```json\n{"items": [{"value": 1}, ...]}\n```', + 'example' + ); + + assert.match(transformed, /```json \{lint="false"\}/); +}); + +test('normalizes tabs used to indent Markdown list items', () => { + const transformed = transformContent('# Example\n\n \t- Item', 'example'); + + assert.match(transformed, /^- Item$/m); + assert.doesNotMatch(transformed, /^\s*\t/m); +}); + +test('exempts generated upstream prose from Vale', () => { + const transformed = transformContent('# Example', 'example'); + + assert.match(transformed, /^$/m); + assert.match(transformed, /$/); +}); diff --git a/helper-scripts/influxdb3-plugins/test/reporting.test.js b/helper-scripts/influxdb3-plugins/test/reporting.test.js index 74477544af..613024fc1a 100644 --- a/helper-scripts/influxdb3-plugins/test/reporting.test.js +++ b/helper-scripts/influxdb3-plugins/test/reporting.test.js @@ -219,7 +219,10 @@ test('matches the shared page on slug, not on the product stub slug', () => { test('ignores the section index and the writer guidance file', () => { assert.deepEqual( - detectRemovedPlugins([], ['_index.md', 'CLAUDE.md', 'README.md']), + detectRemovedPlugins( + [], + ['_index.md', 'AGENTS.md', 'CLAUDE.md', 'README.md'] + ), [] ); }); diff --git a/helper-scripts/influxdb3-plugins/test/stubs.test.js b/helper-scripts/influxdb3-plugins/test/stubs.test.js index 5422dfde79..ab1832f0e5 100644 --- a/helper-scripts/influxdb3-plugins/test/stubs.test.js +++ b/helper-scripts/influxdb3-plugins/test/stubs.test.js @@ -62,6 +62,33 @@ canonical: self ); }); +test('preserves known initialisms in generated titles and menu labels', () => { + const content = renderStub( + { + ...BASIC_TRANSFORMATION, + name: 'nws_weather', + slug: 'nws-weather', + stubSlug: 'nws-weather', + }, + 'enterprise' + ); + + assert.match(content, /^title: NWS weather plugin$/m); + assert.match(content, /^ name: NWS weather$/m); + assert.match(content, /NWS weather plugin on GitHub/); +}); + +test('does not repeat plugin in titles that already include it', () => { + const content = renderStub( + { ...BASIC_TRANSFORMATION, name: 'stock_plugin' }, + 'core' + ); + + assert.match(content, /^title: Stock plugin$/m); + assert.match(content, /Stock plugin on GitHub/); + assert.doesNotMatch(content, /plugin plugin/); +}); + const MAD_CHECK = { name: 'mad_check', slug: 'mad-check', diff --git a/helper-scripts/influxdb3-plugins/test/sync-results.test.js b/helper-scripts/influxdb3-plugins/test/sync-results.test.js index 87540adc7a..f483aad83e 100644 --- a/helper-scripts/influxdb3-plugins/test/sync-results.test.js +++ b/helper-scripts/influxdb3-plugins/test/sync-results.test.js @@ -7,6 +7,7 @@ import { processPlugin, selectPlugins, shouldRunDiscovery, + mappingForDiscoveredPlugin, } from '../port_to_docs.js'; const CONFIG_PLUGINS = { @@ -66,6 +67,29 @@ test('skips discovery when the run names specific plugins', () => { assert.equal(shouldRunDiscovery('notifier,state_change'), false); }); +test('derives conventional README and shared-page paths for a new plugin', () => { + const mapping = mappingForDiscoveredPlugin( + { name: 'nws_weather', slug: 'nws-weather' }, + CONFIG_PLUGINS + ); + + assert.deepEqual(mapping, { + source: '../../../.ext/influxdb3_plugins/influxdata/nws_weather/README.md', + target: + '../../content/shared/influxdb3-plugins/plugins-library/official/nws-weather.md', + }); +}); + +test('uses an explicit mapping when a plugin needs nonstandard paths', () => { + const customMapping = { source: 'custom-readme', target: 'custom-page' }; + const mapping = mappingForDiscoveredPlugin( + { name: 'notifier', slug: 'notifier' }, + { notifier: customMapping } + ); + + assert.equal(mapping, customMapping); +}); + test('ignores empty entries from a trailing or doubled comma', () => { const { selected, unknown } = selectPlugins(CONFIG_PLUGINS, 'notifier,,'); From 46c5024beea54af1cc435e90abde52dec75cd487 Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Tue, 8 Sep 2026 20:59:53 -0500 Subject: [PATCH 6/8] ci(plugins): schedule documentation sync --- .github/workflows/sync-plugins.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-plugins.yml b/.github/workflows/sync-plugins.yml index 919c887eac..e3ea1e058a 100644 --- a/.github/workflows/sync-plugins.yml +++ b/.github/workflows/sync-plugins.yml @@ -1,8 +1,9 @@ name: Sync InfluxDB 3 plugin documentation -# The daily cron is added once the library backfill lands, so the schedule -# never runs against a knowingly incomplete plugin library. on: + schedule: + # Run daily at 06:00 UTC. + - cron: '0 6 * * *' workflow_dispatch: inputs: plugins: From 8bf1bf8e2e25350eab8798ffa30a7cf3ea8c231e Mon Sep 17 00:00:00 2001 From: Jason Stirnaman Date: Tue, 8 Sep 2026 22:09:28 -0500 Subject: [PATCH 7/8] feat(agents): add changed-file verification What changed: - Compact canonical agent instructions and skill entrypoints, with routed references. - Add a changed-file verifier and enforced instruction size limits. Why: - Reduce baseline agent context while retaining targeted operational guidance. Impact: - Agents can plan focused manual validation without re-running commit hooks. Verification: - yarn test:verify-changed - yarn test:agent-instruction-limits - yarn build:agent:instructions - yarn validate:agent-instructions - git diff --check --- .agents/instructions/content.md | 231 +----- .agents/instructions/layouts.md | 148 +--- .agents/skills/ai-visibility/SKILL.md | 244 +------ .../ai-visibility/references/discovery.md | 4 + .../references/markdown-artifacts.md | 4 + .../references/structured-data.md | 4 + .agents/skills/content-editing/SKILL.md | 608 +--------------- .../references/fact-checking.md | 5 + .../references/shared-content.md | 5 + .agents/skills/cypress-e2e-testing/SKILL.md | 431 +---------- .../references/content-mapping.md | 4 + .../references/failures.md | 4 + .../cypress-e2e-testing/references/runner.md | 4 + .agents/skills/docs-cli-workflow/SKILL.md | 191 +---- .../references/cli-examples.md | 5 + .agents/skills/docs-testing/SKILL.md | 247 +------ .../docs-testing/references/agent-assets.md | 5 + .../docs-testing/references/content-checks.md | 5 + .../references/specialized-checks.md | 5 + .agents/skills/hugo-template-dev/SKILL.md | 678 +----------------- .../references/product-data.md | 4 + .../references/runtime-testing.md | 5 + .../references/shortcodes.md | 4 + .agents/skills/influxdb3-test-setup/SKILL.md | 287 +------- .../influxdb3-test-setup/references/core.md | 4 + .../references/enterprise.md | 4 + .../references/environment.md | 4 + .agents/skills/vale-linting/SKILL.md | 444 +----------- .../vale-linting/references/configuration.md | 5 + .../vale-linting/references/running-vale.md | 4 + .../vale-linting/references/vocabulary.md | 4 + .agents/skills/vale-rule-config/SKILL.md | 521 +------------- .../vale-rule-config/references/regex.md | 4 + .../vale-rule-config/references/rule-types.md | 4 + .../vale-rule-config/references/testing.md | 4 + .claude/rules/content.md | 231 +----- .claude/rules/layouts.md | 148 +--- .github/instructions/content.instructions.md | 231 +----- .github/instructions/layouts.instructions.md | 148 +--- AGENTS.md | 148 ++-- content/AGENTS.md | 231 +----- .../agent-instruction-limits.test.mjs | 34 + helper-scripts/agent-instruction-limits.js | 21 + helper-scripts/validate-agent-instructions.js | 28 + layouts/AGENTS.md | 148 +--- package.json | 3 + scripts/__tests__/verify-changed.test.mjs | 68 ++ scripts/verify-changed.mjs | 188 +++++ 48 files changed, 806 insertions(+), 4955 deletions(-) create mode 100644 .agents/skills/ai-visibility/references/discovery.md create mode 100644 .agents/skills/ai-visibility/references/markdown-artifacts.md create mode 100644 .agents/skills/ai-visibility/references/structured-data.md create mode 100644 .agents/skills/content-editing/references/fact-checking.md create mode 100644 .agents/skills/content-editing/references/shared-content.md create mode 100644 .agents/skills/cypress-e2e-testing/references/content-mapping.md create mode 100644 .agents/skills/cypress-e2e-testing/references/failures.md create mode 100644 .agents/skills/cypress-e2e-testing/references/runner.md create mode 100644 .agents/skills/docs-cli-workflow/references/cli-examples.md create mode 100644 .agents/skills/docs-testing/references/agent-assets.md create mode 100644 .agents/skills/docs-testing/references/content-checks.md create mode 100644 .agents/skills/docs-testing/references/specialized-checks.md create mode 100644 .agents/skills/hugo-template-dev/references/product-data.md create mode 100644 .agents/skills/hugo-template-dev/references/runtime-testing.md create mode 100644 .agents/skills/hugo-template-dev/references/shortcodes.md create mode 100644 .agents/skills/influxdb3-test-setup/references/core.md create mode 100644 .agents/skills/influxdb3-test-setup/references/enterprise.md create mode 100644 .agents/skills/influxdb3-test-setup/references/environment.md create mode 100644 .agents/skills/vale-linting/references/configuration.md create mode 100644 .agents/skills/vale-linting/references/running-vale.md create mode 100644 .agents/skills/vale-linting/references/vocabulary.md create mode 100644 .agents/skills/vale-rule-config/references/regex.md create mode 100644 .agents/skills/vale-rule-config/references/rule-types.md create mode 100644 .agents/skills/vale-rule-config/references/testing.md create mode 100644 helper-scripts/__tests__/agent-instruction-limits.test.mjs create mode 100644 helper-scripts/agent-instruction-limits.js create mode 100644 scripts/__tests__/verify-changed.test.mjs create mode 100644 scripts/verify-changed.mjs diff --git a/.agents/instructions/content.md b/.agents/instructions/content.md index e533bf4928..5b6401a0ef 100644 --- a/.agents/instructions/content.md +++ b/.agents/instructions/content.md @@ -5,210 +5,27 @@ paths: - "content/**/*.md" --- -# Content File Guidelines - -**Frontmatter reference**: [DOCS-FRONTMATTER.md](../../DOCS-FRONTMATTER.md) -**Shortcodes reference**: [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) -**Working examples**: [content/example.md](../../content/example.md) - -**For complete content editing workflow**, see -[content-editing skill](../skills/content-editing/SKILL.md) which covers: - -- Creating and editing content with CLI tools -- Shared content management and testing -- Fact-checking with MCP server -- Complete validation workflows - -## CLI Tools for Content Workflow - -The unified `docs` CLI provides tools for content creation and editing. -For decision guidance on when to use CLI vs direct editing, see -[docs-cli-workflow skill](../skills/docs-cli-workflow/SKILL.md). - -### Creating New Content - -Use `docs create` for AI-assisted scaffolding: - -```bash -# Create from draft -docs create drafts/feature.md --products influxdb3_core - -# Create and open files in editor (non-blocking) -docs create drafts/feature.md --products influxdb3_core --open - -# Create and open, wait for editor (blocking) -docs create drafts/feature.md --products influxdb3_core --open --wait -``` - -### Editing Existing Content - -Use `docs edit` to quickly find and open content files: - -```bash -# Find and list files (no editor) -docs edit /influxdb3/core/admin/databases/ --list - -# Open in editor (non-blocking, exits immediately) -docs edit /influxdb3/core/admin/databases/ - -# Open and wait for editor (blocking, interactive) -docs edit /influxdb3/core/admin/databases/ --wait - -# Use specific editor -docs edit /influxdb3/core/admin/databases/ --editor nano -``` - -**Options:** - -- Both commands are **non-blocking by default** (agent-friendly) -- Use `--wait` for interactive editing sessions -- Use `--list` with `docs edit` to see files without opening -- Accepts both product keys (`influxdb3_core`) and paths (`/influxdb3/core`) - -### Other CLI Commands - -```bash -# Add placeholder syntax to code blocks -docs placeholders - -# Audit documentation coverage -docs audit --products influxdb3_core - -# Generate release notes -docs release-notes v3.1.0 v3.2.0 --products influxdb3_core -``` - -For complete CLI reference, run `docs --help`. - -## Shared Content Management - -When editing files with `source:` frontmatter (shared content): - -- **Recommended**: Use `docs edit ` - automatically finds and opens all - related files -- **Manual**: If editing directly, remember to touch sourcing files to trigger - Hugo rebuild - -For complete shared content workflow, see -[content-editing skill](../skills/content-editing/SKILL.md). - -## Required for All Content Files - -Every content file needs: - -```yaml -title: # Page h1 heading -description: # SEO meta description -menu: - product_menu_key: # Identifies the Hugo menu specific to the current product - name: # Navigation link text - parent: # Parent menu item (if nested) -weight: # Sort order (1-99, 101-199, 201-299...) -``` - -## Testing After Content Changes - -```bash -# 1. Verify Hugo build -npx hugo --quiet - -# 2. Validate links (build first; see DOCS-TESTING.md) -link-checker map content/path/*.md | xargs link-checker check - -# 3. Test code blocks (if applicable) -yarn test:codeblocks:all -``` - -For comprehensive testing workflows, see -[content-editing skill](../skills/content-editing/SKILL.md). - -### Line protocol fences - -Use `lp` for InfluxDB line protocol examples. -The code-block linter validates `lp` fences and blocks malformed syntax in CI. -Qualified field keys use `family::field`; only the first `::` identifies the -family delimiter, so later `::` sequences remain part of the field name. -For an intentionally invalid example, add `{lint="false"}` to the fence. - -## Style Guidelines - -- Use semantic line feeds (one sentence per line) -- Test all code examples before committing -- Use appropriate shortcodes for UI elements -- Follow Google Developer Documentation Style Guide -- Use active voice, present tense, second person -- Use data-ownership framing: when writing import/write/load guidance, point the - verb at the resource the user owns ("import your data into a database or - table"), not at the product ("import data into InfluxDB"). The user owns their - data in their own object storage; InfluxDB reads and writes it but doesn't take - custody of it. -- Phrase recommendations in first-person plural: "We recommend...", not - third-party attributions such as "The Telegraf project recommends...". - Docs speak with InfluxData's voice, even when a recommendation originates - in an upstream project's guidance. -- Set `weight` at the page level (top-level frontmatter), not on the menu - entry. Menu items inherit the page weight, and page-level weight keeps - sorting consistent outside menu contexts, such as `children` shortcode - listings. (Hugo sorts unweighted pages after weighted ones, so mixing the - two placements within a section breaks list ordering.) - -## Most Common Shortcodes - -**Callouts**: - -```markdown -> [!Note] -> [!Warning] -> [!Caution] -> [!Important] -> [!Tip] -``` - -**Required elements**: - -```markdown -{{< req >}} -{{< req type="key" >}} -``` - -**Code placeholders**: - -````markdown -```sh { placeholders="DATABASE_NAME|API_TOKEN" } -curl -X POST https://cloud2.influxdata.com/api/v2/write?bucket=DATABASE_NAME -``` -```` - -Replace the following: - -- {{% code-placeholder-key %}}`DATABASE_NAME`{{% /code-placeholder-key %}}: - your database name - -**Tabbed content**: - -```markdown -{{< tabs-wrapper >}} -{{% tabs %}} -[Tab 1](#) -[Tab 2](#) -{{% /tabs %}} -{{% tab-content %}} -Content for tab 1 -{{% /tab-content %}} -{{% tab-content %}} -Content for tab 2 -{{% /tab-content %}} -{{< /tabs-wrapper >}} -``` - -For complete shortcodes reference, see -[DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md). - -## Related Resources - -- **Complete workflow**: [content-editing skill](../skills/content-editing/SKILL.md) -- **CLI decision guidance**: - [docs-cli-workflow skill](../skills/docs-cli-workflow/SKILL.md) -- **Frontmatter**: [DOCS-FRONTMATTER.md](../../DOCS-FRONTMATTER.md) -- **Shortcodes**: [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) -- **Contributing**: [DOCS-CONTRIBUTING.md](../../DOCS-CONTRIBUTING.md) +# Content files + +Use [DOCS-FRONTMATTER.md](../../DOCS-FRONTMATTER.md) and +[DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) as the syntax authority. +Use the [content-editing skill](../skills/content-editing/SKILL.md) for workflow +and the [docs-cli-workflow skill](../skills/docs-cli-workflow/SKILL.md) to select +the `docs` CLI or direct editing. + +## Requirements + +- Page frontmatter supplies `title`, `description`, appropriate `menu`, and + page-level `weight` when it appears in navigation. +- Do not add a body h1. Use semantic line feeds, active present-tense second + person, long CLI options, and `python` rather than `py` fences. +- Use `lp` for line protocol. Add `{lint="false"}` only to intentional invalid + examples. +- Do not hardcode production docs URLs when a relative link or `relref` works. +- Shared files have no frontmatter. A stub's `source:` must begin `/shared/`. + Direct shared edits require touching every source stub; `docs edit` finds them. +- Use resource-ownership language for import/write/load guidance and write + recommendations in InfluxData's first-person plural voice. + +Run `yarn verify:changed -- ` to select manual checks. See +[content/example.md](../../content/example.md) for working shortcode examples. diff --git a/.agents/instructions/layouts.md b/.agents/instructions/layouts.md index 63c276bf5c..68caef09e8 100644 --- a/.agents/instructions/layouts.md +++ b/.agents/instructions/layouts.md @@ -5,131 +5,23 @@ paths: - "layouts/**/*.html" --- -# Layout and Shortcode Implementation Guidelines - -**Shortcodes reference**: [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) -**Test examples**: [content/example.md](../../content/example.md) - -**For detailed Hugo template development workflow**, see -[hugo-template-dev skill](../skills/hugo-template-dev/SKILL.md) which covers: - -- Hugo template syntax and data access patterns -- Build-time vs runtime testing strategies -- Shortcode implementation best practices -- Complete TDD workflow for Hugo templates - -## No Magic Values in Template Logic - -Templates operate on data and stay ignorant of the values in that data. -A product name, version segment, or `data/products.yml` key must never appear -as a string literal in template logic. -Nobody should have to edit a template because a product was renamed or added. - -Never write any of these in `layouts/**`: - -- A slice of product names or version segments used in a condition, such as a - list of the versions that count as current or the products that support Flux. -- A single hardcoded product comparison that branches behavior, such as testing - whether the first path segment equals a specific product. -- Deriving a `data/products.yml` key by matching the URL path when the page - already declares one. - -This file is generated into `layouts/AGENTS.md`, and Hugo parses every file -under `layouts/` as a template, so it carries no Go template examples. -For the annotated before and after, see the -[hugo-template-dev skill](../skills/hugo-template-dev/SKILL.md). - -Do this instead: - -1. Put the fact in `data/products.yml` as a per-product field — a boolean such - as `supports_flux`, `has_support_contract`, or `search_includes_resources` — - and read it with a `| default` that covers products that don't set it. -2. Resolve the product with `partial "product/get-data.html"` or - `partial "product/get-context.html"`, which read the page's cascade `product` - param. - Every product section declares `product` and `version` by cascade in its - section `_index.md`, so the key is stated rather than guessed. -3. When two templates need the same decision, extract it into one partial so - the two can't drift. - `layouts/partials/product/is-latest.html` is the worked example. - -The one exception is a value that must match an external system rather than a -product fact. -The Algolia search tag in `layouts/partials/header/search-attributes.html` -stays path-derived because Algolia indexed every record under the crawled URL. -Comment any such case in the template so the next reader doesn't "fix" it. - -For the before/after example and the incident behind this rule, see -[hugo-template-dev skill](../skills/hugo-template-dev/SKILL.md). - -## Implementing Shortcodes - -When creating or modifying Hugo layouts and shortcodes: - -1. Use test-driven development using `/cypress/` -2. Use Hugo template syntax and functions -3. Follow existing patterns in `/layouts/shortcodes/` -4. Test in [content/example.md](../../content/example.md) -5. Document new shortcodes in [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) - -## Shortcode Pattern - -```html - -{{ $param := .Get 0 }} -{{ $namedParam := .Get "name" }} - -
- {{ .Inner | markdownify }} -
-``` - -## Testing - -**IMPORTANT:** Use test-driven development with Cypress. - -Add shortcode usage examples to `content/example.md` to verify: - -- Rendering in browser -- Hugo build succeeds -- No console errors -- JavaScript functionality works as expected (check browser console for errors) -- Interactive elements behave correctly (click links, buttons, etc.) - -### TDD Workflow - -1. Add Cypress tests (high-level to start). -2. Run tests and make sure they fail. -3. Implement code changes -4. Run tests and make sure they pass. -5. Add and refine tests. -6. Repeat. - -### Manual Testing Workflow - -1. Make changes to shortcode/layout files -2. Wait for Hugo to rebuild (check terminal output) -3. Get the server URL from the log -4. Open browser DevTools console (F12) -5. Test the functionality and check for JavaScript errors -6. Verify the feature works as intended before marking complete - -See [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) for complete shortcode -documentation. - -### Line protocol render hook - -`layouts/_default/_markup/render-codeblock-lp.html` renders `lp` code fences. -Keep its output Chroma-compatible (`.highlight > pre.chroma > code.language-lp`) -and HTML-escape source text before marking generated markup safe. -For malformed source, render escaped plain text rather than partial highlighting. - -## Related Resources - -- **Complete Hugo template workflow**: - [hugo-template-dev skill](../skills/hugo-template-dev/SKILL.md) -- **Shortcodes reference**: [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) -- **Test examples**: [content/example.md](../../content/example.md) -- **Article-level page actions** (buttons/links next to the page title — when - to use, how to add a new one): - [DOCS-PAGE-ACTIONS.md](../../DOCS-PAGE-ACTIONS.md) +# Hugo layouts and shortcodes + +Use the [hugo-template-dev skill](../skills/hugo-template-dev/SKILL.md) for +implementation and runtime verification. Follow existing shortcode patterns and +document user-facing shortcodes in [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md). + +## Requirements + +- Do not encode product names, versions, URL segments, or product-key guesses + in template branching. Put product facts in `data/products.yml`, resolve page + context through product partials, and share repeated decisions in a partial. +- The sole exception is an externally mandated path-derived value; explain it + in a template comment. +- Add behavior coverage in Cypress and an example in + [content/example.md](../../content/example.md) when appropriate. A Hugo build + alone is insufficient for runtime behavior. +- Keep the line-protocol render hook Chroma-compatible and HTML-escape source; + malformed input renders as escaped plain text. + +Run `yarn verify:changed -- ` to identify manual checks. diff --git a/.agents/skills/ai-visibility/SKILL.md b/.agents/skills/ai-visibility/SKILL.md index 3249743cf0..fced33121a 100644 --- a/.agents/skills/ai-visibility/SKILL.md +++ b/.agents/skills/ai-visibility/SKILL.md @@ -1,235 +1,25 @@ --- name: ai-visibility -description: > - Review documentation PRs, pages, or navigation for visibility to AI - consumers — crawlers building training corpora, RAG retrievers, and - autonomous agents — across the model lifecycle (pre-training, training, - post-training, inference/RAG). Use whenever the user asks how docs appear - to LLMs or AI agents, asks to review a PR or page for AI visibility, - discoverability, GEO, or agent experience, or mentions llms.txt, - llms-full.txt, markdown twins, JSON-LD, canonical links, sitemap-md, or - "how would a model navigate this" — even if they don't say "AI - visibility" explicitly. Also covers SEO-relevant frontmatter review - (titles, descriptions, canonicals) since the same metadata drives both - search snippets and AI retrieval. +description: Review documentation PRs, pages, or navigation for visibility to AI consumers — crawlers building training corpora, RAG retrievers, and autonomous agents — across the model lifecycle (pre-training, training, post-training, inference/RAG). Use whenever the user asks how docs appear to LLMs or AI agents, asks to review a PR or page for AI visibility, discoverability, GEO, or agent experience, or mentions llms.txt, llms-full.txt, markdown twins, JSON-LD, canonical links, sitemap-md, or how a model would navigate a page. --- -# AI visibility review +# AI visibility -Review documentation the way AI systems consume it, not the way humans read -it. Different consumers read different surfaces at different times: crawlers -snapshot HTML for training corpora, RAG retrievers fetch Markdown twins and -chunk them, and agents walk URLs and bind to machine-readable contracts. -A page can be excellent for human readers and invisible — or misleading — -to every one of these consumers. This review finds those gaps with -evidence, not assumptions. +| Review target | Load | +| -------------------------------------- | -------------------------------------------------------------------- | +| Frontmatter, canonical, and navigation | [references/discovery.md](references/discovery.md) | +| Markdown twins and corpora | [references/markdown-artifacts.md](references/markdown-artifacts.md) | +| JSON-LD and agent retrieval | [references/structured-data.md](references/structured-data.md) | -## Inputs +Prerequisites: identify the published URL, source content, and generated +artifact path. Review the full route from crawl discovery through retrieval; do +not infer visibility from one tag. Distinguish current implementation evidence +from recommendations. -Accept any of: +```sh +yarn build:md +yarn check:md-coherence +yarn check:jsonld-links +``` -- **A PR**: `gh pr view ` and `gh pr diff ` for the change; the - PR preview site for rendered output (see the reference file for preview - URL shape and its known artifacts). -- **Page URLs**: production or preview. -- **A section or product**: review the navigation graph and discovery - paths, not just individual pages. - -## Step 0: Scope the review before fetching anything - -State a one-paragraph review plan first: what the target is, which pages -you will fetch, and which layers matter most for this target. An unscoped -review fetches everything at full depth and burns most of its budget -proving things that were never in doubt. - -- **PR**: review the changed pages and the discovery paths they depend - on. Pre-existing product-wide gaps get one line and a pointer, not a - re-investigation. -- **Single page**: full depth on that page; discovery paths checked once. -- **Section or product**: sample, don't enumerate. The product root, one - or two task pages, one reference page, and one API page (if any) reveal - every defect class; reading all 285 pages reveals the same classes - slower. Use corpus-level greps (`llms-full.txt`) to measure how - widespread a defect is after you find it on a sampled page. -- **Match depth to the product**: for a GUI product, the agent-harness - layer is "what can be done headless" (install, config files), not API - contracts. For a legacy product, lead with currency and - compatibility-statement checks. Don't run every layer at uniform depth - when the target makes a layer mostly moot. - -## Step 1: Fetch the surfaces — verify, don't assume - -A review is read-only. **Never build the site (`hugo`, `yarn build:*`, -docker) for a review** — everything a review needs is already published -(production, the PR preview) or readable in source (templates, data -files, content). Local builds take minutes, stall in constrained -environments, and leave artifacts in the working tree, and they verify -nothing that a fetch plus a source read doesn't. If a claim can't be -verified from published artifacts and source, label it an inference and -include the one-line command that would verify it. - -Every claim in the report must come from a fetched artifact or a read -source file. Three gotchas that produce false findings if skipped: - -- **Always `curl --compressed`.** Production serves gzip; raw bytes look - like binary garbage and a naive fetch "finds" nothing. -- **Minified HTML drops attribute quotes.** `grep '`. Match with quote-tolerant patterns - (for example `]*canonical[^>]*>`) or you will report present - metadata as missing. -- **Check preview AND production before calling something a bug.** See - [PR Preview](../../../DOCS-DEPLOYING.md#pr-preview) for deployment behavior - and known parity gaps. Its advertised Markdown twins 404, so check twin - fidelity on production or staging. Other preview-only defects need - investigation; don't dismiss them as expected prefix behavior. - -For each page, capture: - -| Surface | What to check | -| ---------------- | ---------------------------------------------------------------------------------------------- | -| `` | ``, meta description, `<link rel=canonical>`, `<link rel=alternate type=text/markdown>` | -| JSON-LD | Which `@type` nodes exist, and whether `@id` references resolve (entity graph coherence) | -| Headings | h2/h3 hierarchy, stable anchor ids, question-shaped phrasing | -| Markdown twin | Exists? Served as `text/markdown`? Conversion quality (see Step 3, RAG) | -| robots / noindex | Anything blocking AI crawlers from this path | - -### Frontmatter and SEO signals - -Frontmatter is the single source for the page's `<title>`, meta -description, JSON-LD fields, and twin metadata — one weak field degrades -every surface at once. Check in the source files (not just rendered -output): - -- **`description`**: present, specific to the page, roughly one to two - sentences. A missing description falls back to generic text; a - duplicated description (for example, a section landing reusing the - product root's) makes search snippets interchangeable and page - embeddings near-identical — the retriever can't tell the pages apart. - Grep for duplicates across the section; don't check pages one at a - time. -- **`title`**: distinct within the product, matches how users phrase the - task. Title is the strongest single retrieval and snippet signal. -- **Canonical intent**: self-canonical by default; a cross-product - canonical (shared content pointing at the canonical edition) is a - deliberate consolidation — verify it points where intended, and flag - it as a trade-off, not a bug, when it's by design. -- **Dates**: `lastmod`/`date` flow into twins, JSON-LD `dateModified`, - and sitemap `<lastmod>` — stale dates on current content undersell - freshness to both rankers and agents. - -## Step 2: Walk the discovery paths - -An AI consumer can only use what it can find. Walk each path end-to-end -and record where it works and where it dead-ends: - -1. **`llms.txt` path**: site root `llms.txt` → the product or section - entry → its artifacts. Check all three granularities: the per-section - `index.section.md` links, the per-page `.md` twins, and the per-product - `llms-full.txt` flattened corpus. Verify the corpus file exists (fetch - it, don't trust the link) and contains the pages under review. A - product missing from `llms.txt` — or listed without a `llms-full.txt` - corpus — is invisible to every llms.txt-first agent regardless of page - quality. -2. **Sitemap path**: `sitemap.xml` and `sitemap-md.xml` → page and twin - URLs. -3. **HTML head path**: canonical + markdown-alternate + JSON-LD from any - page reached by crawl or search. -4. **In-site navigation**: hops from the product root to the page; menu - placement; `related:` links; anchor-deep links. - -Report the paths as a table (works / dead end), and trace dead ends to -their cause in code or data — the fix usually lives in a template, a -build script, or a data file, not in the page (see the reference file for -where these live in this repo). - -## Step 3: Analyze per consumption layer - -Judge each finding by which lifecycle stage it affects. The same page -serves four consumers with different needs and different latencies: - -### Pre-training / training (months; shapes the next model generation) - -- Is the content server-rendered, public, and crawlable? Anything behind - authentication, a running instance, or client-side JS is absent from - every training corpus, permanently. -- What concrete facts can a model memorize from this page? Ports, paths, - prefixes, version numbers, header schemes. Vague prose trains nothing. -- Name the hallucination risk that gaps create: when docs publish an API's - conventions but not its surface, future models invent the missing - endpoints by analogy with products they know better. For new products, - the first crawl impression is the seed corpus — thin coverage now is - parametric ignorance later. - -### Post-training (instruction and agentic tuning) - -- Are examples self-contained and runnable — imports and auth visible, no - "as in the previous example"? Instruction-tuning corpora favor - Q\&A-shaped sections and copy-adaptable code. -- Do placeholders follow a consistent, machine-recognizable convention - (UPPER\_SNAKE tokens with replace-instructions)? - -### RAG / retrieval (weeks; the layer you can move now) - -- **Chunk quality**: each h2 section should stand alone with the answer in - its first sentence. Retrievers extract passages without surrounding - context. -- **Twin fidelity**: read the generated Markdown twin, don't trust that it - mirrors the HTML. Look for conversion bugs (lost whitespace around links - and code spans, broken tab-group rendering) and for repeated boilerplate - (beta banners, sunset notices) consuming the first chunk of every page - in a section — that makes section pages embed near-identically and - surfaces the banner instead of the answer. -- **The dead-end test**: for the page's core question, what does a - docs-grounded assistant retrieve and what can it actually answer? "Check - your instance" or "see the UI" is a terminal non-answer for every hosted - assistant. - -### Agent harness (immediate) - -- Can an agent execute the page's guidance headlessly? Token auth in a - fenced command beats "log in via the browser." -- Are machine-readable contracts published or fetchable (OpenAPI, JSON - schema, MCP)? An interactive UI is not a contract. -- Are URL shapes predictable and anchors stable enough to construct from a - task description? Count the hops from product root to the answer. - -## Step 4: Report - -Lead with the verdict — what's solid, what's broken, in two or three -sentences. Then: - -- **Facts with evidence.** Every finding cites the fetched URL, file path, - or command output that proves it. Distinguish confirmed facts from - inferences. -- **Separate findings by level.** Page-level (fix in the content), - product-level (fix in data or config — for example a missing corpus - entry), and pipeline-level (fix in shared templates or converters, - affects all products). Mislabeling a pipeline bug as a page bug sends - the fix to the wrong place. -- **Prioritized recommendations.** Order by leverage: discovery-path - breaks first (they gate everything else), then machine-readable - contract gaps, then content coverage, then twin hygiene. -- **Credit what works.** Inherited infrastructure (head links, JSON-LD, - sitemap entries) that the change gets for free belongs in the report — - it scopes the real gaps and prevents re-litigating solved problems. - -Produce the report; don't file anything unsolicited. Then offer -follow-ups. If the user accepts: - -- File pipeline-caused findings as **general** issues citing the observed - pages as evidence — not product-scoped issues, which misleads triage. -- For small mechanical fixes (a cross-link, a frontmatter field), prefer a - direct PR over an issue. - -## Repo specifics - -Before reviewing anything on docs.influxdata.com or in this repo, read -[DOCS-AI-VISIBILITY.md](../../../DOCS-AI-VISIBILITY.md) — it maps the three -Markdown artifact layers, the eligibility predicates and their source -files, the products.yml gate that controls `llms.txt` inclusion, and the PR -preview workflow's known artifacts. Reviews that skip it rediscover (or -misdiagnose) the same plumbing every time. - -[DOCS-DEPLOYING.md](../../../DOCS-DEPLOYING.md#llm-markdown-generation) -covers how those artifacts are generated, if a finding traces back to the -build rather than to content. +Load the applicable reference only for its artifact class. diff --git a/.agents/skills/ai-visibility/references/discovery.md b/.agents/skills/ai-visibility/references/discovery.md new file mode 100644 index 0000000000..68ac16d123 --- /dev/null +++ b/.agents/skills/ai-visibility/references/discovery.md @@ -0,0 +1,4 @@ +# Discovery + +Review titles, descriptions, canonical URLs, navigation, and internal links as +one discovery path. Make facts explicit and use the canonical product metadata. diff --git a/.agents/skills/ai-visibility/references/markdown-artifacts.md b/.agents/skills/ai-visibility/references/markdown-artifacts.md new file mode 100644 index 0000000000..2ec0672a1c --- /dev/null +++ b/.agents/skills/ai-visibility/references/markdown-artifacts.md @@ -0,0 +1,4 @@ +# Markdown artifacts + +Build Markdown artifacts after changing their sources, then check that alternate +paths and generated corpora remain coherent. diff --git a/.agents/skills/ai-visibility/references/structured-data.md b/.agents/skills/ai-visibility/references/structured-data.md new file mode 100644 index 0000000000..8b7685025e --- /dev/null +++ b/.agents/skills/ai-visibility/references/structured-data.md @@ -0,0 +1,4 @@ +# Structured data + +Build the site before checking JSON-LD links. Validate identifiers and URLs +against rendered pages, not only template source. diff --git a/.agents/skills/content-editing/SKILL.md b/.agents/skills/content-editing/SKILL.md index a309647642..1d9e02696a 100644 --- a/.agents/skills/content-editing/SKILL.md +++ b/.agents/skills/content-editing/SKILL.md @@ -3,601 +3,27 @@ name: content-editing description: "Create, edit, and validate InfluxData documentation. Manages Hugo shared content across InfluxDB products, runs Vale style linting and Hugo builds, validates frontmatter and code blocks, and fact-checks via the documentation MCP server. Use when creating new doc pages, editing markdown .md files, managing shared content, running Vale or Hugo builds, or testing InfluxDB, Telegraf, or Flux documentation." --- -# Content Editing Workflow +# Content editing -## Quick Decision Tree +Use this skill for Markdown content. It routes work; it does not replace the +frontmatter, shortcode, or contributor references. -``` -CLI vs direct editing? → See docs-cli-workflow skill -Shared content changes? → Touch sourcing files (Part 1) -Run tests? → Hugo build, Vale, code-block lint, E2E (Part 2) -Verify technical accuracy? → MCP server (Part 4) -Run/fix Vale? → vale-linting skill. Author Vale rules/regex? → vale-rule-config skill -``` - -## Using `docs create` and `docs edit` - -**For detailed guidance on when and how to use the docs CLI tools**, see the **docs-cli-workflow** skill, which covers: - -- When to suggest `docs create` vs direct file creation -- When to suggest `docs edit` vs direct file editing -- CLI command syntax and examples -- How to present recommendations to users -- Edge cases and user preference handling - -**Quick reference for this workflow:** +| Situation | Do this | Load when needed | +| ----------------------- | ------------------------------------------- | ------------------------------------------------------------ | +| New page or direct edit | Choose `docs` CLI or direct edit | `../docs-cli-workflow/SKILL.md` | +| Shared source | Find and update every source stub | [references/shared-content.md](references/shared-content.md) | +| Technical claim | Verify against the documentation search MCP | [references/fact-checking.md](references/fact-checking.md) | +| Validation | Run the changed-file verifier | `../docs-testing/SKILL.md` | -```bash -# Create new documentation from draft -docs create <draft-path> --products <product-key> +Prerequisites: identify the product from `data/products.yml`; read applicable +frontmatter and shortcode references; preserve semantic line feeds. Shared files +contain no frontmatter and product stubs use `source: /shared/...`. -# Edit existing documentation by URL or path -docs edit <url-or-path> - -# List files without opening editor (agent-friendly) +```sh docs edit <url-or-path> --list - -# Add placeholder syntax to code blocks -docs placeholders <file.md> - -# Important: Both commands are non-blocking by default -# - Launch editor in background -# - Return immediately (agent-friendly) -# - Use --wait flag for blocking behavior -``` - -## Part 1: Shared Content Management - -### What is Shared Content? - -Content that appears in multiple products uses Hugo's **content adapter** pattern: - -``` -content/ -├── shared/ -│ └── influxdb3/ -│ └── admin/ -│ └── databases.md # Actual content here -├── influxdb3/ -│ ├── core/ -│ │ └── admin/ -│ │ └── databases.md # Frontmatter only, has source: -│ └── enterprise/ -│ └── admin/ -│ └── databases.md # Frontmatter only, has source: -``` - -**Frontmatter file example:** - -```yaml ---- -title: Database Management -source: /shared/influxdb3/admin/databases.md ---- -``` - -### CRITICAL: Touching Sourcing Files - -**When you edit a shared content file, Hugo does NOT automatically rebuild pages that reference it.** - -You MUST touch the frontmatter files to trigger a rebuild: - -```bash -# Manual approach -touch content/influxdb3/core/admin/databases.md -touch content/influxdb3/enterprise/admin/databases.md -``` - -### Automatic (Recommended) - -**Use `docs edit`** - it handles this automatically: - -```bash -# This command will: -# 1. Find the shared source file -# 2. Find ALL frontmatter files that reference it -# 3. Open all files for editing -# 4. You edit the shared file -# 5. Hugo sees changes to frontmatter files and rebuilds - -docs edit /influxdb3/core/admin/databases/ -``` - -### Programmatic Detection - -If you need to handle this in code: - -```javascript -// Check if a file is a sourcing file (frontmatter only) -import { readFileSync } from 'fs'; -import matter from 'gray-matter'; - -function isSharedContent(filePath) { - const content = readFileSync(filePath, 'utf8'); - const { data, content: body } = matter(content); - - // If has source: frontmatter and minimal/no body content - return data.source && body.trim().length < 50; -} - -function getSharedSource(filePath) { - const content = readFileSync(filePath, 'utf8'); - const { data } = matter(content); - return data.source; // e.g., "/shared/influxdb3/admin/databases.md" -} -``` - -### Why This Matters - -**Failure to touch sourcing files means:** - -- Hugo won't rebuild the pages -- Your changes won't appear in test/preview -- Tests will fail because content hasn't changed -- Published site won't reflect your edits - -### Check for Path Differences and Add `alt_links` - -When creating or editing shared content, check if the URL paths differ between products. If they do, add `alt_links` frontmatter to each product file to cross-reference the equivalent pages. - -See [DOCS-FRONTMATTER.md](../../../DOCS-FRONTMATTER.md#alternative-links-alt_links) for syntax and examples. - -### Check product resource terms are cross-referenced - -Product resource terms often appear inside `code-placeholder-key` shortcode text and bullet item text. -Example product resource terms: - -- "database token" -- "database name" - -## Part 2: Testing Workflow - -After making content changes, run tests to validate: - -### 1. Hugo Build Test (Required) - -```bash -# Verify Hugo can build the site -yarn hugo --quiet - -# Look for errors like: -# - Template errors -# - Missing partials -# - Invalid frontmatter -# - Broken shortcodes -``` - -### 2. Link Validation (Recommended) - -Link validation uses the `link-checker` binary (see -[DOCS-TESTING.md § "Link Validation with Link-Checker"](../../../DOCS-TESTING.md#link-validation-with-link-checker) -for installation). It validates the rendered HTML, so build the site first. - -```bash -# Build the site (link-checker reads public/ HTML) -yarn hugo --quiet - -# Map changed Markdown to public HTML, then check the links -link-checker map content/influxdb3/core/path/*.md | xargs link-checker check - -# Or check a built HTML subtree directly -link-checker check public/influxdb3/core/get-started/ - -# This checks: -# - Internal links (relative paths) -# - Cross-references -# - Anchor links -``` - -### 3. Code Block Syntax Lint (Fast — Runs on Every PR) - -Parse/compile-only check for fenced code blocks. No credentials, no network, no Docker needed. - -```bash -# Lint changed content files (exits 1 if any JSON/YAML/TOML block fails) -yarn lint-codeblocks content/influxdb3/core/admin/tokens/*.md - -# Run the linter's own test suite -yarn test:lint-codeblocks -``` - -**Blocking policy (mirrors CI):** - -| Language | On failure | -| ------------------------ | ---------------------------------- | -| JSON, YAML, TOML, LP | `::error::` — fails the PR check | -| bash, python, javascript | `::warning::` — informational only | - -**Normalization:** declared `placeholders="TOKEN|DURATION"` fence attributes and Hugo shortcodes (`{{< >}}`, `{{% %}}`) are substituted before parsing. See `DOCS-TESTING.md § "Parse/compile code-block lint"` for details. - -### 4. Code Block Execution Testing (For Pages with Runnable Examples) - -```bash -# Test all code examples in documentation -yarn test:codeblocks:all - -# Tests code blocks marked with: -# - testable: true -# - Validates syntax -# - Can execute examples (for supported languages) -``` - -### 5. E2E Testing (For Specific Pages) - -Use the **cypress-e2e-testing** skill for comprehensive page testing: - -```bash -# Test specific content file -node cypress/support/run-e2e-specs.js content/influxdb3/core/admin/databases/_index.md - -# Test API reference pages (requires yarn build:api-docs first) -node cypress/support/run-e2e-specs.js \ - --spec "cypress/e2e/content/api-reference.cy.js" \ - content/influxdb3/core/api/_index.md -``` - -**Important prerequisites:** - -- API tests: Run `yarn build:api-docs` first -- Markdown validation: Run `yarn hugo --quiet && yarn build:md` first - -See **cypress-e2e-testing** skill for detailed test workflow. - -### 6. Style Linting (Pre-commit) - -Vale style linting runs automatically via pre-commit hooks, but you can run it manually: - -```bash -# Lint specific files -.ci/vale/vale.sh --config=.vale.ini content/influxdb3/core/path/to/file.md - -# Lint with minimum alert level -.ci/vale/vale.sh --config=.vale.ini --minAlertLevel=warning content/path/ - -# Sync Vale packages (after .vale.ini changes) -.ci/vale/vale.sh sync -``` - -**Common issues:** - -- `admin` flagged → Use "administrator" in prose, or it's in a code context -- Duration literals (`30d`) → These are valid InfluxDB syntax -- Technical terms flagged → Add to `.ci/vale/styles/InfluxDataDocs/Terms/ignore.txt` - -See **vale-linting** skill for comprehensive Vale workflow. - -### 7. Visual Preview (Optional) - -```bash -# Start Hugo development server -yarn hugo server - -# Visit http://localhost:1313 -# Preview your changes in browser -``` - -## Part 3: Vale Style Linting - -Vale checks documentation for style guide violations, spelling errors, and branding consistency. - -**For writing Vale rules and understanding regex patterns**, see the **vale-rule-config** skill. - -### Running Vale - -```bash -# Basic linting (all markdown files) -.ci/vale/vale.sh content/**/*.md - -# Lint specific product -.ci/vale/vale.sh content/influxdb3/core/**/*.md - -# With specific config and alert level -.ci/vale/vale.sh \ - --config=content/influxdb3/cloud-dedicated/.vale.ini \ - --minAlertLevel=error \ - content/influxdb3/cloud-dedicated/write-data/**/*.md -``` - -### Understanding Vale Alerts - -Vale reports three alert levels: - -- **Error** (red): Critical issues - branding violations, broken style rules, rejected terms -- **Warning** (yellow): Style guide recommendations - should be fixed -- **Suggestion** (blue): Optional improvements - consider fixing - -### Fixing Common Vale Issues - -**Spelling/vocabulary errors:** - -```bash -# If Vale flags a legitimate term, add it to vocabulary -echo "YourTerm" >> .ci/vale/styles/config/vocabularies/InfluxDataDocs/accept.txt -``` - -**Style violations:** -Vale will suggest the correct form. For example: - -``` -content/file.md:25:1: Use 'InfluxDB 3' instead of 'InfluxDB v3' -``` - -Simply make the suggested change. - -**False positives:** -If Vale incorrectly flags something: - -1. Check if it's a new technical term that should be in vocabulary -2. See if the rule needs refinement (consult **vale-rule-config** skill) -3. Add inline comments to disable specific rules if necessary: - -```markdown -<!-- vale InfluxDataDocs.Spelling = NO --> -This paragraph contains technical terms that Vale might flag. -<!-- vale InfluxDataDocs.Spelling = YES --> -``` - -### When to Run Vale - -- **Before committing**: Pre-commit hooks run Vale automatically -- **After content changes**: Run manually to catch issues early -- **In CI/CD**: Automated on pull requests - -## Part 4: Fact-Checking with the Documentation MCP Server - -The **InfluxDB documentation MCP server** lets you search InfluxDB documentation (the rendered `content` managed in this repository) and related InfluxData references (source code READMEs, community forums, and some third-party tool documentation) directly from your AI assistant. - -### When to Use the Documentation MCP Server - -The primary source of content in the Documentation MCP Server is the fully rendered `public` HTML from this repository. -Use the Documentation MCP Server when the information here is inconclusive, when you need to deepen your understanding of InfluxData products and integrations, or when identifying content gaps in the documentation. - -**Use for:** - -- Verifying technical accuracy of claims -- Checking current API syntax -- Confirming feature availability across products -- Understanding complex product behavior -- Finding related documentation and code examples -- Identifying and analyzing content gaps in the documentation - -**Don't use for:** - -- Basic style/grammar checks (use Vale) -- Link validation (use `link-checker`) -- Testing code examples (use `yarn test:codeblocks`) - -### Setup - -The documentation MCP server is hosted at `https://influxdb-docs.mcp.kapa.ai`—no local installation required. - -Already configured in [`.mcp.json`](/.mcp.json). Two server entries are available: - -- **`influxdb-docs`** (API key) — Set `INFLUXDATA_DOCS_KAPA_API_KEY` env var. 60 req/min. -- **`influxdb-docs-oauth`** (OAuth) — No setup. Authenticates via Google or GitHub on first use. 40 req/hr, 200 req/day. - -### Available Tool - -The MCP server exposes a semantic search tool: - -```text -search_influxdb_knowledge_sources +yarn verify:changed -- <changed-content-files> ``` -**What it does:** - -- Searches all InfluxDB documentation for a given query -- Returns relevant chunks in descending order of relevance -- Each chunk includes `source_url` and Markdown `content` - -**Example queries:** - -- "How do I create a database in InfluxDB 3 Core?" -- "What's the difference between InfluxDB 3 Core and Enterprise clustering?" -- "Show me InfluxQL SELECT syntax for filtering by time range" - -### Example Workflow: Fact-Checking During Editing - -```markdown -## Scenario: Editing database management documentation - -1. Draft claims: "InfluxDB 3 supports up to 10,000 databases per instance" - -2. Ask your AI assistant to verify using the MCP server: - "What are the database limits in InfluxDB 3 Core and Enterprise?" - -3. MCP response returns documentation chunks with actual limits - -4. Update draft with accurate information - -5. Cite the source_url in documentation if needed -``` - -### Best Practices - -**DO:** - -- Ask specific, focused questions -- Verify claims about features, limits, syntax -- Cross-check answers with source URLs provided -- Use for understanding complex interactions - -**DON'T:** - -- Rely solely on MCP without reviewing source docs -- Use for subjective style decisions -- Expect real-time product behavior (it searches documentation, not live systems) -- Use as a replacement for testing (always test code examples) - -## Part 5: Complete Example Workflows - -### Example 1: Creating New Multi-Product Documentation - -```bash -# Step 1: Create content from draft -docs create database-tutorial.md --products influxdb3_core,influxdb3_enterprise - -# CLI scaffolds files: -# - content/shared/influxdb3/guides/database-tutorial.md -# - content/influxdb3/core/guides/database-tutorial.md (frontmatter) -# - content/influxdb3/enterprise/guides/database-tutorial.md (frontmatter) - -# Step 2: Verify technical accuracy -# Ask your AI assistant (with MCP configured) to verify claims: -# "Verify database creation syntax for InfluxDB 3" - -# Step 3: Test Hugo build -yarn hugo --quiet - -# Step 4: Run E2E tests -node cypress/support/run-e2e-specs.js \ - content/influxdb3/core/guides/database-tutorial.md - -# Step 5: Validate links (build first, then map + check) -yarn hugo --quiet -link-checker map content/influxdb3/core/guides/database-tutorial.md | \ - xargs link-checker check - -# Step 6: Test code examples (if tutorial has code blocks) -yarn test:codeblocks:all -``` - -### Example 2: Editing Shared Content - -```bash -# Step 1: Find and edit the content -docs edit https://docs.influxdata.com/influxdb3/core/reference/sql/ - -# CLI automatically: -# - Finds content/shared/influxdb3/reference/sql/_index.md -# - Finds ALL frontmatter files referencing it: -# * content/influxdb3/core/reference/sql/_index.md -# * content/influxdb3/enterprise/reference/sql/_index.md -# * content/influxdb3/cloud-dedicated/reference/sql/_index.md -# - Opens all files (sourcing files will be touched when saved) - -# Step 2: Make edits to the shared source file - -# Step 3: Fact-check changes with MCP -# Ask your AI assistant: "Verify SQL WHERE clause syntax in InfluxDB 3" - -# Step 4: Test the build -yarn hugo --quiet - -# Step 5: Test affected pages -node cypress/support/run-e2e-specs.js \ - content/influxdb3/core/reference/sql/_index.md \ - content/influxdb3/enterprise/reference/sql/_index.md - -# Step 6: Validate links in SQL reference (build first, then map + check) -yarn hugo --quiet -link-checker map content/influxdb3/core/reference/sql/_index.md | \ - xargs link-checker check -``` - -### Example 3: Quick Fix Without CLI - -```bash -# Step 1: Fix typo directly (you know the file) -# Edit content/influxdb3/core/get-started/_index.md - -# Step 2: Test Hugo build -yarn hugo --quiet - -# Step 3: Quick visual check -yarn hugo server -# Visit http://localhost:1313/influxdb3/core/get-started/ - -# Done! (No need for comprehensive testing on typo fixes) -``` - -## Part 6: Troubleshooting - -| Problem | Solution | -| ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Hugo build fails | Run `yarn hugo` (no `--quiet`) for detailed errors — check frontmatter YAML, shortcode tags, partial refs | -| Shared content edits not appearing | Touch sourcing files: `grep -r "source: /shared/path" content/` then `touch` each, or use `docs edit` | -| MCP not responding | Verify network allows `*.kapa.ai`; check rate limits (40 req/hr OAuth, 60 req/min API key); if using API key, verify `INFLUXDATA_DOCS_KAPA_API_KEY` is set | -| Cypress tests fail | See **cypress-e2e-testing** skill; check `cat /tmp/hugo_server.log \| tail -50`, run `yarn cypress open`, run `yarn build:api-docs` if API content missing | - -## Part 7: Quick Reference - -| Task | Command | -| -------------------------- | --------------------------------------------------------------------------------- | -| Create new content | `docs create draft.md --products <key-or-path>` | -| Edit by URL | `docs edit https://docs.influxdata.com/...` | -| List files without editing | `docs edit <url> --list` | -| Add placeholders to code | `docs placeholders file.md` or `docs placeholders file.md --dry` | -| Audit documentation | `docs audit --products influxdb3_core` or `docs audit --products /influxdb3/core` | -| Generate release notes | `docs release-notes v3.1.0 v3.2.0 --products influxdb3_core` | -| Build Hugo site | `yarn hugo --quiet` | -| Run Vale linting | `.ci/vale/vale.sh --config=.vale.ini content/path/` | -| Test links | `link-checker map content/path/*.md \| xargs link-checker check` (build first) | -| Lint code block syntax | `yarn lint-codeblocks content/path/*.md` | -| Test code blocks | `yarn test:codeblocks:all` | -| Test specific page | `yarn test:e2e content/path/file.md` | -| Fact-check with MCP | Ask AI assistant with `search_influxdb_knowledge_sources` tool configured | -| Preview locally | `yarn hugo server` (visit localhost:1313) | -| Generate API docs | `yarn build:api-docs` (before API reference tests) | - -**Note:** `--products` accepts both product keys (`influxdb3_core`) and content paths (`/influxdb3/core`). - -## Part 8: Security Review for Deployment and Install Docs - -Install guides, deployment recipes, and runnable code samples can teach users -an insecure default without anyone intending it. Apply this lens whenever you -write or edit a guide that starts a service, publishes a port, or handles a -credential. - -### What to flag - -- **Ports published to all interfaces.** A `docker run --publish 8888:8080` or - Compose `"8888:8080"` binds to `0.0.0.0` (all interfaces). For a UI or admin - service, default the example to the loopback interface - (`127.0.0.1:8888:8080`) and document how to widen exposure deliberately. -- **Credentials entered into a network-reachable service without auth.** If a - guide tells users to paste a token, password, or key into a service that has - no built-in authentication, state plainly that reaching the service equals - holding the credential, and recommend least-privilege credentials. -- **Firewall assumptions.** Docker port publishing adds its own firewall rules - and can bypass `ufw`/`firewalld`. Don't imply a host firewall protects a - published port; tell users to verify reachability from another host. -- **"Production" framing.** Don't label a setup "production" if it only means - "run locally against a production instance." Say which one you mean. -- **Recipes we don't test.** Prefer describing a secure pattern (loopback + - authenticating reverse proxy + TLS) and linking to upstream docs over - shipping a full config we'd have to test and maintain in CI. - -### Apply it consistently for the reader - -- Use a GitHub-flavored callout (`> [!Important]` or `> [!Caution]`) with a - short `####` sub-heading, placed where the user makes the decision (near the - first exposing example), not buried at the end. -- State the current behavior factually. Don't forecast or discount unreleased - fixes--alarmist or workaround-flavored copy ages badly and gets cached by - search engines and LLMs. Track product-side concerns in a separate issue. -- Make the safe path the copy-paste path. A loopback-bound example does more - than a paragraph of warning. - -## Related Skills - -- **docs-cli-workflow** - When to use CLI vs direct editing (decision guidance) -- **vale-linting** - Running Vale, fixing flagged content, vocabulary, and product config (operator workflow) -- **vale-rule-config** - Authoring and testing custom Vale rules and regex (rule-author workflow) -- **cypress-e2e-testing** - Detailed Cypress test execution and debugging -- **hugo-template-dev** - Hugo template syntax and development - -## Checklist: Before Claiming Content is Complete - -- [ ] Content created/edited using appropriate method (CLI or direct) -- [ ] If shared content: Sourcing files touched (or used `docs edit`) -- [ ] If shared content: Check for path differences and add `alt_links` if paths vary -- [ ] Technical accuracy verified (MCP fact-check if needed) -- [ ] Hugo builds without errors (`yarn hugo --quiet`) -- [ ] Vale style linting passes (`.ci/vale/vale.sh --config=.vale.ini content/path/`) -- [ ] Links validated (`link-checker` — see DOCS-TESTING.md) -- [ ] Code examples tested (if applicable) -- [ ] E2E tests pass for affected pages -- [ ] Visual preview confirms changes look correct -- [ ] If the guide starts a service, publishes a port, or handles a credential: security review applied (see Part 8) -- [ ] Related documentation updated (if needed) +Load `references/shared-content.md` only for shared sources. Load +`references/fact-checking.md` only when verifying a technical claim. For full +human guidance, see [DOCS-CONTRIBUTING.md](../../../DOCS-CONTRIBUTING.md). diff --git a/.agents/skills/content-editing/references/fact-checking.md b/.agents/skills/content-editing/references/fact-checking.md new file mode 100644 index 0000000000..518b4c9c51 --- /dev/null +++ b/.agents/skills/content-editing/references/fact-checking.md @@ -0,0 +1,5 @@ +# Fact checking + +Use the hosted documentation search MCP for API syntax and product behavior. +Read the primary product documentation, record uncertainty, and do not turn a +search snippet into an unsupported technical claim. diff --git a/.agents/skills/content-editing/references/shared-content.md b/.agents/skills/content-editing/references/shared-content.md new file mode 100644 index 0000000000..1451fa14c5 --- /dev/null +++ b/.agents/skills/content-editing/references/shared-content.md @@ -0,0 +1,5 @@ +# Shared content + +Use `source: /shared/...` only in product stubs. Shared source files have no +frontmatter. Find every consumer before a direct edit; update or touch each stub +so Hugo rebuilds it. `docs edit <url>` performs this discovery. diff --git a/.agents/skills/cypress-e2e-testing/SKILL.md b/.agents/skills/cypress-e2e-testing/SKILL.md index 1ea7cdb2f5..0d336ad32e 100644 --- a/.agents/skills/cypress-e2e-testing/SKILL.md +++ b/.agents/skills/cypress-e2e-testing/SKILL.md @@ -3,426 +3,17 @@ name: cypress-e2e-testing description: "Run, validate, and analyze Cypress E2E tests for the InfluxData documentation site. Covers Hugo server management, test execution modes, and failure analysis. Use when writing or running Cypress tests, verifying rendered pages, checking JSON-LD/structured data, or debugging E2E failures after template, layout, or content changes." --- -# Cypress E2E Testing Skill +# Cypress E2E testing -## Purpose +| Goal | Command | Load | +| --------------------------- | ------------------------------------------------------------------ | -------------------------------------------------------------- | +| Map a content page to tests | `node cypress/support/run-e2e-specs.js <file>` | [references/content-mapping.md](references/content-mapping.md) | +| Run one explicit spec | `node cypress/support/run-e2e-specs.js --spec <spec> --no-mapping` | [references/runner.md](references/runner.md) | +| Diagnose failure | inspect runner output and artifacts | [references/failures.md](references/failures.md) | -This skill guides agents through running Cypress end-to-end tests for the documentation site, including understanding when Hugo starts automatically vs. manually, interpreting test results, and debugging failures. +Prerequisites: use this after selecting E2E coverage in docs-testing. The runner +manages its Hugo server; do not start a competing server. Add a failing behavior +test before changing templates or interactive UI. -For comprehensive testing documentation, see **[DOCS-TESTING.md](../../../DOCS-TESTING.md)**. - -## Key Insight: Hugo Server Management - -**The test runner (`run-e2e-specs.js`) automatically manages Hugo.** - -- **Port 1315** is used for testing (not 1313) -- If port 1315 is free → starts Hugo automatically -- If port 1315 is in use → checks if it's a working Hugo server and reuses it -- Hugo logs written to `/tmp/hugo_server.log` - -**You do NOT need to start Hugo separately** unless you want to keep it running between test runs for faster iteration. - -## Quick Reference - -| Task | Command | -| ------------------------------- | ------------------------------------------------------------------------------------------------------- | -| Test content file | `node cypress/support/run-e2e-specs.js content/path/to/file.md` | -| Test with specific spec | `node cypress/support/run-e2e-specs.js --spec "cypress/e2e/content/spec.cy.js" content/path/to/file.md` | -| Functionality test (no content) | `node cypress/support/run-e2e-specs.js --spec "cypress/e2e/page-context.cy.js" --no-mapping` | -| Test shortcode examples | `yarn test:shortcode-examples` | - -## Prerequisites - -```bash -# Install dependencies (required) -yarn install - -# Verify Cypress is available -yarn cypress --version -``` - -### API Reference Tests: Additional Prerequisites - -**API reference pages require generation before testing.** The pages don't exist until you run: - -```bash -# Generate API documentation content from OpenAPI specs -yarn build:api-docs -``` - -This step: - -- Processes OpenAPI specs in `api-docs/` directories -- Generates Hugo content pages in `content/*/api/` -- Creates operation pages, tag pages, and index pages - -**Without this step**, all API reference tests will fail with 404 errors. - -**Quick check** - verify API content exists: - -```bash -# Should list generated API content directories -ls content/influxdb3/core/api/ - -# If "No such file or directory", run: yarn build:api-docs -``` - -### Markdown Validation Tests: Additional Prerequisites - -**Markdown validation tests require generated markdown files.** Run: - -```bash -# Build Hugo site first (generates HTML in public/) -npx hugo --quiet - -# Generate LLM-friendly markdown from HTML -yarn build:md -``` - -This creates `.md` files in the `public/` directory that the markdown validation tests check. - -**Without this step**, markdown validation tests will fail with missing file errors. - -## Test Execution Modes - -### Mode 1: Content-Specific Tests (Default) - -Tests specific content files by mapping them to URLs. - -```bash -# Single file -node cypress/support/run-e2e-specs.js content/influxdb3/core/_index.md - -# Multiple files -node cypress/support/run-e2e-specs.js content/influxdb3/core/_index.md content/influxdb3/enterprise/_index.md - -# With specific test spec -node cypress/support/run-e2e-specs.js \ - --spec "cypress/e2e/content/api-reference.cy.js" \ - content/influxdb3/core/reference/api/_index.md -``` - -**What happens:** - -1. Maps content files to URLs (e.g., `content/influxdb3/core/_index.md` → `/influxdb3/core/`) -2. Starts Hugo on port 1315 (if not running) -3. Runs Cypress tests against mapped URLs -4. Stops Hugo when done - -### Mode 2: Functionality Tests (`--no-mapping`) - -Tests UI functionality without requiring content file paths. - -```bash -# Run functionality test -node cypress/support/run-e2e-specs.js \ - --spec "cypress/e2e/page-context.cy.js" \ - --no-mapping -``` - -**Use when:** Testing JavaScript components, theme switching, navigation, or other UI behavior not tied to specific content. - -### Mode 3: Reusing an Existing Hugo Server - -For faster iteration during development: - -```bash -# Terminal 1: Start Hugo manually on port 1315 -npx hugo server --port 1315 --environment testing --noHTTPCache - -# Terminal 2: Run tests (will detect and reuse existing server) -node cypress/support/run-e2e-specs.js \ - --spec "cypress/e2e/content/api-reference.cy.js" \ - content/influxdb3/core/reference/api/_index.md -``` - -## Available Test Specs - -| Spec File | Purpose | -| ------------------------------------------------------- | ----------------------------------------------------------- | -| `cypress/e2e/content/api-reference.cy.js` | API reference pages (Hugo-native templates, layouts, links) | -| `cypress/e2e/content/index.cy.js` | General content validation | -| `cypress/e2e/content/markdown-content-validation.cy.js` | LLM markdown generation | -| `cypress/e2e/page-context.cy.js` | Page context and navigation | - -## Understanding Test Output - -### Success Output - -``` -✅ e2e tests completed successfully -📊 Detailed Test Results: - • Total Tests: 25 - • Tests Passed: 25 - • Tests Failed: 0 -``` - -### Failure Output - -``` -ℹ️ Note: 3 test(s) failed. -📊 Detailed Test Results: - • Total Tests: 25 - • Tests Passed: 22 - • Tests Failed: 3 - -📋 Failed Spec Files: - • cypress/e2e/content/api-reference.cy.js - - Failures: 3 - - Failed Tests: - * has API info - Error: Expected to find element '.article--description' -``` - -### Common Failure Patterns - -| Error | Likely Cause | Solution | -| ----------------------------------- | ----------------------------------- | -------------------------------- | -| All API tests fail with 404 | API content not generated | Run `yarn build:api-docs` first | -| `Expected to find element 'X'` | Selector changed or element removed | Update test or fix template | -| `Timed out waiting for element` | Page load issue or JS error | Check Hugo logs, browser console | -| `cy.request() failed` | Broken link or 404 | Fix the link in content | -| `Hugo server died during execution` | Build error or memory issue | Check `/tmp/hugo_server.log` | - -## Debugging Failures - -### Step 1: Check Hugo Logs - -```bash -cat /tmp/hugo_server.log | tail -50 -``` - -Look for: - -- Template errors (`error calling partial`) -- Build failures -- Missing data files - -### Step 2: Run Test in Interactive Mode - -```bash -# Start Hugo manually -npx hugo server --port 1315 --environment testing - -# In another terminal, open Cypress interactively -yarn cypress open -``` - -### Step 3: Inspect the Page - -Visit `http://localhost:1315/path/to/page/` in a browser and: - -- Open DevTools Console for JavaScript errors -- Inspect elements to verify selectors -- Check Network tab for failed requests - -### Step 4: Run Single Test with Verbose Output - -```bash -DEBUG=cypress:* node cypress/support/run-e2e-specs.js \ - --spec "cypress/e2e/content/api-reference.cy.js" \ - content/influxdb3/core/reference/api/_index.md -``` - -## Test Configuration - -The test runner uses these settings: - -```javascript -{ - browser: 'chrome', - baseUrl: 'http://localhost:1315', - video: false, // Disabled in CI - defaultCommandTimeout: 10000, // 15000 in CI - pageLoadTimeout: 30000, // 45000 in CI -} -``` - -## Writing New Tests - -### Basic Test Structure - -```javascript -describe('Feature Name', () => { - beforeEach(() => { - cy.visit('/path/to/page/'); - }); - - it('validates expected behavior', () => { - cy.get('.selector').should('exist'); - cy.get('.selector').should('be.visible'); - cy.get('.selector').contains('Expected text'); - }); -}); -``` - -### Testing Components - -```javascript -describe('Component Name', () => { - it('initializes correctly', () => { - cy.visit('/path/with/component/'); - - // Wait for component initialization - cy.get('[data-component="my-component"]', { timeout: 5000 }) - .should('be.visible'); - - // Verify component rendered expected elements - cy.get('[data-component="my-component"] .child-element') - .should('have.length.at.least', 1); - }); -}); -``` - -### Using Real Configuration Data - -Import real configuration data (from `data/*.yml`) via `cy.task('getData')` instead of hardcoding expected values. This keeps tests in sync with the source of truth. - -```javascript -describe('Product shortcodes', function () { - let products; - - before(function () { - // Load products.yml via the getData task defined in cypress.config.js - cy.task('getData', 'products').then((data) => { - products = data; - }); - }); - - it('renders the correct product name', function () { - cy.visit('/influxdb3/core/_test/shortcodes/'); - // Assert against YAML data, not a hardcoded string - cy.get('[data-testid="product-name"]').should( - 'contain.text', - products.influxdb3_core.name - ); - }); - - it('renders current-version from YAML', function () { - cy.visit('/influxdb/v2/_test/shortcodes/'); - // Derive expected value the same way the Hugo shortcode does - const patch = products.influxdb.latest_patches?.v2; - const expected = patch ? patch.replace(/\.\d+$/, '') : ''; - cy.get('[data-testid="current-version"] .current-version').should( - 'have.text', - expected - ); - }); -}); -``` - -**Key principles:** - -- Load YAML data in `before()` — available to all tests in the suite -- Derive expected values from the data, mirroring shortcode logic -- Only hardcode what you must: content paths and test page URLs -- Derive boolean flags from data fields (e.g., `product.distributed_architecture`, `product.limits`) - -See `cypress/e2e/content/shortcodes.cy.js` and `cypress/e2e/content/latest-patch-shortcode.cy.js` for full examples. - -### Testing Links - -```javascript -it('contains valid internal links', () => { - cy.get('body').then(($body) => { - if ($body.find('a[href^="/"]').length === 0) { - cy.log('No internal links found'); - return; - } - - cy.get('a[href^="/"]').each(($a) => { - cy.request($a.attr('href')).its('status').should('eq', 200); - }); - }); -}); -``` - -### Testing structured data (JSON-LD) - -The `layouts/partials/header/*-jsonld.html` partials emit schema.org JSON-LD. -The right assertions depend on the node's scope: - -- **Page-scoped nodes** (`TechArticle`, `SoftwareApplication`) describe a - specific page or product. Assert presence/shape where they belong and - **absence** where they don't — the absence check guards against over-emission - (e.g. a SoftwareApplication node leaking onto deep pages instead of only - product landing roots). -- **Global nodes** (`Organization`) describe the site's single entity and are - emitted site-wide with a stable `@id`. Assert **exactly one** per page across - page classes — that catches both omission (a page class emitting nothing) and - accidental duplicate emission. - -Parse `<script type="application/ld+json">` by `@type`, then assert. See -`cypress/e2e/content/jsonld-organization.cy.js` and `jsonld-techarticle.cy.js` -for the established pattern: - -```javascript -function ldByType(win$, doc, type) { - return [...win$(doc).find('script[type="application/ld+json"]')] - .map((s) => { try { return JSON.parse(s.textContent); } catch { return null; } }) - .filter((j) => j && j['@type'] === type); -} - -// Page-scoped: present on the landing root, absent on a deep page. -it('emits SoftwareApplication on the product root, none on deep pages', () => { - cy.visit('/influxdb3/core/'); - cy.document().then((doc) => { - expect(ldByType(Cypress.$, doc, 'SoftwareApplication')).to.have.length(1); - }); - cy.visit('/influxdb3/core/admin/'); - cy.document().then((doc) => { - expect(ldByType(Cypress.$, doc, 'SoftwareApplication')).to.have.length(0); - }); -}); - -// Global: exactly one on every page class (hub, root, deep article). -it('emits exactly one Organization on a deep page', () => { - cy.visit('/influxdb3/core/admin/'); - cy.document().then((doc) => { - expect(ldByType(Cypress.$, doc, 'Organization')).to.have.length(1); - }); -}); -``` - -**Cypress proves the markup is emitted where intended. It does not validate -schema correctness.** For that, use the Schema Markup Validator -(`https://validator.schema.org`) — **not** the Google Rich Results Test, which -reports "no items detected" for `Organization`, `TechArticle`, and -`SoftwareApplication` because they aren't rich-result types (only `FAQPage` -is). See the hugo-template-dev skill, "Validating structured data (JSON-LD)". - -## CI/CD Considerations - -In CI environments: - -- Video recording is disabled to save resources -- Timeouts are increased (15s command, 45s page load) -- Memory management is enabled -- Only 1 test kept in memory at a time - -## Related Files - -- **Test runner**: `cypress/support/run-e2e-specs.js` -- **Hugo server helper**: `cypress/support/hugo-server.js` -- **URL mapper**: `cypress/support/map-files-to-urls.js` -- **Config**: `cypress.config.js` -- **Comprehensive docs**: `DOCS-TESTING.md` - -## Checklist for Test Validation - -Before concluding test analysis: - -- [ ] For API tests: Verify `yarn build:api-docs` was run (check `ls content/*/api/`) -- [ ] All tests passed, or failures are understood -- [ ] Hugo logs checked for build errors -- [ ] Failed selectors verified against current templates -- [ ] Broken links identified and reported -- [ ] JavaScript console errors investigated (if relevant) -- [ ] For JSON-LD changes: presence/absence asserted in Cypress, and schema validated via `validator.schema.org` (not the Rich Results Test) - -## Related Skills - -- **hugo-template-dev** - For Hugo template syntax, data access patterns, and runtime testing. Includes the **PR preview-pages mechanism** — when the change is visual or structural (canonical/meta tags, JSON-LD, head fragments, layout reflows) and Cypress is overkill, list affected URLs in the PR description so the preview workflow lands reviewers on the exact pages without local setup. -- **docs-cli-workflow** - For creating/editing documentation content with CLI tools -- **ts-component-dev** (agent) - TypeScript component behavior and interactivity -- **hugo-ui-dev** (agent) - Hugo templates and SASS/CSS styling +Load only the reference matching the task. See [DOCS-TESTING.md](../../../DOCS-TESTING.md) +for suite-wide contributor guidance. diff --git a/.agents/skills/cypress-e2e-testing/references/content-mapping.md b/.agents/skills/cypress-e2e-testing/references/content-mapping.md new file mode 100644 index 0000000000..257895457b --- /dev/null +++ b/.agents/skills/cypress-e2e-testing/references/content-mapping.md @@ -0,0 +1,4 @@ +# Content mapping + +Pass a content path to the runner to map it to relevant tests. Use `--no-mapping` +only with an explicit Cypress spec. diff --git a/.agents/skills/cypress-e2e-testing/references/failures.md b/.agents/skills/cypress-e2e-testing/references/failures.md new file mode 100644 index 0000000000..e970452fca --- /dev/null +++ b/.agents/skills/cypress-e2e-testing/references/failures.md @@ -0,0 +1,4 @@ +# Failures + +First distinguish an application failure from server startup, fixture, or +environment failure. Re-run the smallest affected spec before widening scope. diff --git a/.agents/skills/cypress-e2e-testing/references/runner.md b/.agents/skills/cypress-e2e-testing/references/runner.md new file mode 100644 index 0000000000..ea1a946079 --- /dev/null +++ b/.agents/skills/cypress-e2e-testing/references/runner.md @@ -0,0 +1,4 @@ +# Runner + +The runner starts and stops Hugo as needed. Keep a single runner active for a +port and preserve its output for failure diagnosis. diff --git a/.agents/skills/docs-cli-workflow/SKILL.md b/.agents/skills/docs-cli-workflow/SKILL.md index 2e8c1b66b9..2a459c3aa0 100644 --- a/.agents/skills/docs-cli-workflow/SKILL.md +++ b/.agents/skills/docs-cli-workflow/SKILL.md @@ -3,178 +3,19 @@ name: docs-cli-workflow description: "Guides when to use the docs create/edit CLI tools versus direct file editing for InfluxData documentation. Use when deciding whether to scaffold new pages with docs create, open existing pages with docs edit, or edit Markdown files directly." --- -# docs CLI Workflow Guidance - -## Purpose - -Help recognize when to suggest `docs create` or `docs edit` CLI tools instead of direct file editing. -These tools provide scaffolding, context gathering, and education about conventions that direct editing misses. - -## When This Skill Applies - -Check for these trigger keywords in user messages: - -- "new page", "new doc", "create documentation", "add a page" -- "edit this URL", "edit <https://docs>", "update this page" (with a URL) -- "document this feature", "write docs for" -- "I have a draft", "from this draft" -- Any docs.influxdata.com URL - -**Skip this skill when:** - -- User provides an explicit file path (e.g., "fix typo in content/influxdb3/...") -- Small fixes (typos, broken links) -- User says "just edit it" or similar -- Frontmatter-only changes - -## Decision: Which Tool to Suggest - -### Suggest `docs create` when - -| Trigger | Why CLI is better | -| -------------------------------------- | --------------------------------------------------------- | -| Content targets multiple products | CLI scaffolds shared content pattern automatically | -| User unsure where page should live | CLI analyzes structure, suggests location | -| Draft references existing docs | CLI extracts links, provides context to avoid duplication | -| User seems unfamiliar with conventions | CLI prompt includes style guide, shortcode examples | -| Complex new feature documentation | CLI gathers product metadata, version info | - -### Suggest `docs edit` when - -| Trigger | Why CLI is better | -| -------------------------------------- | ------------------------------------------------------ | -| User provides docs.influxdata.com URL | CLI finds source file(s) including shared content | -| User doesn't know source file location | CLI maps URL to file path(s) | -| Page uses shared content | CLI identifies both frontmatter file AND shared source | - -### Edit directly when - -| Scenario | Why direct is fine | -| -------------------------------- | ------------------------------- | -| User provides explicit file path | They already know where to edit | -| Small typo/link fixes | CLI overhead not worth it | -| User says "just edit it" | Explicit preference to skip CLI | -| Frontmatter-only changes | No content generation needed | - -## How to Suggest - -When a trigger is detected, present a concise recommendation and wait for confirmation. - -### For `docs create` - -``` -I'd recommend using the docs CLI for this: - -docs create <draft-path> --products <product-key-or-path> - -**Why**: [1-2 sentences explaining the specific benefit] - -Options: -1. **Use CLI** - I'll run the command and guide you through product selection -2. **Edit directly** - Skip the CLI, I'll create/edit files manually - -Which do you prefer? -``` - -### For `docs edit` - -``` -I can use the docs CLI to find the source files for this page: - -docs edit <url-or-path> - -**Why**: [1-2 sentences explaining the benefit] - -Options: -1. **Use CLI** - I'll find and list the relevant files (non-blocking) -2. **I know the file** - Tell me the path and I'll edit directly - -Which do you prefer? -``` - -### Key principles - -- Show the actual command (educational) -- Explain *why* for this specific case -- Always offer the direct alternative -- Keep it brief (4-6 lines max) -- **Wait for user confirmation before running** - -## Edge Cases - -| Situation | Behavior | -| ----------------------------------- | -------------------------------------------------------------------------------------------------------- | -| Already in a `docs create` workflow | Don't re-suggest | -| URL points to non-existent page | Suggest creating a draft, then run `docs create --url <url> --from-draft <draft>` instead of `docs edit` | -| User provides both URL and draft | Suggest `docs create --url <url> --from-draft <draft>` | -| User declines CLI twice in session | Stop suggesting, respect preference | - -## After User Confirms - -Run the appropriate command and let the CLI handle the rest. -No additional guidance needed—the CLI manages product selection, file generation, and context gathering. - -## CLI Reference - -The unified `docs` CLI includes all documentation tooling commands. - -**Product targeting:** `--products` accepts both product keys (`influxdb3_core`) and content paths (`/influxdb3/core`). - -```bash -# CREATE: Create new documentation from a draft -docs create <draft-path> --products <key-or-path> -docs create <draft-path> --products /influxdb3/core,/influxdb3/enterprise -docs create <draft-path> --products influxdb3_core --open # Non-blocking -docs create --url <url> --from-draft <draft-path> # Create at URL - -# EDIT: Find and edit existing documentation -docs edit <url-or-path> # Non-blocking, agent-friendly -docs edit <url-or-path> --list # List files without opening -docs edit <url-or-path> --wait # Block until editor closes -docs edit <url-or-path> --editor nano # Use specific editor - -# PLACEHOLDERS: Add placeholder syntax to code blocks -docs placeholders <file.md> # Add { placeholders="PATTERN" } syntax -docs placeholders <file.md> --dry # Preview changes without writing - -# AUDIT: Audit documentation coverage -docs audit --products influxdb3_core # Default version: main -docs audit --products /influxdb3/core --version v3.3.0 # Specific version -docs audit --products influxdb3_core,influxdb3_enterprise -docs audit --repos ~/github/influxdata/influxdb # Direct repo path - -# RELEASE-NOTES: Generate release notes from commits -docs release-notes v3.1.0 v3.2.0 --products influxdb3_core -docs release-notes v3.1.0 v3.2.0 --products /influxdb3/core,/influxdb3/enterprise -docs release-notes v3.1.0 v3.2.0 --repos ~/repos/influxdb - -# Examples -docs edit https://docs.influxdata.com/influxdb3/core/admin/databases/ -docs edit /influxdb3/core/admin/databases/ -docs placeholders content/influxdb3/core/admin/databases/create.md -``` - -**Note:** `--products` and `--repos` are mutually exclusive for `audit` and `release-notes`. - -**Editor Selection** (checked in order): - -1. `--editor` flag -2. `DOCS_EDITOR` environment variable -3. `VISUAL` environment variable -4. `EDITOR` environment variable -5. System default - -**Important for AI Agents**: - -- Both `docs edit` and `docs create --open` commands are non-blocking by default (launch editor in background and exit immediately) -- This prevents agents from hanging while waiting for user editing -- Use `--wait` only when you need to block until editing is complete -- For `docs create`, omit `--open` to skip editor entirely (files are created and CLI exits) - -For full CLI documentation, run `docs --help`. - -## Related Skills - -- **hugo-template-dev** - For Hugo template syntax, data access patterns, and runtime testing -- **cypress-e2e-testing** - For running and debugging Cypress E2E tests -- **ts-component-dev** (agent) - TypeScript component behavior and interactivity +# docs CLI workflow + +| Need | Use | +| ------------------------------- | --------------------------------------- | +| Scaffold a page from a draft | `docs create <draft> --products <keys>` | +| Locate or edit an existing page | `docs edit <url-or-path> --list` | +| Edit shared content safely | `docs edit <url-or-path>` | +| Small known correction | Direct edit | + +Prerequisites: identify the product key in `data/products.yml` and inspect the +target before changing it. Commands are non-blocking by default; use `--wait` +only for an interactive editor. Use `docs --help` for options. + +Load [references/cli-examples.md](references/cli-examples.md) only when command +selection needs examples or edge cases. Route content validation to +`../docs-testing/SKILL.md`. diff --git a/.agents/skills/docs-cli-workflow/references/cli-examples.md b/.agents/skills/docs-cli-workflow/references/cli-examples.md new file mode 100644 index 0000000000..a9ef4145bc --- /dev/null +++ b/.agents/skills/docs-cli-workflow/references/cli-examples.md @@ -0,0 +1,5 @@ +# CLI examples + +Use `docs create draft.md --products influxdb3_core` to scaffold and +`docs edit /influxdb3/core/path/ --list` to inspect a mapped page. Use `--wait` +only when an interactive editor must remain attached. diff --git a/.agents/skills/docs-testing/SKILL.md b/.agents/skills/docs-testing/SKILL.md index 193153d9db..02b37dc19a 100644 --- a/.agents/skills/docs-testing/SKILL.md +++ b/.agents/skills/docs-testing/SKILL.md @@ -3,241 +3,24 @@ name: docs-testing description: Testing decision guide for agents working in docs-v2. Maps changed file types to the exact test commands to run, documents what runs automatically in hooks and CI, and flags coverage gaps. Load Part 1 for the decision table; load later parts only when executing a specific test type. --- -# docs-testing Skill +# Documentation testing -## Part 1: Decision Table — What to Run +`git commit` runs staged-file hooks. Do not run `yarn lint` before committing +unless diagnosing a hook failure. Start with `yarn verify:changed`. -Identify the changed file type and run the corresponding commands. Pre-commit hooks run automatically; "Run locally" items require manual invocation before or after committing. - -| Changed file | Pre-commit (auto) | Run locally | CI (auto on PR) | -| ---------------------------------------------------------------------------------------- | ----------------------------------------------- | ---------------------------------------------------------------------------------------------------------- | -------------------------------------------------- | -| `content/**/*.md` | Vale, markdown checks | `yarn lint-codeblocks <files>` · `link-checker map+check` | Vale, link-checker, codeblock lint | -| `content/shared/**/*.md` | Vale (via product globs) | Same as content. Also find consuming products: `grep -r "source:.*<filename>" content/` | Same | -| `layouts/**/*.html` | render-hook whitespace (if render hook) | `node cypress/support/run-e2e-specs.js --spec cypress/e2e/... --no-mapping` | pr-render-check, Cypress (if layout/asset changed) | -| `assets/js/*.ts` | TypeScript build (auto-staged) | — | — | -| `assets/**/*.{js,mjs,css,scss}` or `layouts/*.html` or `content/example.md` | prettier, eslint | (pre-push auto) Cypress shortcode examples | pr-render-check | -| `data/products.yml` | build-agent-instructions, check-feedback-links | — | pr-feedback-links | -| `api-docs/**/*.yml` | — | `yarn build:api-docs` then Cypress for affected pages | — | -| `*.sh` | shellcheck | — | — | -| `README.md`, `DOCS-*.md`, `AGENTS.md`, `CLAUDE.md`, `.github/**/*.md`, `.claude/**/*.md` | remark (auto-fixed), Vale (instructions config) | — | pr-remark-check | -| `lefthook.yml`, `.github/workflows/*.yml` | — | Manual review | — | -| `scripts/**` or `layouts/index.llmstxt.txt` or `scripts/lib/corpus-paths.js` | — | `yarn build:ts && npx hugo --quiet && yarn build:md && yarn check:md-coherence && yarn check:jsonld-links` | pr-ai-artifacts-check | - -**Shared content rule:** A change to `content/shared/foo.md` affects every product that has `source: /shared/foo.md` in a stub. Run the lint commands against the shared file; the CI link-checker and Vale resolve stubs to products automatically. - -*** - -## Part 2: Automation Coverage - -Do not re-run these manually — they run automatically. - -### Pre-commit (lefthook.yml) - -Runs on `git commit` against staged files: - -- `deprecated-markdown-patterns` — banned shortcodes, `py` fence identifier -- `check-support-links` — non-standard support.influxdata.com URLs -- `check-source-paths` — `source:` must start with `/shared/` -- `check-render-hook-whitespace` — whitespace leaks in render hooks -- `check-feedback-links` — `data/products.yml` product feedback URLs -- Vale per-product — one hook per product, matches its content glob -- `lint-markdown-instructions` + `lint-instructions` — remark + Vale for repo docs -- `build-typescript` — compiles `assets/js/*.ts` -- `prettier` — formats JS/CSS/TS (auto-staged) -- `lint-js` — ESLint on `assets/js/` -- `shellcheck` — shell script lint - -### Pre-push (lefthook.yml) - -Runs on `git push`: - -- `packages-audit` — `yarn audit` (fails on default branch only) -- `e2e-shortcode-examples` — Cypress shortcode test (triggers on asset/layout/example.md changes) - -Code block execution tests are **disabled** in pre-push hooks. Run them manually. - -### CI checks on every PR - -| Workflow | What it checks | Blocks merge? | -| -------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------- | -| `pr-vale-check.yml` | Vale on changed markdown + shared content | Errors only | -| `pr-link-check.yml` | Links in changed pages (also download/install pages when `data/products.yml` changes) | Warnings only | -| `pr-release-check.yml` | Reminds to bump `data/products.yml` when release notes advance; on a version bump, reminds to confirm download artifacts are published | No (reminders only) | -| `test.yml` (lint-codeblocks job) | Parse/compile check on changed content | JSON/YAML/TOML/LP errors fail | -| `pr-render-check.yml` | Whitespace-escaped code blocks, Cypress render | Yes (render artifacts) | -| `pr-remark-check.yml` | Remark lint on repo docs | No | -| `pr-ai-artifacts-check.yml` | Markdown twins, llms-full corpora, JSON-LD `@id` references (full site build) | Yes | -| `block-ephemeral-docs.yml` | Blocks PLAN.md and HANDOVER.md on master | Yes | -| `pr-feedback-links.yml` | Rendered feedback link validation | Warnings only | -| `pr-lockfile-lint.yml` | yarn.lock integrity | Yes | -| `auto-label.yml` | Applies product labels | No | -| `pr-preview.yml` | Deploys a full-site preview to staging S3 | No | - -Code block **execution** is NOT a PR check. It runs on demand via `workflow_dispatch`. - -### Dependency updates (org-managed) - -**Dependabot runs org-wide** for all influxdata repos — it is managed by the -security team, not by a workflow in this repo. Do not add a parallel -dependency-update mechanism. Two implications for agents: - -- When you add or update a third-party GitHub Action, **pin it by full commit - SHA** with a version comment (see `.github/workflows/pr-lockfile-lint.yml`) so - Dependabot can bump it cleanly. -- A repo-level `.github/dependabot.yml`, if present, supplements the org config; - scheduled `github-actions` version updates require an explicit entry there. - Coordinate with the security team before changing dependency automation. - -*** - -## Part 3: Running Specific Tests - -Load this section only when you need to run a specific test type. - -### Codeblock lint (parse/compile) - -```sh -# Single file or glob -yarn lint-codeblocks content/influxdb3/core/admin/tokens/admin/*.md - -# Compact summary (counts by severity, top failing files) -yarn lint-codeblocks:pretty content/**/*.md - -# Self-tests -yarn test:lint-codeblocks -``` - -Exit code 1 if any JSON/YAML/TOML/LP block fails to parse. -bash/python/JS failures are warnings only. - -`lp` validates InfluxDB line protocol, including qualified field keys such as -`family::field`. -Use `{lint="false"}` only for intentionally invalid examples. - -Linter normalizes `{ placeholders="..." }` fence attributes and strips Hugo shortcodes inside fences before parsing. - -### Code block execution (pytest) - -Prerequisites: Docker installed, `docker build -t influxdata/docs-pytest:latest -f Dockerfile.pytest .`, `.env.test` file in the product directory. - -```sh -yarn test:codeblocks:influxdb3_core -yarn test:codeblocks:influxdb3_enterprise -yarn test:codeblocks:telegraf -yarn test:codeblocks:v2 -yarn test:codeblocks:cloud -yarn test:codeblocks:all -``` - -For InfluxDB 3 Core/Enterprise local server setup: see the [influxdb3-test-setup skill](../influxdb3-test-setup/SKILL.md). - -For CI manual dispatch: - -```sh -gh workflow run "Test Code Blocks" \ - --repo influxdata/docs-v2 \ - -f products=core,telegraf \ - -f use_default_group=false -``` - -### Link validation - -```sh -# Map content file(s) to HTML paths, then check -link-checker map content/influxdb3/core/get-started/ | xargs link-checker check - -# Changed files in last commit -git diff --name-only HEAD~1 HEAD | grep '\.md$' | \ - xargs link-checker map | xargs link-checker check - -# With production config (same as CI) -link-checker check \ - --config .ci/link-checker/production.lycherc.toml \ - public/path/to/page/ -``` - -macOS: build from source (`cargo build --release` in `docs-tooling/link-checker`). - -### Vale style linting - -```sh -# Default config -.ci/vale/vale.sh content/influxdb3/core/**/*.md - -# Product-specific config -.ci/vale/vale.sh \ - --config=content/influxdb3/cloud-dedicated/.vale.ini \ - --minAlertLevel=error \ - content/influxdb3/cloud-dedicated/write-data/**/*.md - -# Repo docs (DOCS-*.md, .github/, .claude/) -.ci/vale/vale.sh --config=.vale-instructions.ini README.md -``` - -Vocabulary edits: `.ci/vale/styles/config/vocabularies`. -Rule authoring: see [vale-linting skill](../vale-linting/SKILL.md) and [vale-rule-config skill](../vale-rule-config/SKILL.md). - -### Cypress E2E tests - -```sh -# Test a content file -node cypress/support/run-e2e-specs.js content/influxdb3/core/_index.md - -# Run a specific spec (no content mapping) -node cypress/support/run-e2e-specs.js \ - --spec "cypress/e2e/content/jsonld-organization.cy.js" \ - --no-mapping - -# Shortcode examples -yarn test:shortcode-examples - -# All E2E tests -yarn test:e2e -``` - -The runner manages Hugo on port 1315 automatically. For writing Cypress tests: see [cypress-e2e-testing skill](../cypress-e2e-testing/SKILL.md). - -### Markdown generation validation +| Changed files | Manual check | Reference | +| -------------------------- | -------------------------------------- | -------------------------------------------------------------------- | +| Content Markdown | code-block lint and link check | [references/content-checks.md](references/content-checks.md) | +| Shared content | Include all consuming stubs | [references/content-checks.md](references/content-checks.md) | +| Agent assets | build and validate adapters | [references/agent-assets.md](references/agent-assets.md) | +| Layouts/assets/API/scripts | Deferred by verifier; select test here | [references/specialized-checks.md](references/specialized-checks.md) | ```sh -# Build prerequisites -yarn build:ts -npx hugo --quiet - -# Generate markdown for a path -yarn build:md --public-dir public --path influxdb3/core/get-started --limit 10 - -# Validate output -node cypress/support/run-e2e-specs.js \ - --spec "cypress/e2e/content/markdown-content-validation.cy.js" - -# Check autodiscovery coherence (run after full build:md) -yarn check:md-coherence +yarn verify:changed -- <paths> +yarn verify:changed -- --run <paths> +yarn verify:changed -- --staged ``` -*** - -## Part 4: Known Coverage Gaps - -Flag these when writing or reviewing content in affected areas. - -| Area | Gap | Workaround | -| ----------------------------------- | ------------------------------------------------------------------------------------------- | ------------------------------------------------------ | -| SQL, InfluxQL syntax | `lint-codeblocks` skips these languages | Manual review only | -| Code block execution | Only `influxdb3_core` and `influxdb3_enterprise` run against a live server in CI | Run locally with `yarn test:codeblocks:<product>` | -| `v1`, `explorer` products | No pytest Docker Compose service | No automated execution test; only codeblock lint | -| `assets/js` unit tests | No unit test framework | ESLint only; behavior tested via Cypress E2E | -| Accessibility | No automated a11y tests in CI | Manual audit or browser tool | -| Shared content → product resolution | Shared file changes affect consuming products; local test runs don't auto-detect consumers | `grep -r "source:.*<filename>" content/` to find stubs | -| Cross-product link correctness | link-checker checks pages in isolation; cross-product canonicals aren't verified end-to-end | PR preview + manual check | - -*** - -## References - -- Full contributor testing guide: [DOCS-TESTING.md](../../../DOCS-TESTING.md) -- Cypress test writing: [cypress-e2e-testing skill](../cypress-e2e-testing/SKILL.md) -- Vale rule authoring: [vale-linting skill](../vale-linting/SKILL.md), [vale-rule-config skill](../vale-rule-config/SKILL.md) -- InfluxDB 3 local server setup: [influxdb3-test-setup skill](../influxdb3-test-setup/SKILL.md) -- LLM markdown generation: [scripts/README.md](../../../scripts/README.md) -- Code block test performance: [test/TEST-PERFORMANCE.md](../../../test/TEST-PERFORMANCE.md) +Use `--run` only for checks not covered by hooks. Load the referenced material +only for the category being changed. Use Cypress for runtime behavior, Vale for +style, and Hugo-template-dev for templates; this skill owns check selection. diff --git a/.agents/skills/docs-testing/references/agent-assets.md b/.agents/skills/docs-testing/references/agent-assets.md new file mode 100644 index 0000000000..6581ee5604 --- /dev/null +++ b/.agents/skills/docs-testing/references/agent-assets.md @@ -0,0 +1,5 @@ +# Agent assets + +Run `yarn build:agent:instructions` followed by +`yarn validate:agent-instructions`. Generated adapters and long-form references +are exempt from canonical entrypoint line limits. diff --git a/.agents/skills/docs-testing/references/content-checks.md b/.agents/skills/docs-testing/references/content-checks.md new file mode 100644 index 0000000000..ec7cb91916 --- /dev/null +++ b/.agents/skills/docs-testing/references/content-checks.md @@ -0,0 +1,5 @@ +# Content checks + +Run `yarn lint-codeblocks <files>`. Build rendered pages before link validation: +`link-checker map <files> | xargs link-checker check`. For shared sources include +every product stub that references the source. diff --git a/.agents/skills/docs-testing/references/specialized-checks.md b/.agents/skills/docs-testing/references/specialized-checks.md new file mode 100644 index 0000000000..c8b3bb8275 --- /dev/null +++ b/.agents/skills/docs-testing/references/specialized-checks.md @@ -0,0 +1,5 @@ +# Specialized checks + +Layouts require Hugo and Cypress runtime coverage. Assets require their scoped +build or lint. API specs require `yarn build:api-docs`. Shell, workflows, and +other deferred paths need a deliberate review until verifier support expands. diff --git a/.agents/skills/hugo-template-dev/SKILL.md b/.agents/skills/hugo-template-dev/SKILL.md index 56bccfaa50..06bb4fbd9a 100644 --- a/.agents/skills/hugo-template-dev/SKILL.md +++ b/.agents/skills/hugo-template-dev/SKILL.md @@ -3,672 +3,22 @@ name: hugo-template-dev description: "Hugo template development for InfluxData docs-v2, enforcing build and runtime testing to catch template errors that build-only validation misses. Use when creating or editing Hugo layouts, partials, or shortcodes, debugging template errors, or accessing site data in templates." --- -# Hugo Template Development Skill +# Hugo template development -## Purpose +| Change | Required evidence | Load | +| ------------------------- | --------------------------------------------- | -------------------------------------------------------------- | +| Template or partial | Hugo build plus runtime Cypress coverage | [references/runtime-testing.md](references/runtime-testing.md) | +| Product-specific behavior | Data-driven product context, no magic values | [references/product-data.md](references/product-data.md) | +| New shortcode | example page, docs, and Cypress behavior test | [references/shortcodes.md](references/shortcodes.md) | -This skill enforces proper Hugo template development practices, including **mandatory runtime testing** to catch errors that static builds miss. +Prerequisites: inspect nearby patterns and the page cascade. Never branch on a +hardcoded product name, version, or URL segment when `data/products.yml` and +the product context partial provide the fact. A build does not prove runtime +behavior. -## Critical Testing Requirement - -**Hugo's `npx hugo --quiet` only validates template syntax, not runtime execution.** - -Template errors like accessing undefined fields, nil values, or incorrect type assertions only appear when Hugo actually renders pages. You MUST test templates by running the server. - -## Mandatory Testing Protocol - -### For ANY Hugo Template Change - -After modifying files in `layouts/`, `layouts/partials/`, or `layouts/shortcodes/`: - -**Step 1: Start Hugo server in the background and capture output** - -```bash -rm -f /tmp/hugo-1315.log -npx hugo server --port 1315 >/tmp/hugo-1315.log 2>&1 & -sleep 5 -head -50 /tmp/hugo-1315.log -``` - -This keeps the server running for the next steps while still showing startup -output. - -**Success criteria:** - -- No `error calling partial` messages -- No `can't evaluate field` errors -- No `template: ... failed` messages -- Server shows "Web Server is available at <http://localhost:1315/>" - -**If errors appear:** Fix the template and repeat Step 1 before proceeding. - -**Step 2: Verify the page renders** - -```bash -curl -s -o /dev/null -w "%{http_code}" http://localhost:1315/PATH/TO/PAGE/ -``` - -**Expected:** HTTP 200 status code - -**Step 3: Browser testing (if MCP browser tools available)** - -If `mcp__claude-in-chrome__*` tools are available, use them for visual inspection: - -``` -# Navigate and screenshot -mcp__claude-in-chrome__navigate({ url: "http://localhost:1315/PATH/", tabId: ... }) -mcp__claude-in-chrome__computer({ action: "screenshot", tabId: ... }) - -# Check for JavaScript errors -mcp__claude-in-chrome__read_console_messages({ tabId: ..., onlyErrors: true }) -``` - -This catches runtime JavaScript errors that template changes may introduce. - -**Step 4: Stop the test server** - -```bash -pkill -f "hugo server --port 1315" -``` - -### Quick Test Command - -Use this one-liner to test and get immediate feedback: - -```bash -rm -f /tmp/hugo-1315.log -npx hugo server --port 1315 >/tmp/hugo-1315.log 2>&1 & -sleep 5 -grep -E "(error|Error|ERROR|fail|FAIL)" /tmp/hugo-1315.log | head -20 -pkill -f "hugo server --port 1315" 2>/dev/null -``` - -If output is empty, no errors were detected. - -## Preparing template changes for PR review - -The repo's preview workflow (`.github/workflows/pr-preview.yml`) deploys the -whole built site to a stable staging URL -(`https://test2.docs.influxdata.com/pr-preview/pr-<N>/`) on every push — a -hosted preview that replaces "check out my branch and run `npx hugo server`" -for reviewers. It builds with `--environment production`, so it reflects -production behavior (minification, fingerprinted/SRI JS, real analytics). -Subdirectory-baseURL product detection is handled by -`layouts/partials/base-path-offset.html`, not a per-environment config. - -**When you change `layouts/`, `assets/`, or `data/`, list the pages the -reviewer should focus on in the PR body** — this doesn't affect what deploys -(the whole site always does), but it adds deep links to the sticky preview -comment so reviewers don't have to navigate manually. The URL extractor -(`.github/scripts/parse-pr-urls.js`) matches: - -- Production URLs (`https://docs.influxdata.com/<path>`) -- Localhost URLs (`http://localhost:1313/<path>`) -- Bare paths with a known product namespace from `data/products.yml` - (`/influxdb3/...`, `/telegraf/...`, etc.) - -**URLs inside fenced code blocks are stripped** before extraction — list them -as bare paths or markdown links, not inside backtick fences. Pair each URL -with an "Expected" column (DOM element, attribute value, copy) so the -reviewer knows what to verify rather than guessing. - -For wider behavioral coverage (interactive UI, JS errors, navigation), prefer -the cypress-e2e-testing skill. The preview-pages mechanism is the right tool -for **visual / structural** verification — exactly the cases where Cypress is -overkill or doesn't cover what changed. - -### Autodiscovery coherence guard (when touching Markdown-alternate paths) - -If your template change affects `<link rel="alternate" type="text/markdown">`, -the `/sitemap-md.xml` layout, the `/llms.txt` template, or any of the inputs -to `scripts/lib/corpus-paths.js` (notably `data/products.yml`), run after the -full build: - -```bash -npx hugo --quiet && yarn build:md && yarn build:llms-full && yarn check:md-coherence -``` - -`check:md-coherence` runs two coherence checks: head-link → `.md` file -existence, and Hugo `/llms.txt` ↔ `getCorpusPaths()` agreement. Catches drift -between Hugo template logic and the JS derivation from `products.yml`. -See `DOCS-TESTING.md` "Autodiscovery coherence guard" for details. - -## Validating structured data (JSON-LD) - -The `layouts/partials/header/*-jsonld.html` partials emit schema.org JSON-LD -(`Organization`, `TechArticle`, `SoftwareApplication`, `FAQPage`). When you add -or change one: - -**Use the Schema Markup Validator (`https://validator.schema.org`), NOT the -Google Rich Results Test.** - -The Rich Results Test only reports types eligible for a *visual* search -enhancement. Most JSON-LD this repo emits is **not** eligible, so the Rich -Results Test reports "no items detected" even for valid markup — a false -negative that looks like failure: - -| Emitted type | Rich Results Test | Why | -| --------------------- | ----------------- | --------------------------------------------------------------------------------- | -| `Organization` | Not reported | Feeds the knowledge graph / entity resolution, never a rich result | -| `TechArticle` | Not reported | Google's Article rich result fires only for `Article`/`NewsArticle`/`BlogPosting` | -| `SoftwareApplication` | Not reported | Google retired the general software-app rich result | -| `FAQPage` | Reported | One of the few eligible types here | - -`validator.schema.org` validates every schema.org type regardless of -rich-result eligibility — that's what confirms the node is well-formed. - -**Validation steps:** - -1. **Structural (local, scriptable):** parse the emitted block to prove it's - valid JSON and schema-shaped. The minifier emits `type=application/ld+json` - (unquoted) — match the attribute loosely: - - ```bash - python3 - <<'EOF' - import re, json - html = open('public/index.html', encoding='utf-8', errors='replace').read() - for b in re.findall(r'<script type=["\']?application/ld\+json["\']?>(.*?)</script>', html, re.S): - j = json.loads(b) # raises on malformed JSON - print('OK', j.get('@type')) - EOF - ``` - -2. **Schema (manual, reviewer):** paste a deployed preview URL into - `validator.schema.org`; expect 0 errors. Note this in the PR description and - do **not** ask reviewers to use the Rich Results Test for non-`FAQPage` nodes. - -3. **Regression (Cypress):** assertions depend on scope. Page-scoped nodes - (`TechArticle`/`SoftwareApplication`) — assert presence where they belong and - *absence* where they don't (over-emission guard). Global nodes - (`Organization`, emitted site-wide with a stable `@id`) — assert *exactly - one* per page class, which catches both omission and duplicates. See the - cypress-e2e-testing skill, "Testing structured data (JSON-LD)". - -## Common Hugo Template Errors - -### 1. Accessing Keys with Hyphens or Dynamic Names - -Hugo's dot notation only works for keys that are valid Go identifiers -(letters, digits, underscores). This repo's data dir is `article_data` -(underscore), so `.Site.Data.article_data` works. But a hyphenated key — or a -key held in a variable — must use `index`: - -**Wrong (hyphen breaks dot notation; `dataKey` is a variable):** - -```go -{{ .Site.Data.my-data.influxdb }} -{{ .Site.Data.article_data.dataKey }} -``` - -**Correct:** - -```go -{{ index .Site.Data "my-data" "influxdb" }} -{{ index .Site.Data "article_data" $dataKey }} -``` - -### 2. Nil Field Access - -**Wrong:** - -```go -{{ range $articles }} - {{ .path }} {{/* Fails if item is nil or wrong type */}} -{{ end }} -``` - -**Correct:** - -```go -{{ range $articles }} - {{ if . }} - {{ with index . "path" }} - {{ . }} - {{ end }} - {{ end }} -{{ end }} -``` - -### 3. Type Assertion on Interface{} - -**Wrong:** - -```go -{{ range $data }} - {{ .fields.menuName }} -{{ end }} -``` - -**Correct:** - -```go -{{ range $data }} - {{ if isset . "fields" }} - {{ $fields := index . "fields" }} - {{ if isset $fields "menuName" }} - {{ index $fields "menuName" }} - {{ end }} - {{ end }} -{{ end }} +```sh +npx hugo --quiet +node cypress/support/run-e2e-specs.js <content-file> ``` -### 4. Empty Map vs Nil Check - -**Problem:** Hugo's `{{ if . }}` passes for empty maps `{}`: - -```go -{{/* This doesn't catch empty maps */}} -{{ if $data }} - {{ .field }} {{/* Still fails if $data is {} */}} -{{ end }} -``` - -**Solution:** Check for specific keys: - -```go -{{ if and $data (isset $data "field") }} - {{ index $data "field" }} -{{ end }} -``` - -## Hugo Data Access Patterns - -### Safe Nested Access - -```go -{{/* Build up access with nil checks at each level */}} -{{ $articleDataRoot := index .Site.Data "article_data" }} -{{ if $articleDataRoot }} - {{ $influxdbData := index $articleDataRoot "influxdb" }} - {{ if $influxdbData }} - {{ $productData := index $influxdbData $dataKey }} - {{ if $productData }} - {{ with $productData.articles }} - {{/* Safe to use . here */}} - {{ end }} - {{ end }} - {{ end }} -{{ end }} -``` - -### Iterating Over Data Safely - -```go -{{ range $idx, $item := $articles }} - {{/* Declare variables with defaults */}} - {{ $path := "" }} - {{ $name := "" }} - - {{/* Safely extract values */}} - {{ if isset $item "path" }} - {{ $path = index $item "path" }} - {{ end }} - - {{ if $path }} - {{/* Now safe to use $path */}} - {{ end }} -{{ end }} -``` - -## File Organization - -### Layouts Directory Structure - -``` -layouts/ -├── _default/ # Default templates -├── partials/ # Reusable template fragments -│ └── api/ # API-specific partials -├── shortcodes/ # Content shortcodes -└── TYPE/ # Type-specific templates (api/, etc.) - └── single.html # Single page template -``` - -### Partial Naming - -- Use descriptive names: `api/sidebar-nav.html`, not `nav.html` -- Group related partials in subdirectories -- Include comments at the top describing purpose and required context - -## No Magic Values in Template Logic - -**Principle:** Templates operate on data and stay ignorant of the values in that data. A product name, version segment, or `data/products.yml` key must never appear as a string literal in template logic. Nobody should have to edit a template because a product was renamed or added. - -### Why this rule exists - -`header/coveo-meta-data.html` and `header/search-attributes.html` both answered the same question — does this page document the current version of its product? — and each answered it with its own hardcoded list of version segments. The lists drifted. `explorer` and `controller` made it into the Algolia list but not the Coveo list, so InfluxDB 3 Explorer and Telegraf Controller were indexed as current by one search system and as stale by the other. Neither list was wrong on its face. The duplication was. - -### What counts as a magic value - -| Pattern | Example | -| ------------------------------------------- | --------------------------------------------------------------- | -| A list of product names or version segments | `{{ $alwaysLatest := slice "cloud" "core" "enterprise" }}` | -| A hardcoded comparison that branches | `{{ if eq $product "platform" }}` | -| An exclusion list | `{{ if not (in (slice "chronograf" "kapacitor") $product) }}` | -| A key inferred from the URL | `{{ findRE "[^/]+.*?" .RelPermalink }}` to build a products key | - -Find them with: - -```bash -grep -rnE '(slice|in |eq |ne )[^}]*"(core|enterprise|cloud|clustered|explorer|controller|platform|resources|influxdb|telegraf|chronograf|kapacitor|flux)' layouts/ --exclude=AGENTS.md -``` - -Not every hit is a violation. String literals in class names, URLs, and display text are fine. The rule is about **branching on product identity**. - -### Fix 1: Move the fact into products.yml - -Name the field after the fact, not the product, and choose the `default` so only -the exceptions need the field. - -**Before** (`layouts/partials/footer/search.html`): - -```go -{{ $productPathData := findRE "[^/]+.*?" .RelPermalink }} -{{ $product := index $productPathData 0 }} -{{ $version := index $productPathData 1 }} -{{ $fluxSupported := slice "influxdb" "enterprise_influxdb" }} -{{ $influxdbFluxSupport := slice "v1" "v2" "cloud" }} -{{ $includeFlux := and (in $fluxSupported $product) (in $influxdbFluxSupport $version) }} -{{ $includeResources := not (in (slice "cloud-serverless" "cloud-dedicated" "clustered" "core" "enterprise" "explorer") $version) }} -``` - -**After:** - -```go -{{ $ctx := partial "product/get-context.html" . }} -{{/* - Both flags come from data/products.yml so adding a product never requires - editing this template. -*/}} -{{ $includeFlux := $ctx.data.supports_flux | default false }} -{{ $includeResources := $ctx.data.search_includes_resources | default true }} -``` - -`data/products.yml`: - -```yaml -influxdb: - supports_flux: true -influxdb3_core: - search_includes_resources: false -``` - -### Fix 2: Resolve the product from the page, not the path - -`layouts/partials/product/get-data.html` and -`layouts/partials/product/get-context.html` read the page's cascade `product` -param. Every product section declares `product` and `version` by cascade in its -section `_index.md`, so the key is stated rather than guessed. - -```go -{{ $productData := partial "product/get-data.html" . }} -{{ $ctx := partial "product/get-context.html" . }} -{{ $ctx.key }} {{/* "influxdb3_cloud_dedicated" */}} -{{ $ctx.data }} {{/* the products.yml entry */}} -{{ $ctx.product }} {{/* first path segment, for path-scoped rules */}} -``` - -Parsing `.RelPermalink` gets the key wrong under `/influxdb3/`, where the path -segment is `influxdb3` but the keys are `influxdb3_core`, `influxdb3_cloud`, and -so on. It also breaks under a subpath-mounted baseURL, where the PR preview's -`/pr-preview/pr-N/` prefix becomes the "product." - -### Fix 3: Extract a shared decision into one partial - -When two templates need the same answer, give them one partial to call. -`layouts/partials/product/is-latest.html` is the worked example: both search -templates now call it, so they can't disagree about which pages are current. - -```go -{{ $isLatest := partial "product/is-latest.html" . }} -``` - -### The one exception - -A value that must match an external system rather than a product fact stays as -it is. The Algolia search tag in `header/search-attributes.html` is path-derived -because Algolia indexed every record under the crawled URL, and changing the tag -would orphan those records. Comment any such case in the template so the next -reader doesn't "fix" it. - -## Separation of Concerns: Templates vs TypeScript - -**Principle:** Hugo templates handle structure and data binding. TypeScript handles behavior and interactivity. - -### What Goes Where - -| Concern | Location | Example | -| ---------------- | --------------------------- | ----------------------------------- | -| HTML structure | `layouts/**/*.html` | Navigation markup, tab containers | -| Data binding | `layouts/**/*.html` | `{{ .Title }}`, `{{ range .Data }}` | -| Static styling | `assets/styles/**/*.scss` | Layout, colors, typography | -| User interaction | `assets/js/components/*.ts` | Click handlers, scroll behavior | -| State management | `assets/js/components/*.ts` | Active tabs, collapsed sections | -| DOM manipulation | `assets/js/components/*.ts` | Show/hide, class toggling | - -### Anti-Pattern: Inline JavaScript in Templates - -**Wrong - JavaScript mixed with template:** - -```html -{{/* DON'T DO THIS */}} -<nav class="api-nav"> - {{ range $articles }} - <button onclick="toggleSection('{{ .id }}')">{{ .name }}</button> - {{ end }} -</nav> - -<script> -function toggleSection(id) { - document.getElementById(id).classList.toggle('is-open'); -} -</script> -``` - -**Correct - Clean separation:** - -Template (`layouts/partials/api/sidebar-nav.html`): - -```html -<nav class="api-nav" data-component="api-nav"> - {{ range $articles }} - <button class="api-nav-group-header" aria-expanded="false"> - {{ .name }} - </button> - <ul class="api-nav-group-items"> - {{/* items */}} - </ul> - {{ end }} -</nav> -``` - -TypeScript (`assets/js/components/api-nav.ts`): - -```typescript -interface ApiNavOptions { - component: HTMLElement; -} - -export default function initApiNav({ component }: ApiNavOptions): void { - const headers = component.querySelectorAll('.api-nav-group-header'); - - headers.forEach((header) => { - header.addEventListener('click', () => { - const isOpen = header.classList.toggle('is-open'); - header.setAttribute('aria-expanded', String(isOpen)); - header.nextElementSibling?.classList.toggle('is-open', isOpen); - }); - }); -} -``` - -Register in `main.js`: - -```javascript -import initApiNav from './components/api-nav.js'; - -const componentRegistry = { - 'api-nav': initApiNav, - // ... other components -}; -``` - -### Data Passing Pattern - -Pass Hugo data to TypeScript via `data-*` attributes: - -Template: - -```html -<div - data-component="api-toc" - data-headings="{{ .headings | jsonify | safeHTMLAttr }}" - data-scroll-offset="80" -> -</div> -``` - -TypeScript: - -```typescript -interface TocOptions { - component: HTMLElement; -} - -interface TocData { - headings: string[]; - scrollOffset: number; -} - -function parseData(component: HTMLElement): TocData { - const headingsRaw = component.dataset.headings; - const headings = headingsRaw ? JSON.parse(headingsRaw) : []; - const scrollOffset = parseInt(component.dataset.scrollOffset || '0', 10); - - return { headings, scrollOffset }; -} - -export default function initApiToc({ component }: TocOptions): void { - const data = parseData(component); - // Use data.headings and data.scrollOffset -} -``` - -### Minimal Inline Scripts (Exception) - -The **only** acceptable inline scripts are minimal initialization that MUST run before component registration: - -```html -{{/* Acceptable: Critical path, no logic, runs immediately */}} -<script> - document.documentElement.dataset.theme = - localStorage.getItem('theme') || 'light'; -</script> -``` - -Everything else belongs in `assets/js/`. - -### File Organization for Components - -``` -assets/ -├── js/ -│ ├── main.js # Entry point, component registry -│ ├── components/ -│ │ └── api-toc.ts # API table of contents behavior -│ └── utils/ -│ └── dom-helpers.ts # Shared DOM utilities -└── styles/ - └── layouts/ - ├── _api-layout.scss # API page layout (3-column, sidebar, TOC) - └── _api-operations.scss # Operation rendering (methods, params, responses) -``` - -### TypeScript Component Checklist - -When creating a new interactive feature: - -1. [ ] Create TypeScript file in `assets/js/components/` -2. [ ] Define interface for component options -3. [ ] Export default initializer function -4. [ ] Register in `main.js` componentRegistry -5. [ ] Add `data-component` attribute to HTML element -6. [ ] Pass data via `data-*` attributes (not inline JS) -7. [ ] **NO inline `<script>` tags in templates** - -## Debugging Templates - -### Enable Verbose Mode - -```bash -npx hugo server --port 1315 --verbose 2>&1 | head -100 -``` - -### Print Variables for Debugging - -```go -{{/* Temporary debugging - REMOVE before committing */}} -<pre>{{ printf "%#v" $myVariable }}</pre> -``` - -### Check Data File Loading - -```bash -# Verify data files exist and are valid YAML -cat data/article_data/influxdb3/core/api/articles.yml | head -20 -``` - -## Integration with CI/CD - -### Pre-commit Hook (Recommended) - -Add to `.lefthook.yml` or pre-commit configuration: - -```yaml -pre-commit: - commands: - hugo-template-test: - glob: "layouts/**/*.html" - run: | - timeout 20 npx hugo server --port 1315 2>&1 | grep -E "error|Error" && exit 1 || exit 0 - pkill -f "hugo server --port 1315" 2>/dev/null -``` - -### GitHub Actions Workflow - -```yaml -- name: Test Hugo templates - run: | - npx hugo server --port 1315 & - sleep 10 - curl -f http://localhost:1315/ || exit 1 - pkill -f hugo -``` - -## Quick Reference - -| Action | Command | -| ------------------------- | -------------------------------------------------------------------- | -| Test templates (runtime) | `npx hugo server --port 1315 2>&1 \| head -50` | -| Build only (insufficient) | `npx hugo --quiet` | -| Check specific page | `curl -s -o /dev/null -w "%{http_code}" http://localhost:1315/path/` | -| Stop test server | `pkill -f "hugo server --port 1315"` | -| Debug data access | `<pre>{{ printf "%#v" $var }}</pre>` | - -## Remember - -1. **Never trust `npx hugo --quiet` alone** - it only checks syntax -2. **Always run the server** to test template changes -3. **Check error output first** before declaring success -4. **Use `isset` and `index`** for safe data access -5. **Hyphenated keys require `index` function** - dot notation fails -6. **No product names in template logic** - put the fact in `data/products.yml` - and resolve the product with the `product/get-context.html` partial - -## Related Resources - -- **`api-docs/README.md`**: API documentation workflow, tags.yml format, overlays, generation pipeline -- **cypress-e2e-testing** skill: E2E testing of UI components and pages -- **docs-cli-workflow** skill: Creating/editing documentation content -- **ts-component-dev** agent: TypeScript component behavior and interactivity -- **ui-testing** agent: Cypress E2E testing for UI components +Load the route-specific reference only when that condition applies. diff --git a/.agents/skills/hugo-template-dev/references/product-data.md b/.agents/skills/hugo-template-dev/references/product-data.md new file mode 100644 index 0000000000..f47733e761 --- /dev/null +++ b/.agents/skills/hugo-template-dev/references/product-data.md @@ -0,0 +1,4 @@ +# Product data + +Store product facts in `data/products.yml` and resolve the page product through +the established product context partials. Do not infer a product key from URLs. diff --git a/.agents/skills/hugo-template-dev/references/runtime-testing.md b/.agents/skills/hugo-template-dev/references/runtime-testing.md new file mode 100644 index 0000000000..d70b3044d6 --- /dev/null +++ b/.agents/skills/hugo-template-dev/references/runtime-testing.md @@ -0,0 +1,5 @@ +# Runtime testing + +Run a Hugo build for syntax, then Cypress for browser behavior. Check rendered +markup, console errors, interactions, and error paths where the change affects +them. diff --git a/.agents/skills/hugo-template-dev/references/shortcodes.md b/.agents/skills/hugo-template-dev/references/shortcodes.md new file mode 100644 index 0000000000..c500b55acb --- /dev/null +++ b/.agents/skills/hugo-template-dev/references/shortcodes.md @@ -0,0 +1,4 @@ +# Shortcodes + +Follow neighboring implementations, add an example to `content/example.md`, and +update `DOCS-SHORTCODES.md` for a public shortcode. diff --git a/.agents/skills/influxdb3-test-setup/SKILL.md b/.agents/skills/influxdb3-test-setup/SKILL.md index 4b55456dcc..e474bac10b 100644 --- a/.agents/skills/influxdb3-test-setup/SKILL.md +++ b/.agents/skills/influxdb3-test-setup/SKILL.md @@ -3,281 +3,22 @@ name: influxdb3-test-setup description: "Set up InfluxDB 3 Core and Enterprise instances for running documentation code block tests. Handles service initialization, worktree-specific databases, and test environment configuration. Use when preparing to run InfluxDB 3 code block tests, starting Core or Enterprise instances, or configuring .env.test and worktree-specific test databases." --- -# InfluxDB 3 Test Setup Skill +# InfluxDB 3 test setup -## Purpose +| Need | Route | +| --------------------------------- | ------------------------------------------------------ | +| Core test service | [references/core.md](references/core.md) | +| Enterprise test service | [references/enterprise.md](references/enterprise.md) | +| Credentials or worktree isolation | [references/environment.md](references/environment.md) | -This skill guides agents through setting up InfluxDB 3 Core and Enterprise instances for testing documentation code blocks. It covers service initialization, creating worktree-specific databases for test isolation, and configuring the test environment. +Prerequisites: Docker is running, the selected product is known, and the +worktree-specific `.env.test` is configured. Never reuse another worktree's +database or token. Do not start services unless code-block execution is actually +required; parse/compile lint needs neither service nor Docker. -## Architecture Overview - -``` -~/influxdata-docs/.influxdb3/ # Shared across all worktrees -├── enterprise/ -│ ├── .env # License email (INFLUXDB3_ENTERPRISE_LICENSE_EMAIL) -│ └── data/ # Enterprise data (persists license) -└── plugins/ # Shared plugins - -<worktree>/test/.influxdb3/ # Per-worktree (gitignored) -├── core/ -│ ├── .token # Core auth token -│ ├── data/ # Core data -│ └── plugins/ # Custom plugins -└── .env.test # Test credentials -``` - -**Key Design Decisions:** - -- **Core**: Per-worktree instance (port 8282) - data isolated to worktree -- **Enterprise**: Shared instance (port 8181) - license persists across worktrees -- **Databases**: Create worktree-named databases for test isolation on shared Enterprise - -## Quick Reference - -| Task | Command | -| ----------------------- | ---------------------------------------------------------------------- | -| Initialize Core | `./test/scripts/init-influxdb3.sh core` | -| Initialize Enterprise | `./test/scripts/init-influxdb3.sh enterprise` | -| Initialize both | `./test/scripts/init-influxdb3.sh all` | -| Check Core status | `curl -i http://localhost:8282/ping` | -| Check Enterprise status | `curl -i http://localhost:8181/ping -H "Authorization: Bearer $TOKEN"` | -| Run code block tests | `yarn test:codeblocks:v2` | - -## Setup Workflows - -### Workflow 1: Core Only (Per-Worktree) - -Use when testing Core-specific documentation or when you need complete isolation. - -```bash -# 1. Initialize Core -./test/scripts/init-influxdb3.sh core - -# 2. Verify it's running -curl -i http://localhost:8282/ping - -# 3. Get your token (JSON format) -jq -r .token test/.influxdb3/core/.token - -# 4. Create a database for testing -curl -X POST "http://localhost:8282/api/v3/configure/database" \ - -H "Authorization: Bearer $(jq -r .token test/.influxdb3/core/.token)" \ - -H "Content-Type: application/json" \ - -d '{"db": "test_db"}' +```sh +yarn test:codeblocks:influxdb3_core +yarn test:codeblocks:influxdb3_enterprise ``` -### Workflow 2: Enterprise (Shared Instance) - -Use when testing Enterprise-specific documentation. Creates worktree-named database for isolation. - -```bash -# 1. First-time only: Create .env file with license email -mkdir -p ~/influxdata-docs/.influxdb3/enterprise -echo 'INFLUXDB3_ENTERPRISE_LICENSE_EMAIL=your-email@example.com' > \ - ~/influxdata-docs/.influxdb3/enterprise/.env - -# 2. Initialize Enterprise (generates admin token and starts service) -./test/scripts/init-influxdb3.sh enterprise - -# 3. Get admin token from the generated file -ADMIN_TOKEN=$(jq -r .token ~/influxdata-docs/.influxdb3/enterprise/admin-token.json) - -# 4. Verify it's running -curl -i http://localhost:8181/ping \ - -H "Authorization: Bearer $ADMIN_TOKEN" - -# 5. Create worktree-named database for test isolation -WORKTREE_NAME=$(basename "$(pwd)" | tr '-' '_') -curl -X POST "http://localhost:8181/api/v3/configure/database" \ - -H "Authorization: Bearer $ADMIN_TOKEN" \ - -H "Content-Type: application/json" \ - -d "{\"db\": \"${WORKTREE_NAME}_db\"}" -``` - -### Workflow 3: Both Services - -Use when testing documentation that covers both Core and Enterprise. - -```bash -# Initialize both -./test/scripts/init-influxdb3.sh all - -# Verify both are running -curl -i http://localhost:8282/ping # Core (no auth by default) -curl -i http://localhost:8181/ping -H "Authorization: Bearer $TOKEN" # Enterprise -``` - -## Creating Worktree-Specific Databases - -When using the shared Enterprise instance, create databases named after the worktree to isolate test data: - -```bash -# Get worktree name (converts hyphens to underscores for valid DB names) -WORKTREE_NAME=$(basename "$(pwd)" | tr '-' '_') -echo "Database name: ${WORKTREE_NAME}_db" - -# Create database -curl -X POST "http://localhost:8181/api/v3/configure/database" \ - -H "Authorization: Bearer $ADMIN_TOKEN" \ - -H "Content-Type: application/json" \ - -d "{\"db\": \"${WORKTREE_NAME}_db\"}" - -# List databases to verify -curl "http://localhost:8181/api/v3/configure/database?format=json" \ - -H "Authorization: Bearer $ADMIN_TOKEN" -``` - -**Naming Convention:** - -- Worktree: `docs-v2-influxdb3-version-detection-headers` -- Database: `docs_v2_influxdb3_version_detection_headers_db` - -## Test Environment Configuration - -### Configure .env.test for Code Block Tests - -Create `content/<product>/.env.test` with test credentials: - -```bash -# For Enterprise testing -ADMIN_TOKEN=$(jq -r .token ~/influxdata-docs/.influxdb3/enterprise/admin-token.json) -cat > content/influxdb3/enterprise/.env.test << EOF -INFLUX_HOST=http://localhost:8181 -INFLUX_TOKEN=$ADMIN_TOKEN -INFLUX_DATABASE=YOUR_WORKTREE_DB -EOF - -# For Core testing -cat > content/influxdb3/core/.env.test << EOF -INFLUX_HOST=http://localhost:8282 -INFLUX_TOKEN=$(jq -r .token test/.influxdb3/core/.token) -INFLUX_DATABASE=test_db -EOF -``` - -### Run Code Block Tests - -```bash -# Test specific product -yarn test:codeblocks:v2 - -# Or run pytest directly -docker compose run --rm v2-pytest -``` - -## Troubleshooting - -### Enterprise Won't Start - -**Symptom:** Container exits immediately - -**Check:** - -```bash -# View container logs -docker logs influxdb3-enterprise - -# Common issues: -# 1. Missing .env file -ls -la ~/influxdata-docs/.influxdb3/enterprise/.env - -# 2. Wrong env var name (must be INFLUXDB3_ENTERPRISE_LICENSE_EMAIL) -cat ~/influxdata-docs/.influxdb3/enterprise/.env - -# 3. Missing admin-token.json (run init script to generate) -ls -la ~/influxdata-docs/.influxdb3/enterprise/admin-token.json -``` - -### Core Token Not Working - -**Symptom:** 401 Unauthorized - -**Check:** - -```bash -# Verify token file exists and is valid JSON -cat test/.influxdb3/core/.token -jq . test/.influxdb3/core/.token # Should parse without errors - -# Verify token is accessible in container (as secret) -docker exec influxdb3-core cat /run/secrets/influxdb3-core-token -``` - -### When Enterprise Is Unavailable - -If Enterprise won't start (license expired, port conflict, Docker issue): - -1. **Fix the root cause first** — check container logs, verify the license email - is valid (not a placeholder), and re-run the init script. -2. **Use Core for non-Enterprise features** — Core requires no license and can - verify most shared InfluxDB 3 behavior (queries, writes, databases, tables). -3. **Flag Enterprise-only gaps** — if the feature under test is Enterprise-only - (clustering, RBAC, table/database retention, read replicas), state explicitly - that runtime verification was not possible and why. Do not guess at behavior. - -### Port Already in Use - -**Symptom:** "port is already allocated" - -**Fix:** - -```bash -# Find what's using the port -lsof -i :8181 # Enterprise -lsof -i :8282 # Core - -# Stop existing containers -docker compose down influxdb3-enterprise influxdb3-core -``` - -### Getting the Admin Token - -The init script generates and saves admin tokens to JSON files for both Core and Enterprise: - -```bash -# Core token -cat test/.influxdb3/core/.token -jq -r .token test/.influxdb3/core/.token - -# Enterprise token -cat ~/influxdata-docs/.influxdb3/enterprise/admin-token.json -jq -r .token ~/influxdata-docs/.influxdb3/enterprise/admin-token.json - -# Export for use in commands -export INFLUXDB3_CORE_TOKEN=$(jq -r .token test/.influxdb3/core/.token) -export INFLUXDB3_ENTERPRISE_TOKEN=$(jq -r .token ~/influxdata-docs/.influxdb3/enterprise/admin-token.json) - -# Use in API calls -curl -i http://localhost:8282/ping -H "Authorization: Bearer $INFLUXDB3_CORE_TOKEN" -curl -i http://localhost:8181/ping -H "Authorization: Bearer $INFLUXDB3_ENTERPRISE_TOKEN" -``` - -**Token File Format** (both Core and Enterprise): - -```json -{ - "token": "64-character-hexadecimal-token", - "description": "Admin token for InfluxDB 3 Core/Enterprise" -} -``` - -## Service Comparison - -| Aspect | Core | Enterprise | -| ------------- | ---------------- | -------------------------------- | -| Port | 8282 | 8181 | -| Data location | Per-worktree | Shared | -| Auth default | Optional | Required | -| License | None | Trial/Paid | -| Use case | Isolated testing | Shared testing with worktree DBs | - -## Related Files - -- **Init script**: `test/scripts/init-influxdb3.sh` -- **Docker Compose**: `compose.yaml` (services: influxdb3-core, influxdb3-enterprise) -- **Test config**: `content/<product>/.env.test` - -## Related Skills - -- **cypress-e2e-testing** - For running E2E tests on documentation UI -- **docs-cli-workflow** - For creating/editing documentation content +Load the product reference only after choosing a runnable code-block test. diff --git a/.agents/skills/influxdb3-test-setup/references/core.md b/.agents/skills/influxdb3-test-setup/references/core.md new file mode 100644 index 0000000000..e0e84dc15d --- /dev/null +++ b/.agents/skills/influxdb3-test-setup/references/core.md @@ -0,0 +1,4 @@ +# Core setup + +Use the worktree's Core service configuration and obtain its test token from the +configured local service. Keep test database names worktree-specific. diff --git a/.agents/skills/influxdb3-test-setup/references/enterprise.md b/.agents/skills/influxdb3-test-setup/references/enterprise.md new file mode 100644 index 0000000000..bd7b3b26b7 --- /dev/null +++ b/.agents/skills/influxdb3-test-setup/references/enterprise.md @@ -0,0 +1,4 @@ +# Enterprise setup + +Use the Enterprise compose service and the worktree-specific environment. Verify +the service health before running Enterprise code-block tests. diff --git a/.agents/skills/influxdb3-test-setup/references/environment.md b/.agents/skills/influxdb3-test-setup/references/environment.md new file mode 100644 index 0000000000..143f816838 --- /dev/null +++ b/.agents/skills/influxdb3-test-setup/references/environment.md @@ -0,0 +1,4 @@ +# Environment + +Keep `.env.test` local and never copy credentials from another worktree. Stop +after setup if required secrets or Docker services are unavailable. diff --git a/.agents/skills/vale-linting/SKILL.md b/.agents/skills/vale-linting/SKILL.md index 7d3b18ff75..890e89f7c0 100644 --- a/.agents/skills/vale-linting/SKILL.md +++ b/.agents/skills/vale-linting/SKILL.md @@ -3,438 +3,22 @@ name: vale-linting description: "Run, debug, and maintain Vale style linting for InfluxData documentation: run Vale, interpret alerts, manage vocabulary, configure product-specific .vale.ini files, and understand which rules are enabled and why. Use when running Vale, investigating or fixing Vale warnings, adding accept/ignore vocabulary terms, or setting up a product config. To author custom rule patterns and regex, see vale-rule-config." --- -# Vale Style Linting Workflow +# Vale linting -## Purpose +| Need | Route | +| ----------------------- | ---------------------------------------------------------- | +| Run a content check | [references/running-vale.md](references/running-vale.md) | +| Select product config | [references/configuration.md](references/configuration.md) | +| Add accepted vocabulary | [references/vocabulary.md](references/vocabulary.md) | +| Create a rule | `../vale-rule-config/SKILL.md` | -This skill covers the complete Vale linting workflow for InfluxData documentation, including running Vale, understanding the rule configuration, adding vocabulary terms, and creating custom rules. +Prerequisites: choose the config from the content product and treat hook output +as the normal content lint. Fix source wording before adding vocabulary or +suppression. Run Vale directly only to diagnose or validate the specific change. -**Use this skill when:** - -- Running Vale style checks on documentation -- Debugging Vale warnings or errors -- Adding terms to the vocabulary (accept/ignore lists) -- Creating or modifying custom Vale rules -- Understanding why certain patterns are flagged - -## Quick Reference - -```bash -# Run Vale on specific files -.ci/vale/vale.sh --config=.vale.ini content/influxdb3/core/**/*.md - -# Run with minimum alert level -.ci/vale/vale.sh --config=.vale.ini --minAlertLevel=warning content/path/ - -# Sync Vale packages (after .vale.ini changes) -.ci/vale/vale.sh sync - -# Show Vale configuration -.ci/vale/vale.sh ls-config -``` - -## Part 1: How Vale Runs - -### Execution via `.ci/vale/vale.sh` - -The wrapper script `.ci/vale/vale.sh` runs Vale using: - -1. **Local binary** (preferred) — if `vale` is installed and version >= 3.x -2. **Docker fallback** — `jdkato/vale:v${VALE_VERSION}` (pinned version), - only if the Docker daemon is running -3. **Graceful skip** — if neither is available, the wrapper prints a warning - and exits 0 so other pre-commit hooks still run - -```bash -# The wrapper handles binary vs Docker automatically -.ci/vale/vale.sh --config=.vale.ini content/path/ - -# In CI, the pr-vale-check.yml workflow installs the Vale binary -# directly (reads version from vale.sh), so Docker is not needed. -``` - -**Sandboxed agent sessions (no Docker daemon, GitHub release downloads -blocked):** Vale can't run locally, so the wrapper skips it (exit 0) with a -warning. Do NOT use `git commit --no-verify` — that bypasses all hooks, not -just Vale. Let the commit proceed normally; the `pr-vale-check.yml` workflow -is the authoritative gate and blocks merge on errors. It sets `VALE_STRICT=1`, -which makes the wrapper fail instead of skip if Vale is ever unavailable in -CI. To run Vale in a sandbox anyway, ask an environment admin to allowlist -the Vale release host (`github.com/vale-cli/vale`) in the environment's -network policy. - -**Critical limitation:** Only files inside the repository are accessible when using Docker fallback. Files in `/tmp` or other external paths will silently fail. - -**macOS note:** The CI script `.github/scripts/vale-check.sh` uses `declare -A` (associative arrays) which requires bash 4+. macOS ships bash 3.2. Use `/opt/homebrew/bin/bash` or run tests in CI instead. - -### Configuration Files - -| File | Purpose | -| ----------------------------------------------------- | --------------------------------------------------- | -| `.vale.ini` | Main configuration (styles, packages, rule toggles) | -| `.ci/vale/styles/InfluxDataDocs/` | Custom rules for InfluxData docs | -| `.ci/vale/styles/config/vocabularies/InfluxDataDocs/` | Vocabulary (accept/reject terms) | -| `content/*/.vale.ini` | Product-specific overrides | - -## Part 2: Understanding Vale Rules - -### Rule Sources - -Vale uses multiple rule sources, configured in `.vale.ini`: - -```ini -BasedOnStyles = Vale, InfluxDataDocs, Google, write-good -``` - -1. **Vale** - Built-in rules (Spelling, Terms, Repetition) -2. **InfluxDataDocs** - Custom rules for this repo -3. **Google** - Google Developer Documentation Style Guide -4. **write-good** - Plain English suggestions - -### Disabled Rules (and Why) - -Rules are disabled in two categories across `.vale.ini` and all product configs: - -**Mechanical rules disabled** (replaced by custom equivalents or incompatible with InfluxDB syntax): - -| Rule | Reason | -| ------------------- | ------------------------------------------------------------------------------------------------- | -| `Google.Acronyms` | Custom `InfluxDataDocs.Acronyms` handles this | -| `Google.DateFormat` | Custom `InfluxDataDocs.DateFormat` handles this | -| `Google.Ellipses` | Custom `InfluxDataDocs.Ellipses` handles this | -| `Google.Headings` | Too strict for technical doc headings | -| `Google.WordList` | Custom `InfluxDataDocs.WordList` handles this | -| `Google.Units` | Flags InfluxDB duration literals (30d, 24h); custom `InfluxDataDocs.Units` checks byte units only | -| `Vale.Spelling` | Custom `InfluxDataDocs.Spelling` handles this | -| `Vale.Terms` | False positives from URLs, file paths, and code | - -**Style rules disabled** (high false-positive rate in technical docs): - -| Rule | Reason | -| --------------------- | ------------------------------------------------------- | -| `Google.Contractions` | Not relevant to InfluxData style | -| `Google.FirstPerson` | Tutorials use "I" intentionally | -| `Google.Passive` | Technical docs use passive voice legitimately | -| `Google.We` | "We recommend" is standard in docs | -| `Google.Will` | Future tense is standard in docs | -| `write-good.Cliches` | High false positive rate | -| `write-good.Passive` | Duplicate of Google.Passive concern | -| `write-good.So` | Starting with "So" is fine | -| `write-good.ThereIs` | Often the clearest phrasing | -| `write-good.TooWordy` | Flags legitimate terms: aggregate, expiration, multiple | -| `write-good.Weasel` | Context-dependent, better handled during content review | - -### Active Custom Rules - -| Rule | Purpose | -| ------------------------------- | --------------------------------------------- | -| `InfluxDataDocs.Spelling` | Spell checking with technical term exclusions | -| `InfluxDataDocs.Units` | Byte units only (allows duration literals) | -| `InfluxDataDocs.WordList` | Terminology standards (admin → administrator) | -| `InfluxDataDocs.Capitalization` | Heading case rules | -| `InfluxDataDocs.Branding` | Product name consistency | - -## Part 3: Adding Vocabulary Terms - -### Accept List (Technical Terms) - -Add terms to `.ci/vale/styles/config/vocabularies/InfluxDataDocs/accept.txt`: - -```text -# Case-insensitive by default -CPUs -systemd -preconfigured - -# Regex patterns for variations -[Dd]ownsampl(e|ed|es|ing) -subprocess(es)? - -# Exact case matching (regex) -(?i)InfluxQL -``` - -### Ignore List (Spelling Only) - -Add terms to `.ci/vale/styles/InfluxDataDocs/Terms/ignore.txt`: - -```text -# Simple words (case-insensitive) -cgroup -humantime -deadman - -# The ignore.txt is referenced by Spelling.yml -``` - -**Key difference in this repo:** - -- `accept.txt` - Terms that are part of the shared spelling vocabulary (via `Vocab = InfluxDataDocs`). In this repository, it does **not** currently create substitution rules because `Vale.Terms` is disabled in `.vale.ini`. If `Vale.Terms` is enabled in the future, these terms may also drive substitution behavior. -- `ignore.txt` - Additional terms to skip in spell checking only (an ignore list layered on top of the shared vocabulary; no substitution rules). - -### When to Use Each - -| Scenario | File | -| --------------------------------------- | ------------ | -| Technical term that's spelled correctly | `ignore.txt` | -| Preferred capitalization (API, CLI) | `accept.txt` | -| Product name with specific casing | `accept.txt` | -| Variable name appearing in prose | `ignore.txt` | - -## Part 4: Creating Custom Rules - -> This section covers where rules live in this repo and shows two real -> examples. For the rule-authoring deep dive — rule types, the regexp2 engine, -> PCRE lookarounds, and testing patterns in isolation — use the -> **vale-rule-config** skill. - -### Rule File Location - -Custom rules go in `.ci/vale/styles/InfluxDataDocs/`: - -``` -.ci/vale/styles/InfluxDataDocs/ -├── Acronyms.yml -├── Branding.yml -├── Capitalization.yml -├── Ellipses.yml -├── Spelling.yml -├── Units.yml # Custom: allows duration literals -├── WordList.yml -└── Terms/ - ├── ignore.txt - └── query-functions.txt +```sh +.ci/vale/vale.sh --minAlertLevel=error <files> ``` -### Example: Custom Units Rule - -This rule validates byte units while allowing InfluxDB duration literals: - -```yaml -# .ci/vale/styles/InfluxDataDocs/Units.yml -extends: existence -message: "Put a nonbreaking space between the number and the unit in '%s'." -link: "https://developers.google.com/style/units-of-measure" -nonword: true -level: warning -# Only check byte units. Duration units (ns, ms, s, min, h, d) are excluded -# because InfluxDB duration literals use no space (e.g., 30d, 24h, 1h). -tokens: - - \b\d+(?:B|kB|MB|GB|TB|PB) -``` - -### Example: Spelling Rule with Scope Exclusions - -```yaml -# .ci/vale/styles/InfluxDataDocs/Spelling.yml -extends: spelling -message: "Did you really mean '%s'?" -level: warning -# Exclude from spell checking: -scope: - - ~code # Fenced code blocks - - ~raw # Inline code - - ~table.header - - ~table.cell -ignore: - - InfluxDataDocs/Terms/ignore.txt - - InfluxDataDocs/Terms/query-functions.txt -filters: - # Ignore URL paths - - '/[a-zA-Z0-9/_\-\.\{\}]+' - # Ignore full URLs - - 'https?://[^\s\)\]>"]+' -``` - -## Part 5: Debugging Vale Issues - -### Common Problems - -**"0 errors in stdin"** - -- File is outside the repository (Docker can't access it) -- Solution: Use files inside the repo for testing - -**Rule not triggering** - -- Check if rule is disabled in `.vale.ini` -- Verify rule file has valid YAML syntax -- Run `vale sync` after adding new packages - -**False positives in URLs/code** - -- Add patterns to `TokenIgnores` in `.vale.ini` -- Add scope exclusions (`~code`, `~raw`) to the rule -- Add terms to `ignore.txt` - -### Debugging Commands - -```bash -# Show all configuration -.ci/vale/vale.sh ls-config - -# Validate YAML syntax -node -e "require('js-yaml').load(require('fs').readFileSync('path/to/rule.yml'))" - -# Test specific file -.ci/vale/vale.sh --config=.vale.ini --minAlertLevel=suggestion path/to/file.md -``` - -## Part 6: Vale Cannot Inspect URLs - -`TokenIgnores` in `.vale.ini` strips all URLs before any rules run: - -```ini -TokenIgnores = https?://[^\s\)\]>"]+ -``` - -**This means no Vale rule can match URL content.** An earlier attempt to create a `SupportLink.yml` rule to validate `support.influxdata.com` URL patterns failed for this reason — the URLs were stripped before the rule could see them. Support URL validation uses a separate shell script (`.ci/scripts/check-support-links.sh`) instead. - -Keep this in mind when designing rules: if the pattern to match is inside a URL, use a shell script or pre-commit hook, not a Vale rule. - -## Part 7: TokenIgnores vs Rule Filters - -### TokenIgnores (in .vale.ini) - -Applied globally to all rules. Matches **whole tokens**: - -```ini -TokenIgnores = /[a-zA-Z0-9/_\-\.]+, \ - https?://[^\s\)\]>"]+, \ - `[^`]+` -``` - -**Limitation:** Cannot match substrings within words. If "api" appears in `/api/v3/write`, the URL pattern must match the entire URL to exclude it. - -### Rule Filters (in rule YAML) - -Applied to specific rules. Can match **patterns within text**: - -```yaml -filters: - - '[Ss]erverless' # Allow both cases - - '/[a-zA-Z0-9/_\-\.\{\}]+' # URL paths -``` - -### When to Use Each - -| Use Case | Approach | -| ---------------------------------- | ------------------------------------ | -| Exclude entire URLs from all rules | `TokenIgnores` | -| Exclude inline code from all rules | `TokenIgnores` with backtick pattern | -| Exclude patterns from one rule | Rule-specific `filters` | -| Skip checking inside code blocks | Rule `scope: [~code]` | - -## Part 7: Workflow Examples - -### Adding a New Technical Term - -```bash -# 1. Identify the term and its usage -grep -r "systemd" content/ - -# 2. Add to ignore list (spelling only) -echo "systemd" >> .ci/vale/styles/InfluxDataDocs/Terms/ignore.txt - -# 3. Or add to accept.txt (if it should influence Vale.Terms) -echo "systemd" >> .ci/vale/styles/config/vocabularies/InfluxDataDocs/accept.txt - -# 4. Test the change -.ci/vale/vale.sh --config=.vale.ini content/path/with/term.md -``` - -### Creating a Product-Specific Override - -> \[!Important] -> Product-specific `.vale.ini` files must include the same disabled rules as the -> root `.vale.ini`. Rules disabled in the root config are **not** inherited by -> product-specific configs. Omitting them re-enables the rules for those products. -> For example, omitting `Google.Units = NO` causes duration literals like `7d`, -> `24h` to be flagged as errors in product-specific linting runs. - -```bash -# 1. Create product-specific .vale.ini -cat > content/influxdb3/cloud-dedicated/.vale.ini << 'EOF' -StylesPath = ../../../.ci/vale/styles -MinAlertLevel = warning -Vocab = InfluxDataDocs - -Packages = Google, write-good, Hugo - -[*.md] -BasedOnStyles = Vale, InfluxDataDocs, Cloud-Dedicated, Google, write-good - -# --- Disabled mechanical rules --- -Google.Acronyms = NO -Google.DateFormat = NO -Google.Ellipses = NO -Google.Headings = NO -Google.WordList = NO -Google.Units = NO -Vale.Spelling = NO -Vale.Terms = NO - -# --- Disabled style rules (high false-positive rate in technical docs) --- -Google.Contractions = NO -Google.FirstPerson = NO -Google.Passive = NO -Google.We = NO -Google.Will = NO -write-good.Cliches = NO -write-good.Passive = NO -write-good.So = NO -write-good.ThereIs = NO -write-good.TooWordy = NO -write-good.Weasel = NO - -TokenIgnores = /[a-zA-Z0-9/_\-\.]+, \ - https?://[^\s\)\]>"]+, \ - `[^`]+` -EOF - -# 2. Run Vale with product config -.ci/vale/vale.sh --config=content/influxdb3/cloud-dedicated/.vale.ini \ - content/influxdb3/cloud-dedicated/**/*.md -``` - -### Debugging Why a Pattern is Flagged - -```bash -# 1. Check which rule is triggering -.ci/vale/vale.sh --config=.vale.ini path/to/file.md -# Output shows: InfluxDataDocs.WordList, Vale.Terms, etc. - -# 2. Read the rule file -cat .ci/vale/styles/InfluxDataDocs/WordList.yml - -# 3. Check if term is in vocabulary -grep -i "term" .ci/vale/styles/config/vocabularies/InfluxDataDocs/*.txt - -# 4. Check TokenIgnores patterns -grep TokenIgnores .vale.ini -``` - -## Checklist: Before Committing Vale Changes - -- [ ] Ran `vale sync` if packages changed -- [ ] Tested changes on sample files -- [ ] Verified no unexpected rules are disabled -- [ ] Added comments explaining why rules are disabled -- [ ] Kept vocabulary files alphabetized -- [ ] Used ignore.txt for spelling-only terms -- [ ] Used accept.txt for terms that should influence substitution - -## Related Files - -| File | Purpose | -| ------------------------------------------- | ------------------------------------------------------- | -| `.vale.ini` | Main configuration | -| `.vale-instructions.ini` | Config for non-content files (READMEs, AGENTS.md, etc.) | -| `.ci/vale/vale.sh` | Vale wrapper (local binary or Docker fallback) | -| `.ci/vale/styles/` | All Vale style rules | -| `.ci/scripts/check-support-links.sh` | Support URL validation (can't use Vale — see Part 6) | -| `.github/scripts/vale-check.sh` | CI script: groups files by product config, runs Vale | -| `.github/scripts/resolve-shared-content.sh` | CI script: resolves `content/shared/*` to product pages | -| `.github/workflows/pr-vale-check.yml` | CI workflow: runs Vale on PR changes | -| `lefthook.yml` | Pre-commit hooks that run Vale | -| `DOCS-TESTING.md` | Testing documentation (includes Vale CI section) | +Load the matching reference only when configuration, vocabulary, or alert detail +is needed. diff --git a/.agents/skills/vale-linting/references/configuration.md b/.agents/skills/vale-linting/references/configuration.md new file mode 100644 index 0000000000..e3deacb528 --- /dev/null +++ b/.agents/skills/vale-linting/references/configuration.md @@ -0,0 +1,5 @@ +# Configuration + +Use the default `.vale.ini` unless the product owns a scoped `.vale.ini`. +Configuration chooses styles and vocabulary; do not copy rules to work around a +mis-selected config. diff --git a/.agents/skills/vale-linting/references/running-vale.md b/.agents/skills/vale-linting/references/running-vale.md new file mode 100644 index 0000000000..2e0babae2f --- /dev/null +++ b/.agents/skills/vale-linting/references/running-vale.md @@ -0,0 +1,4 @@ +# Running Vale + +Use the repository wrapper, select the product config when required, and focus +on changed files. Hook output normally supplies this check at commit time. diff --git a/.agents/skills/vale-linting/references/vocabulary.md b/.agents/skills/vale-linting/references/vocabulary.md new file mode 100644 index 0000000000..ff59c0c957 --- /dev/null +++ b/.agents/skills/vale-linting/references/vocabulary.md @@ -0,0 +1,4 @@ +# Vocabulary + +Add only stable proper nouns and necessary technical terms to the appropriate +vocabulary. Prefer correcting prose over suppressing a legitimate alert. diff --git a/.agents/skills/vale-rule-config/SKILL.md b/.agents/skills/vale-rule-config/SKILL.md index 7afb2fdc5a..71b5df6993 100644 --- a/.agents/skills/vale-rule-config/SKILL.md +++ b/.agents/skills/vale-rule-config/SKILL.md @@ -3,515 +3,22 @@ name: vale-rule-config description: "Author and test custom Vale rules for InfluxData documentation: rule types (existence, substitution, conditional), the regexp2 engine and PCRE lookarounds, and testing rule patterns in isolation. Use when writing a new Vale rule, debugging why a rule pattern does not match, or working with Vale regex. To run Vale, manage vocabulary, or fix flagged content, see vale-linting." --- -# Vale Rule Configuration +# Vale rule configuration -## Purpose +| Need | Route | +| -------------------- | ---------------------------------------------------- | +| Choose rule type | [references/rule-types.md](references/rule-types.md) | +| Write or debug regex | [references/regex.md](references/regex.md) | +| Test a rule | [references/testing.md](references/testing.md) | -This skill guides CI/Quality Engineers in writing, testing, and maintaining Vale style linting rules for the InfluxData documentation. It covers Vale's regex engine, rule syntax, configuration files, and best practices for creating effective style rules. +Prerequisites: establish that existing wording, vocabulary, or a current rule +cannot solve the issue. Keep rule scope narrow and test positive, negative, and +near-miss examples. Vale uses regexp2 semantics; do not assume JavaScript regex +behavior. -**Use this skill when:** - -- Writing new Vale rules (existence, substitution, etc.) -- Debugging Vale rule patterns that aren't working -- Understanding Vale's regex capabilities -- Configuring Vale for product-specific style guides -- Managing vocabulary and branding terms - -**For content editors who just need to run Vale and fix issues**, see the **content-editing** skill instead. - -## Quick Decision Tree - -``` -Writing a new Vale rule? -├─ Simple pattern? Use tokens (See Part 1: Basic Rules) -└─ Complex pattern? Use raw or check regex engine (See Part 2: Regex Engine) - -Rule not matching as expected? -├─ Check Vale's regex flavor (See Part 2: Regex Engine) -└─ Test pattern in isolation (See Part 5: Testing) - -Need product-specific terms? -└─ Add to vocabulary files (See Part 3: Vocabulary) - -Need to configure Vale for a product? -└─ Create .vale.ini (See Part 4: Configuration) -``` - -## Part 1: Basic Vale Rule Types - -Vale supports several rule types. Here are the most common: - -### Existence Rules - -Checks if certain patterns exist in the text. - -```yaml -# .ci/vale/styles/InfluxDataDocs/BadWords.yml -extends: existence -message: "Don't use '%s'" -level: error -tokens: - - obviously - - basically - - simply -``` - -**With `nonword: true` for punctuation:** - -```yaml -# .ci/vale/styles/Google/Colons.yml -extends: existence -message: "'%s' should be in lowercase." -link: 'https://developers.google.com/style/colons' -nonword: true -level: warning -scope: sentence -tokens: - - ':\s[A-Z]' -``` - -### Substitution Rules - -Suggests replacements for problematic patterns. - -Substitution rules in this repo expand abbreviations rather than introduce -them (the real terminology rule lives in -`.ci/vale/styles/InfluxDataDocs/WordList.yml`): - -```yaml -# Illustrative — mirrors the style of InfluxDataDocs/WordList.yml -extends: substitution -message: "Use '%s' instead of '%s'" -level: warning -swap: - admin: administrator - repo: repository -``` - -### Conditional Rules - -More complex rules with exceptions. - -```yaml -extends: conditional -first: '\b(if|when)\b' -second: '\bthen\b' -message: "If/when statements should include 'then'" -level: warning -``` - -## Part 2: Vale's Regex Engine - -### Critical: Vale Uses regexp2, Not RE2 - -Vale uses the [regexp2](https://pkg.go.dev/github.com/dlclark/regexp2) library, **not** Go's standard `regexp` package (which uses RE2). This is a common source of confusion because Vale is written in Go. - -### Supported Regex Features - -Vale supports **PCRE-style lookarounds** despite being written in Go: - -- ✅ **Positive lookahead**: `(?=re)` -- ✅ **Negative lookahead**: `(?!re)` -- ✅ **Positive lookbehind**: `(?<=re)` -- ✅ **Negative lookbehind**: `(?<!re)` -- ✅ **Lazy quantifiers**: `*?`, `+?`, `??` -- ✅ **Named groups**: `(?P<name>...)` -- ✅ **Atomic groups**: `(?>...)` - -According to Vale's maintainer: - -> "Vale uses a superset of the Go flavor, supporting PCRE-style lookarounds." - -### Example: Negative Lookbehind - -This pattern matches a colon followed by uppercase letter, but NOT when the colon is part of a URL scheme (like `https:`): - -```yaml -extends: existence -message: "'%s' should be in lowercase." -nonword: true -scope: sentence -tokens: - # ✅ This works! Negative lookbehind is supported - - '(?<!:[^ ]+?):\s[A-Z]' -``` - -**How it works:** - -- `(?<!:[^ ]+?)` - Negative lookbehind: NOT preceded by `:` followed by non-space characters -- `:\s[A-Z]` - Colon, whitespace, uppercase letter - -### Example: Positive Lookbehind - -Match "Internet" only when preceded by whitespace, excluding specific phrases: - -```yaml -extends: existence -message: "'%s' should only be capitalized when starting a sentence." -scope: sentence -tokens: - - '(?<=\s)Internet(?! Service Provider| Protocol)' -``` - -### Critical Limitation: Vale Cannot Match URLs - -`TokenIgnores` in `.vale.ini` strips all URLs before rules run. **No rule — `existence`, `substitution`, or `raw` — can match content inside a URL.** This applies globally and cannot be overridden per-rule. - -For URL pattern validation (e.g., enforcing canonical support URLs), use a shell script or pre-commit hook instead of a Vale rule. See `.ci/scripts/check-support-links.sh` for an example. - -### tokens vs raw - -**tokens:** - -- Automatically wrapped in word boundaries -- Converted to non-capturing groups -- Good for simple patterns - -**raw:** - -- Full control over the pattern -- No automatic processing -- Use for complex regex - -```yaml -# Using raw for full control -extends: existence -message: "Use 'database' instead" -raw: - - '\bDB\b(?!\s+instance)' # DB but not "DB instance" -``` - -## Part 3: Vocabulary Management - -Vocabulary files manage accepted and rejected terms across the documentation. - -### File Locations - -``` -.ci/vale/styles/config/vocabularies/ -└── InfluxDataDocs/ - ├── accept.txt # Accepted terms (won't be flagged) - └── reject.txt # Rejected terms (will be flagged) - -# Only InfluxDataDocs exists today. To add a product-specific vocabulary, -# create a sibling directory (for example, Cloud-Dedicated/) with its own -# accept.txt/reject.txt and reference it via Vocab in the product .vale.ini. -``` - -### accept.txt Format - -One term per line. Case-sensitive by default: - -```text -InfluxDB -InfluxQL -Telegraf -ClickHouse -PostgreSQL -``` - -**Support for regex patterns:** - -```text -# Accept both capitalizations -[Dd]atabase -[Aa]PI - -# Accept with word boundaries -\bDB\b -``` - -### reject.txt Format - -Rejected terms that should never be used: - -```text -Influx -influxdb (lowercase) -big data -simply -obviously -``` - -### Product-Specific Vocabulary - -Create product-specific vocabularies by: - -1. Creating a new vocabulary directory in - `.ci/vale/styles/config/vocabularies/` -2. Adding `accept.txt` and `reject.txt` -3. Configuring in product's `.vale.ini` - -**Example:** - -```yaml -# content/influxdb3/cloud-dedicated/.vale.ini -StylesPath = ../../../.ci/vale/styles -Vocab = Cloud-Dedicated - -[*.md] -BasedOnStyles = Vale, InfluxDataDocs, Cloud-Dedicated, Google, write-good -``` - -## Part 4: Vale Configuration Files - -### Repository-Level Config - -`.vale.ini` in repository root: - -```ini -StylesPath = .ci/vale/styles -MinAlertLevel = suggestion -Vocab = InfluxDataDocs - -[*.md] -BasedOnStyles = Vale, InfluxDataDocs, Google, write-good -``` - -### Product-Specific Config - -Product configs must mirror all disabled rules from root `.vale.ini` (rules disabled in root are NOT inherited). See the `vale-linting` skill for a complete product config example with all disabled rules. - -### Rule Configuration - -Individual rules are YAML files in style directories: - -``` -.ci/vale/styles/ -├── Google/ -│ ├── Colons.yml -│ ├── Headings.yml -│ └── ... -├── InfluxDataDocs/ -│ ├── Branding.yml -│ ├── WordList.yml -│ └── ... -└── config/ - └── vocabularies/ -``` - -## Part 5: Testing Vale Rules - -### Test a Rule in Isolation - -```bash -# Test specific rule on one file -.ci/vale/vale.sh \ - --config=.vale.ini \ - --minAlertLevel=suggestion \ - content/influxdb3/core/get-started/_index.md - -# Test only error-level issues -.ci/vale/vale.sh \ - --config=content/influxdb3/cloud-dedicated/.vale.ini \ - --minAlertLevel=error \ - content/influxdb3/cloud-dedicated/**/*.md -``` - -### Test Rule Pattern Before Adding to Vale - -You can test regex patterns with Python or online tools first: - -```python -import re - -# Test negative lookbehind pattern -pattern = r'(?<!:[^ ]+?):\s[A-Z]' -text = "Install the package: Then run it." - -matches = re.findall(pattern, text) -print(matches) # Should match ": T" - -# Should NOT match URL schemes -text2 = "Visit https://example.com" -matches2 = re.findall(pattern, text2) -print(matches2) # Should be empty -``` - -### Common Issues - -**Pattern not matching:** - -1. Check if you need `nonword: true` for punctuation -2. Verify scope is appropriate (`sentence`, `heading`, etc.) -3. Test with `raw` instead of `tokens` for complex patterns - -**Too many false positives:** - -1. Add exceptions using negative lookahead/lookbehind -2. Adjust scope to be more specific -3. Consider using substitution rule with exceptions - -**Pattern works in Python but not Vale:** - -- Unlikely if you're using PCRE features (Vale supports them) -- Check for differences in whitespace handling -- Try `raw` field for exact pattern control - -## Part 6: Advanced Patterns - -### Excluding Specific Contexts - -```yaml -# Match "database" but not "database instance" or "database cluster" -extends: existence -message: "Use 'DB' for brevity" -tokens: - - '\bdatabase\b(?! instance| cluster)' -``` - -### Case-Insensitive Matching - -```yaml -extends: existence -message: "Use 'InfluxDB' with proper capitalization" -tokens: - - '(?i)influx ?db' # Matches influxdb, influx db, INFLUXDB, etc. -``` - -### Multiple Conditions - -```yaml -extends: conditional -first: '\b(will|shall)\b' -second: '(?:not|n''t)\b' -message: "Use 'won't' or 'will not' consistently" +```sh +.ci/vale/vale.sh --minAlertLevel=warning <fixture-or-file> ``` -### Capture Groups for Messages - -```yaml -extends: substitution -message: "Use '%s' instead of '%s'" -swap: - '(\w+)base': '$1-base' # Changes 'database' to 'data-base' -``` - -## Part 7: Best Practices - -### DO: - -- **Start simple**: Use `tokens` before moving to `raw` -- **Test incrementally**: Add patterns one at a time -- **Use vocabulary files**: For spelling and branding terms -- **Document patterns**: Add comments explaining complex regex -- **Be specific**: Use lookarounds to reduce false positives -- **Check scope**: Use appropriate scope (sentence, heading, etc.) - -### DON'T: - -- **Assume RE2 limitations**: Vale supports lookarounds -- **Over-complicate**: Sometimes simpler patterns work better -- **Ignore performance**: Complex patterns can slow down linting -- **Skip testing**: Always test rules on real content first -- **Forget edge cases**: Test with URLs, code blocks, etc. - -## Part 8: Reference - -### Alert Levels - -- `error`: Critical issues (broken links, branding violations) -- `warning`: Style guide rules -- `suggestion`: Optional improvements - -### Common Scopes - -- `text`: All text content -- `sentence`: Individual sentences -- `paragraph`: Full paragraphs -- `heading`: Heading text only -- `list`: List items only -- `code`: Code blocks (rarely used) - -### Rule Types - -- `existence`: Check if patterns exist -- `substitution`: Suggest replacements -- `conditional`: If X then Y must also exist -- `consistency`: Enforce consistent usage -- `occurrence`: Limit pattern occurrences -- `repetition`: Check for repeated words -- `sequence`: Check word ordering - -## Part 9: Example: Creating a New Rule - -Let's create a rule to enforce "InfluxDB 3" instead of "InfluxDB v3": - -### Step 1: Create the rule file - -```bash -# Create new rule -cat > .ci/vale/styles/InfluxDataDocs/InfluxDB3Version.yml <<'EOF' -extends: substitution -message: "Use '%s' instead of '%s'" -level: warning -link: 'https://docs.influxdata.com/style-guide/#version-names' -swap: - 'InfluxDB v3': 'InfluxDB 3' - 'InfluxDB V3': 'InfluxDB 3' -EOF -``` - -### Step 2: Test on sample content - -```bash -# Test on one file first -.ci/vale/vale.sh content/influxdb3/core/get-started/_index.md -``` - -### Step 3: Refine if needed - -If too many false positives, add exceptions: - -```yaml -extends: existence -message: "Use 'InfluxDB 3' instead of 'InfluxDB v3'" -level: warning -tokens: - # Match "InfluxDB v3" but not in URLs or code - - 'InfluxDB v3(?![`/])' -``` - -### Step 4: Run on full product - -```bash -# Test on entire product -.ci/vale/vale.sh content/influxdb3/**/*.md -``` - -## Related Skills - -- **content-editing** - For content editors who need to run Vale and fix issues -- **cypress-e2e-testing** - For testing documentation after style fixes - -## Resources - -### Official Documentation - -- [Vale documentation](https://vale.sh/docs/) -- [Vale styles guide](https://vale.sh/docs/topics/styles/) -- [regexp2 package](https://pkg.go.dev/github.com/dlclark/regexp2) - -### Community Resources - -- [Vale issue #233 - RFC on lookarounds](https://github.com/errata-ai/vale/issues/233) -- [Vale discussion #817 - Working lookbehind example](https://github.com/errata-ai/vale/discussions/817) -- [Google Developer Documentation Style Guide](https://developers.google.com/style) - -### Internal Documentation - -- [DOCS-CONTRIBUTING.md](../../DOCS-CONTRIBUTING.md) - Vale configuration section -- [DOCS-TESTING.md](../../DOCS-TESTING.md) - Vale testing procedures - -## Checklist: Before Adding a New Vale Rule - -- [ ] Pattern tested in isolation (Python/regex tool) -- [ ] Rule tested on sample content -- [ ] False positives identified and handled -- [ ] Appropriate alert level chosen (error/warning/suggestion) -- [ ] Documentation link added (if applicable) -- [ ] Rule tested on full product content -- [ ] Rule added to appropriate style directory -- [ ] Configuration updated if needed (.vale.ini) -- [ ] PR includes examples of rule in action -- [ ] Team reviewed for style guide alignment +For normal linting and vocabulary, load `../vale-linting/SKILL.md`. Load one +reference only when that specific authoring task requires it. diff --git a/.agents/skills/vale-rule-config/references/regex.md b/.agents/skills/vale-rule-config/references/regex.md new file mode 100644 index 0000000000..ebcf98884b --- /dev/null +++ b/.agents/skills/vale-rule-config/references/regex.md @@ -0,0 +1,4 @@ +# Regex + +Test regexp2 patterns independently. Keep captures and lookarounds minimal, and +escape YAML and regex syntax separately. diff --git a/.agents/skills/vale-rule-config/references/rule-types.md b/.agents/skills/vale-rule-config/references/rule-types.md new file mode 100644 index 0000000000..b3d06224b7 --- /dev/null +++ b/.agents/skills/vale-rule-config/references/rule-types.md @@ -0,0 +1,4 @@ +# Rule types + +Use substitution for replacements, existence for prohibited or required forms, +and conditional rules only when a context restriction is necessary. diff --git a/.agents/skills/vale-rule-config/references/testing.md b/.agents/skills/vale-rule-config/references/testing.md new file mode 100644 index 0000000000..a356ce2d6a --- /dev/null +++ b/.agents/skills/vale-rule-config/references/testing.md @@ -0,0 +1,4 @@ +# Testing rules + +Use a small fixture with matches, non-matches, and boundary cases. Run Vale with +the intended configuration and confirm the alert level and suggestion text. diff --git a/.claude/rules/content.md b/.claude/rules/content.md index ef13e4226d..a645636c75 100644 --- a/.claude/rules/content.md +++ b/.claude/rules/content.md @@ -7,210 +7,27 @@ paths: <!-- Run 'yarn build:agent:instructions' to regenerate it. --> -# Content File Guidelines - -**Frontmatter reference**: [DOCS-FRONTMATTER.md](../../DOCS-FRONTMATTER.md) -**Shortcodes reference**: [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) -**Working examples**: [content/example.md](../../content/example.md) - -**For complete content editing workflow**, see -[content-editing skill](../../.agents/skills/content-editing/SKILL.md) which covers: - -- Creating and editing content with CLI tools -- Shared content management and testing -- Fact-checking with MCP server -- Complete validation workflows - -## CLI Tools for Content Workflow - -The unified `docs` CLI provides tools for content creation and editing. -For decision guidance on when to use CLI vs direct editing, see -[docs-cli-workflow skill](../../.agents/skills/docs-cli-workflow/SKILL.md). - -### Creating New Content - -Use `docs create` for AI-assisted scaffolding: - -```bash -# Create from draft -docs create drafts/feature.md --products influxdb3_core - -# Create and open files in editor (non-blocking) -docs create drafts/feature.md --products influxdb3_core --open - -# Create and open, wait for editor (blocking) -docs create drafts/feature.md --products influxdb3_core --open --wait -``` - -### Editing Existing Content - -Use `docs edit` to quickly find and open content files: - -```bash -# Find and list files (no editor) -docs edit /influxdb3/core/admin/databases/ --list - -# Open in editor (non-blocking, exits immediately) -docs edit /influxdb3/core/admin/databases/ - -# Open and wait for editor (blocking, interactive) -docs edit /influxdb3/core/admin/databases/ --wait - -# Use specific editor -docs edit /influxdb3/core/admin/databases/ --editor nano -``` - -**Options:** - -- Both commands are **non-blocking by default** (agent-friendly) -- Use `--wait` for interactive editing sessions -- Use `--list` with `docs edit` to see files without opening -- Accepts both product keys (`influxdb3_core`) and paths (`/influxdb3/core`) - -### Other CLI Commands - -```bash -# Add placeholder syntax to code blocks -docs placeholders <file.md> - -# Audit documentation coverage -docs audit --products influxdb3_core - -# Generate release notes -docs release-notes v3.1.0 v3.2.0 --products influxdb3_core -``` - -For complete CLI reference, run `docs --help`. - -## Shared Content Management - -When editing files with `source:` frontmatter (shared content): - -- **Recommended**: Use `docs edit <url>` - automatically finds and opens all - related files -- **Manual**: If editing directly, remember to touch sourcing files to trigger - Hugo rebuild - -For complete shared content workflow, see -[content-editing skill](../../.agents/skills/content-editing/SKILL.md). - -## Required for All Content Files - -Every content file needs: - -```yaml -title: # Page h1 heading -description: # SEO meta description -menu: - product_menu_key: # Identifies the Hugo menu specific to the current product - name: # Navigation link text - parent: # Parent menu item (if nested) -weight: # Sort order (1-99, 101-199, 201-299...) -``` - -## Testing After Content Changes - -```bash -# 1. Verify Hugo build -npx hugo --quiet - -# 2. Validate links (build first; see DOCS-TESTING.md) -link-checker map content/path/*.md | xargs link-checker check - -# 3. Test code blocks (if applicable) -yarn test:codeblocks:all -``` - -For comprehensive testing workflows, see -[content-editing skill](../../.agents/skills/content-editing/SKILL.md). - -### Line protocol fences - -Use `lp` for InfluxDB line protocol examples. -The code-block linter validates `lp` fences and blocks malformed syntax in CI. -Qualified field keys use `family::field`; only the first `::` identifies the -family delimiter, so later `::` sequences remain part of the field name. -For an intentionally invalid example, add `{lint="false"}` to the fence. - -## Style Guidelines - -- Use semantic line feeds (one sentence per line) -- Test all code examples before committing -- Use appropriate shortcodes for UI elements -- Follow Google Developer Documentation Style Guide -- Use active voice, present tense, second person -- Use data-ownership framing: when writing import/write/load guidance, point the - verb at the resource the user owns ("import your data into a database or - table"), not at the product ("import data into InfluxDB"). The user owns their - data in their own object storage; InfluxDB reads and writes it but doesn't take - custody of it. -- Phrase recommendations in first-person plural: "We recommend...", not - third-party attributions such as "The Telegraf project recommends...". - Docs speak with InfluxData's voice, even when a recommendation originates - in an upstream project's guidance. -- Set `weight` at the page level (top-level frontmatter), not on the menu - entry. Menu items inherit the page weight, and page-level weight keeps - sorting consistent outside menu contexts, such as `children` shortcode - listings. (Hugo sorts unweighted pages after weighted ones, so mixing the - two placements within a section breaks list ordering.) - -## Most Common Shortcodes - -**Callouts**: - -```markdown -> [!Note] -> [!Warning] -> [!Caution] -> [!Important] -> [!Tip] -``` - -**Required elements**: - -```markdown -{{< req >}} -{{< req type="key" >}} -``` - -**Code placeholders**: - -````markdown -```sh { placeholders="DATABASE_NAME|API_TOKEN" } -curl -X POST https://cloud2.influxdata.com/api/v2/write?bucket=DATABASE_NAME -``` -```` - -Replace the following: - -- {{% code-placeholder-key %}}`DATABASE_NAME`{{% /code-placeholder-key %}}: - your database name - -**Tabbed content**: - -```markdown -{{< tabs-wrapper >}} -{{% tabs %}} -[Tab 1](#) -[Tab 2](#) -{{% /tabs %}} -{{% tab-content %}} -Content for tab 1 -{{% /tab-content %}} -{{% tab-content %}} -Content for tab 2 -{{% /tab-content %}} -{{< /tabs-wrapper >}} -``` - -For complete shortcodes reference, see -[DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md). - -## Related Resources - -- **Complete workflow**: [content-editing skill](../../.agents/skills/content-editing/SKILL.md) -- **CLI decision guidance**: - [docs-cli-workflow skill](../../.agents/skills/docs-cli-workflow/SKILL.md) -- **Frontmatter**: [DOCS-FRONTMATTER.md](../../DOCS-FRONTMATTER.md) -- **Shortcodes**: [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) -- **Contributing**: [DOCS-CONTRIBUTING.md](../../DOCS-CONTRIBUTING.md) +# Content files + +Use [DOCS-FRONTMATTER.md](../../DOCS-FRONTMATTER.md) and +[DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) as the syntax authority. +Use the [content-editing skill](../../.agents/skills/content-editing/SKILL.md) for workflow +and the [docs-cli-workflow skill](../../.agents/skills/docs-cli-workflow/SKILL.md) to select +the `docs` CLI or direct editing. + +## Requirements + +- Page frontmatter supplies `title`, `description`, appropriate `menu`, and + page-level `weight` when it appears in navigation. +- Do not add a body h1. Use semantic line feeds, active present-tense second + person, long CLI options, and `python` rather than `py` fences. +- Use `lp` for line protocol. Add `{lint="false"}` only to intentional invalid + examples. +- Do not hardcode production docs URLs when a relative link or `relref` works. +- Shared files have no frontmatter. A stub's `source:` must begin `/shared/`. + Direct shared edits require touching every source stub; `docs edit` finds them. +- Use resource-ownership language for import/write/load guidance and write + recommendations in InfluxData's first-person plural voice. + +Run `yarn verify:changed -- <files>` to select manual checks. See +[content/example.md](../../content/example.md) for working shortcode examples. diff --git a/.claude/rules/layouts.md b/.claude/rules/layouts.md index 56efe3e68c..abfbe30684 100644 --- a/.claude/rules/layouts.md +++ b/.claude/rules/layouts.md @@ -7,131 +7,23 @@ paths: <!-- Run 'yarn build:agent:instructions' to regenerate it. --> -# Layout and Shortcode Implementation Guidelines - -**Shortcodes reference**: [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) -**Test examples**: [content/example.md](../../content/example.md) - -**For detailed Hugo template development workflow**, see -[hugo-template-dev skill](../../.agents/skills/hugo-template-dev/SKILL.md) which covers: - -- Hugo template syntax and data access patterns -- Build-time vs runtime testing strategies -- Shortcode implementation best practices -- Complete TDD workflow for Hugo templates - -## No Magic Values in Template Logic - -Templates operate on data and stay ignorant of the values in that data. -A product name, version segment, or `data/products.yml` key must never appear -as a string literal in template logic. -Nobody should have to edit a template because a product was renamed or added. - -Never write any of these in `layouts/**`: - -- A slice of product names or version segments used in a condition, such as a - list of the versions that count as current or the products that support Flux. -- A single hardcoded product comparison that branches behavior, such as testing - whether the first path segment equals a specific product. -- Deriving a `data/products.yml` key by matching the URL path when the page - already declares one. - -This file is generated into `layouts/AGENTS.md`, and Hugo parses every file -under `layouts/` as a template, so it carries no Go template examples. -For the annotated before and after, see the -[hugo-template-dev skill](../../.agents/skills/hugo-template-dev/SKILL.md). - -Do this instead: - -1. Put the fact in `data/products.yml` as a per-product field — a boolean such - as `supports_flux`, `has_support_contract`, or `search_includes_resources` — - and read it with a `| default` that covers products that don't set it. -2. Resolve the product with `partial "product/get-data.html"` or - `partial "product/get-context.html"`, which read the page's cascade `product` - param. - Every product section declares `product` and `version` by cascade in its - section `_index.md`, so the key is stated rather than guessed. -3. When two templates need the same decision, extract it into one partial so - the two can't drift. - `layouts/partials/product/is-latest.html` is the worked example. - -The one exception is a value that must match an external system rather than a -product fact. -The Algolia search tag in `layouts/partials/header/search-attributes.html` -stays path-derived because Algolia indexed every record under the crawled URL. -Comment any such case in the template so the next reader doesn't "fix" it. - -For the before/after example and the incident behind this rule, see -[hugo-template-dev skill](../../.agents/skills/hugo-template-dev/SKILL.md). - -## Implementing Shortcodes - -When creating or modifying Hugo layouts and shortcodes: - -1. Use test-driven development using `/cypress/` -2. Use Hugo template syntax and functions -3. Follow existing patterns in `/layouts/shortcodes/` -4. Test in [content/example.md](../../content/example.md) -5. Document new shortcodes in [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) - -## Shortcode Pattern - -```html -<!-- layouts/shortcodes/example.html --> -{{ $param := .Get 0 }} -{{ $namedParam := .Get "name" }} - -<div class="example"> - {{ .Inner | markdownify }} -</div> -``` - -## Testing - -**IMPORTANT:** Use test-driven development with Cypress. - -Add shortcode usage examples to `content/example.md` to verify: - -- Rendering in browser -- Hugo build succeeds -- No console errors -- JavaScript functionality works as expected (check browser console for errors) -- Interactive elements behave correctly (click links, buttons, etc.) - -### TDD Workflow - -1. Add Cypress tests (high-level to start). -2. Run tests and make sure they fail. -3. Implement code changes -4. Run tests and make sure they pass. -5. Add and refine tests. -6. Repeat. - -### Manual Testing Workflow - -1. Make changes to shortcode/layout files -2. Wait for Hugo to rebuild (check terminal output) -3. Get the server URL from the log -4. Open browser DevTools console (F12) -5. Test the functionality and check for JavaScript errors -6. Verify the feature works as intended before marking complete - -See [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) for complete shortcode -documentation. - -### Line protocol render hook - -`layouts/_default/_markup/render-codeblock-lp.html` renders `lp` code fences. -Keep its output Chroma-compatible (`.highlight > pre.chroma > code.language-lp`) -and HTML-escape source text before marking generated markup safe. -For malformed source, render escaped plain text rather than partial highlighting. - -## Related Resources - -- **Complete Hugo template workflow**: - [hugo-template-dev skill](../../.agents/skills/hugo-template-dev/SKILL.md) -- **Shortcodes reference**: [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) -- **Test examples**: [content/example.md](../../content/example.md) -- **Article-level page actions** (buttons/links next to the page title — when - to use, how to add a new one): - [DOCS-PAGE-ACTIONS.md](../../DOCS-PAGE-ACTIONS.md) +# Hugo layouts and shortcodes + +Use the [hugo-template-dev skill](../../.agents/skills/hugo-template-dev/SKILL.md) for +implementation and runtime verification. Follow existing shortcode patterns and +document user-facing shortcodes in [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md). + +## Requirements + +- Do not encode product names, versions, URL segments, or product-key guesses + in template branching. Put product facts in `data/products.yml`, resolve page + context through product partials, and share repeated decisions in a partial. +- The sole exception is an externally mandated path-derived value; explain it + in a template comment. +- Add behavior coverage in Cypress and an example in + [content/example.md](../../content/example.md) when appropriate. A Hugo build + alone is insufficient for runtime behavior. +- Keep the line-protocol render hook Chroma-compatible and HTML-escape source; + malformed input renders as escaped plain text. + +Run `yarn verify:changed -- <files>` to identify manual checks. diff --git a/.github/instructions/content.instructions.md b/.github/instructions/content.instructions.md index 395154e96b..302a0113a9 100644 --- a/.github/instructions/content.instructions.md +++ b/.github/instructions/content.instructions.md @@ -6,210 +6,27 @@ applyTo: "content/**/*.md" <!-- Run 'yarn build:agent:instructions' to regenerate it. --> -# Content File Guidelines - -**Frontmatter reference**: [DOCS-FRONTMATTER.md](../../DOCS-FRONTMATTER.md) -**Shortcodes reference**: [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) -**Working examples**: [content/example.md](../../content/example.md) - -**For complete content editing workflow**, see -[content-editing skill](../../.agents/skills/content-editing/SKILL.md) which covers: - -- Creating and editing content with CLI tools -- Shared content management and testing -- Fact-checking with MCP server -- Complete validation workflows - -## CLI Tools for Content Workflow - -The unified `docs` CLI provides tools for content creation and editing. -For decision guidance on when to use CLI vs direct editing, see -[docs-cli-workflow skill](../../.agents/skills/docs-cli-workflow/SKILL.md). - -### Creating New Content - -Use `docs create` for AI-assisted scaffolding: - -```bash -# Create from draft -docs create drafts/feature.md --products influxdb3_core - -# Create and open files in editor (non-blocking) -docs create drafts/feature.md --products influxdb3_core --open - -# Create and open, wait for editor (blocking) -docs create drafts/feature.md --products influxdb3_core --open --wait -``` - -### Editing Existing Content - -Use `docs edit` to quickly find and open content files: - -```bash -# Find and list files (no editor) -docs edit /influxdb3/core/admin/databases/ --list - -# Open in editor (non-blocking, exits immediately) -docs edit /influxdb3/core/admin/databases/ - -# Open and wait for editor (blocking, interactive) -docs edit /influxdb3/core/admin/databases/ --wait - -# Use specific editor -docs edit /influxdb3/core/admin/databases/ --editor nano -``` - -**Options:** - -- Both commands are **non-blocking by default** (agent-friendly) -- Use `--wait` for interactive editing sessions -- Use `--list` with `docs edit` to see files without opening -- Accepts both product keys (`influxdb3_core`) and paths (`/influxdb3/core`) - -### Other CLI Commands - -```bash -# Add placeholder syntax to code blocks -docs placeholders <file.md> - -# Audit documentation coverage -docs audit --products influxdb3_core - -# Generate release notes -docs release-notes v3.1.0 v3.2.0 --products influxdb3_core -``` - -For complete CLI reference, run `docs --help`. - -## Shared Content Management - -When editing files with `source:` frontmatter (shared content): - -- **Recommended**: Use `docs edit <url>` - automatically finds and opens all - related files -- **Manual**: If editing directly, remember to touch sourcing files to trigger - Hugo rebuild - -For complete shared content workflow, see -[content-editing skill](../../.agents/skills/content-editing/SKILL.md). - -## Required for All Content Files - -Every content file needs: - -```yaml -title: # Page h1 heading -description: # SEO meta description -menu: - product_menu_key: # Identifies the Hugo menu specific to the current product - name: # Navigation link text - parent: # Parent menu item (if nested) -weight: # Sort order (1-99, 101-199, 201-299...) -``` - -## Testing After Content Changes - -```bash -# 1. Verify Hugo build -npx hugo --quiet - -# 2. Validate links (build first; see DOCS-TESTING.md) -link-checker map content/path/*.md | xargs link-checker check - -# 3. Test code blocks (if applicable) -yarn test:codeblocks:all -``` - -For comprehensive testing workflows, see -[content-editing skill](../../.agents/skills/content-editing/SKILL.md). - -### Line protocol fences - -Use `lp` for InfluxDB line protocol examples. -The code-block linter validates `lp` fences and blocks malformed syntax in CI. -Qualified field keys use `family::field`; only the first `::` identifies the -family delimiter, so later `::` sequences remain part of the field name. -For an intentionally invalid example, add `{lint="false"}` to the fence. - -## Style Guidelines - -- Use semantic line feeds (one sentence per line) -- Test all code examples before committing -- Use appropriate shortcodes for UI elements -- Follow Google Developer Documentation Style Guide -- Use active voice, present tense, second person -- Use data-ownership framing: when writing import/write/load guidance, point the - verb at the resource the user owns ("import your data into a database or - table"), not at the product ("import data into InfluxDB"). The user owns their - data in their own object storage; InfluxDB reads and writes it but doesn't take - custody of it. -- Phrase recommendations in first-person plural: "We recommend...", not - third-party attributions such as "The Telegraf project recommends...". - Docs speak with InfluxData's voice, even when a recommendation originates - in an upstream project's guidance. -- Set `weight` at the page level (top-level frontmatter), not on the menu - entry. Menu items inherit the page weight, and page-level weight keeps - sorting consistent outside menu contexts, such as `children` shortcode - listings. (Hugo sorts unweighted pages after weighted ones, so mixing the - two placements within a section breaks list ordering.) - -## Most Common Shortcodes - -**Callouts**: - -```markdown -> [!Note] -> [!Warning] -> [!Caution] -> [!Important] -> [!Tip] -``` - -**Required elements**: - -```markdown -{{< req >}} -{{< req type="key" >}} -``` - -**Code placeholders**: - -````markdown -```sh { placeholders="DATABASE_NAME|API_TOKEN" } -curl -X POST https://cloud2.influxdata.com/api/v2/write?bucket=DATABASE_NAME -``` -```` - -Replace the following: - -- {{% code-placeholder-key %}}`DATABASE_NAME`{{% /code-placeholder-key %}}: - your database name - -**Tabbed content**: - -```markdown -{{< tabs-wrapper >}} -{{% tabs %}} -[Tab 1](#) -[Tab 2](#) -{{% /tabs %}} -{{% tab-content %}} -Content for tab 1 -{{% /tab-content %}} -{{% tab-content %}} -Content for tab 2 -{{% /tab-content %}} -{{< /tabs-wrapper >}} -``` - -For complete shortcodes reference, see -[DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md). - -## Related Resources - -- **Complete workflow**: [content-editing skill](../../.agents/skills/content-editing/SKILL.md) -- **CLI decision guidance**: - [docs-cli-workflow skill](../../.agents/skills/docs-cli-workflow/SKILL.md) -- **Frontmatter**: [DOCS-FRONTMATTER.md](../../DOCS-FRONTMATTER.md) -- **Shortcodes**: [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) -- **Contributing**: [DOCS-CONTRIBUTING.md](../../DOCS-CONTRIBUTING.md) +# Content files + +Use [DOCS-FRONTMATTER.md](../../DOCS-FRONTMATTER.md) and +[DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) as the syntax authority. +Use the [content-editing skill](../../.agents/skills/content-editing/SKILL.md) for workflow +and the [docs-cli-workflow skill](../../.agents/skills/docs-cli-workflow/SKILL.md) to select +the `docs` CLI or direct editing. + +## Requirements + +- Page frontmatter supplies `title`, `description`, appropriate `menu`, and + page-level `weight` when it appears in navigation. +- Do not add a body h1. Use semantic line feeds, active present-tense second + person, long CLI options, and `python` rather than `py` fences. +- Use `lp` for line protocol. Add `{lint="false"}` only to intentional invalid + examples. +- Do not hardcode production docs URLs when a relative link or `relref` works. +- Shared files have no frontmatter. A stub's `source:` must begin `/shared/`. + Direct shared edits require touching every source stub; `docs edit` finds them. +- Use resource-ownership language for import/write/load guidance and write + recommendations in InfluxData's first-person plural voice. + +Run `yarn verify:changed -- <files>` to select manual checks. See +[content/example.md](../../content/example.md) for working shortcode examples. diff --git a/.github/instructions/layouts.instructions.md b/.github/instructions/layouts.instructions.md index 1bf29e2032..2d694f6b00 100644 --- a/.github/instructions/layouts.instructions.md +++ b/.github/instructions/layouts.instructions.md @@ -6,131 +6,23 @@ applyTo: "layouts/**/*.html" <!-- Run 'yarn build:agent:instructions' to regenerate it. --> -# Layout and Shortcode Implementation Guidelines - -**Shortcodes reference**: [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) -**Test examples**: [content/example.md](../../content/example.md) - -**For detailed Hugo template development workflow**, see -[hugo-template-dev skill](../../.agents/skills/hugo-template-dev/SKILL.md) which covers: - -- Hugo template syntax and data access patterns -- Build-time vs runtime testing strategies -- Shortcode implementation best practices -- Complete TDD workflow for Hugo templates - -## No Magic Values in Template Logic - -Templates operate on data and stay ignorant of the values in that data. -A product name, version segment, or `data/products.yml` key must never appear -as a string literal in template logic. -Nobody should have to edit a template because a product was renamed or added. - -Never write any of these in `layouts/**`: - -- A slice of product names or version segments used in a condition, such as a - list of the versions that count as current or the products that support Flux. -- A single hardcoded product comparison that branches behavior, such as testing - whether the first path segment equals a specific product. -- Deriving a `data/products.yml` key by matching the URL path when the page - already declares one. - -This file is generated into `layouts/AGENTS.md`, and Hugo parses every file -under `layouts/` as a template, so it carries no Go template examples. -For the annotated before and after, see the -[hugo-template-dev skill](../../.agents/skills/hugo-template-dev/SKILL.md). - -Do this instead: - -1. Put the fact in `data/products.yml` as a per-product field — a boolean such - as `supports_flux`, `has_support_contract`, or `search_includes_resources` — - and read it with a `| default` that covers products that don't set it. -2. Resolve the product with `partial "product/get-data.html"` or - `partial "product/get-context.html"`, which read the page's cascade `product` - param. - Every product section declares `product` and `version` by cascade in its - section `_index.md`, so the key is stated rather than guessed. -3. When two templates need the same decision, extract it into one partial so - the two can't drift. - `layouts/partials/product/is-latest.html` is the worked example. - -The one exception is a value that must match an external system rather than a -product fact. -The Algolia search tag in `layouts/partials/header/search-attributes.html` -stays path-derived because Algolia indexed every record under the crawled URL. -Comment any such case in the template so the next reader doesn't "fix" it. - -For the before/after example and the incident behind this rule, see -[hugo-template-dev skill](../../.agents/skills/hugo-template-dev/SKILL.md). - -## Implementing Shortcodes - -When creating or modifying Hugo layouts and shortcodes: - -1. Use test-driven development using `/cypress/` -2. Use Hugo template syntax and functions -3. Follow existing patterns in `/layouts/shortcodes/` -4. Test in [content/example.md](../../content/example.md) -5. Document new shortcodes in [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) - -## Shortcode Pattern - -```html -<!-- layouts/shortcodes/example.html --> -{{ $param := .Get 0 }} -{{ $namedParam := .Get "name" }} - -<div class="example"> - {{ .Inner | markdownify }} -</div> -``` - -## Testing - -**IMPORTANT:** Use test-driven development with Cypress. - -Add shortcode usage examples to `content/example.md` to verify: - -- Rendering in browser -- Hugo build succeeds -- No console errors -- JavaScript functionality works as expected (check browser console for errors) -- Interactive elements behave correctly (click links, buttons, etc.) - -### TDD Workflow - -1. Add Cypress tests (high-level to start). -2. Run tests and make sure they fail. -3. Implement code changes -4. Run tests and make sure they pass. -5. Add and refine tests. -6. Repeat. - -### Manual Testing Workflow - -1. Make changes to shortcode/layout files -2. Wait for Hugo to rebuild (check terminal output) -3. Get the server URL from the log -4. Open browser DevTools console (F12) -5. Test the functionality and check for JavaScript errors -6. Verify the feature works as intended before marking complete - -See [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) for complete shortcode -documentation. - -### Line protocol render hook - -`layouts/_default/_markup/render-codeblock-lp.html` renders `lp` code fences. -Keep its output Chroma-compatible (`.highlight > pre.chroma > code.language-lp`) -and HTML-escape source text before marking generated markup safe. -For malformed source, render escaped plain text rather than partial highlighting. - -## Related Resources - -- **Complete Hugo template workflow**: - [hugo-template-dev skill](../../.agents/skills/hugo-template-dev/SKILL.md) -- **Shortcodes reference**: [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md) -- **Test examples**: [content/example.md](../../content/example.md) -- **Article-level page actions** (buttons/links next to the page title — when - to use, how to add a new one): - [DOCS-PAGE-ACTIONS.md](../../DOCS-PAGE-ACTIONS.md) +# Hugo layouts and shortcodes + +Use the [hugo-template-dev skill](../../.agents/skills/hugo-template-dev/SKILL.md) for +implementation and runtime verification. Follow existing shortcode patterns and +document user-facing shortcodes in [DOCS-SHORTCODES.md](../../DOCS-SHORTCODES.md). + +## Requirements + +- Do not encode product names, versions, URL segments, or product-key guesses + in template branching. Put product facts in `data/products.yml`, resolve page + context through product partials, and share repeated decisions in a partial. +- The sole exception is an externally mandated path-derived value; explain it + in a template comment. +- Add behavior coverage in Cypress and an example in + [content/example.md](../../content/example.md) when appropriate. A Hugo build + alone is insufficient for runtime behavior. +- Keep the line-protocol render hook Chroma-compatible and HTML-escape source; + malformed input renders as escaped plain text. + +Run `yarn verify:changed -- <files>` to identify manual checks. diff --git a/AGENTS.md b/AGENTS.md index 688611d643..e2dde9d2c0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,78 +1,70 @@ # InfluxData Documentation (docs-v2) -Shared startup guidance for all AI assistants working in this repository. +This is the harness-neutral repository contract for Codex, Claude Code, Pi, and +other agents. Use this file and `.agents/` as the source of truth. Claude rules, +Copilot instructions, and scoped `AGENTS.md` files are generated adapters. -## Canonical agent resources +## Agent assets -- `AGENTS.md` is the shared repo-wide instruction file. -- `.agents/skills/` contains reusable Agent Skills shared by Codex, Claude - Code, GitHub Copilot, and other compatible harnesses. -- `.agents/instructions/` contains canonical path-specific instruction sources. -- `.github/instructions/*.instructions.md`, `.claude/rules/*.md`, and scoped - `AGENTS.md` files under major directories are generated adapters. -- `.claude/skills` is a symlink to `.agents/skills`. +- Path-specific rules: `.agents/instructions/`. +- Reusable workflows: `.agents/skills/`. After editing `AGENTS.md` or `.agents/**`, run: -```bash +```sh yarn build:agent:instructions yarn validate:agent-instructions ``` -## Core commands +- Keep `.claude/skills` as a symlink to `.agents/skills`. +- Do not add a Pi-specific configuration; Pi uses this contract and skills. -| Task | Command | Notes | -| ---------------- | --------------------------------------- | --------------------------------------------- | -| Install | `CYPRESS_INSTALL_BINARY=0 yarn install` | Use in network-restricted environments | -| Build | `npx hugo --quiet` | \~75s; never cancel | -| Dev server | `npx hugo server` | \~92s; serves on port 1313 | -| Test code blocks | `yarn test:codeblocks:all` | 15-45m; never cancel | -| Lint | `yarn lint` | Runs pre-commit and pre-push hook validations | +## Work safely -## docs CLI +- Work from the current worktree; never hardcode a path to another clone. -Scaffold and manage documentation with the `docs` CLI (`docs --help` for full -reference). Non-blocking by default; use `--wait` for interactive editing. +- Preserve unrelated working-tree changes. -- `docs create <draft> --products <keys>` — scaffold new pages -- `docs edit <url|path>` — open existing pages -- `docs placeholders <file>` — add placeholder syntax to code blocks -- `docs audit --products <keys>` — audit coverage - -See [README.md](README.md) and the -[docs-cli-workflow skill](.agents/skills/docs-cli-workflow/SKILL.md) for details. - -## Worktree and path rules - -- This repo uses git worktrees. The current working directory is the repo root. -- Never hardcode paths back to the main clone under `/Users/*/docs-v2/`. -- Scripts that need `PROJECT_ROOT` derive it from `SCRIPT_DIR`; agents should - use the current working directory, not the main clone path. - -## Search rules +- Never cancel Hugo builds or code-block tests. Give Hugo at least 180 seconds + and long code-block suites 30 minutes. - For InfluxDB 3 content, search these paths in parallel: - `content/shared/influxdb3-*/` - `content/influxdb3/core/` - `content/influxdb3/enterprise/` -- Use Grep and Glob for local content searches. -- Run independent searches in parallel rather than one broad sequential search. - -## Constraints -- Never cancel Hugo builds or code block test runs. -- Use timeouts of at least 180s for Hugo and 30m for long tests. - Use `python`, not `py`, for code block language identifiers. + +## Choose checks + +`git commit` runs staged-file hooks. Do not manually run `yarn lint` before a +commit unless diagnosing a hook failure. Use the changed-file verifier to plan +or run only manual checks: + +```sh +yarn verify:changed -- <path> [<path> ...] +yarn verify:changed -- --staged +yarn verify:changed -- --run <path> [<path> ...] +``` + +Use `npx hugo --quiet` for a full build, `npx hugo server` for local serving, +and `yarn test:codeblocks:all` only when runnable examples require it. +See the [docs-testing skill](.agents/skills/docs-testing/SKILL.md) for routing. + +## Content contract + - Shared content files under `content/shared/` have no frontmatter; consuming pages provide metadata through `source:`. - Shared directories contain prose; product directories are often thin stubs with `source:` references to the shared content. - Product names and versions come from `data/products.yml`. -- Commit format is `type(scope): description`. Beyond a trivial fix, the body - uses the same sections as the pull request template: What changed, Why, - Impact, Verification. See `DOCS-CONTRIBUTING.md`. -- Network-restricted environments may fail on Cypress downloads, Docker builds, - or Alpine package installs. +- Use semantic line feeds, active present-tense second person, long CLI options, + and no body h1 in content. Follow the Google developer documentation style. + +## Repository conventions + +- Use `type(scope): description` commit subjects. Non-trivial commits include + What changed, Why, Impact, and Verification. ## Dependency management @@ -80,58 +72,22 @@ See [README.md](README.md) and the repos. Do not stand up a parallel dependency-update mechanism, and treat a repo-level `.github/dependabot.yml` (if present) as supplementary to the org config, not the source of truth. + - Pin third-party GitHub Actions by full commit SHA (with a version comment) so Dependabot can keep the pins current. `.github/workflows/pr-lockfile-lint.yml` is the reference example. + - Coordinate with the security team before changing dependency automation; org-wide Dependabot security updates do not automatically include scheduled `github-actions` version updates. -## Documentation style - -- Follow the Google Developer Documentation Style Guide for all aspects of communication, including structure and voice. -- Apply the style guide to commit messages, issues, pull request descriptions, - and status comments, not only to content pages. -- Write to inform, not to impress. State a finding directly instead of building - up to it. Do not use section headings, one-sentence paragraphs, or statistics - for dramatic emphasis. -- Use semantic line feeds: one sentence per line. -- Do not add `#` h1 headings in content; `title` frontmatter generates the h1. -- Prefer active voice, present tense, and second person. -- Use long options in CLI examples. -- Keep code blocks within 80 characters where practical. - -## Documentation search (MCP) - -A hosted InfluxDB documentation search server is configured for this repo. -Use it to verify technical accuracy, check API syntax, and find related docs. -Harness-specific setup (Claude Code: `.mcp.json`) lives in each harness's own -instruction file. - -## Plans and design docs - -- Implementation plans → `PLAN.md` at the repo root. Tracked on feature - branches; a required PR check blocks `PLAN.md` and `HANDOVER.md` from merging - to the default branch — remove or promote them before merge. -- Design specs → an existing docs location (`DOCS-*.md` or product `content/` - frontmatter) only if useful post-merge; otherwise keep them in the session or - alongside `PLAN.md` and remove before merge. - -## Where detailed guidance lives - -- `content/AGENTS.md`: frontmatter, shortcodes, shared content, and doc editing - workflow. -- `layouts/AGENTS.md`: Hugo template and shortcode guidance. -- `assets/AGENTS.md`: JS, CSS, and TypeScript guidance. -- `api-docs/AGENTS.md`: API reference generation workflow. -- `DOCS-TESTING.md`: content validation workflow (human contributor reference). -- `.agents/skills/docs-testing/SKILL.md`: agent testing decision guide — maps - changed file types to exact test commands, documents what CI runs automatically, - and flags coverage gaps. -- `.agents/skills/issue-investigation/SKILL.md`: issue investigation guide — - verify reported problems are real before fixing, check git history for existing - fixes, and distinguish root causes. Use before working on any bug report or - broken link. -- `DOCS-SHORTCODES.md`: shortcode reference. -- `DOCS-FRONTMATTER.md`: frontmatter reference. -- `DOCS-CONTRIBUTING.md`: contribution and commit conventions. +- `PLAN.md` and `HANDOVER.md` are ephemeral and blocked on the default branch. + +## Detailed references + +- [Content rules](content/AGENTS.md), [layouts](layouts/AGENTS.md), + [assets](assets/AGENTS.md), and [API docs](api-docs/AGENTS.md). +- [DOCS-CONTRIBUTING.md](DOCS-CONTRIBUTING.md), + [DOCS-FRONTMATTER.md](DOCS-FRONTMATTER.md), + [DOCS-SHORTCODES.md](DOCS-SHORTCODES.md), and + [DOCS-TESTING.md](DOCS-TESTING.md). diff --git a/content/AGENTS.md b/content/AGENTS.md index eb510356e1..ae04f6ddc1 100644 --- a/content/AGENTS.md +++ b/content/AGENTS.md @@ -6,213 +6,30 @@ These instructions apply when working in `content/`. -## Content File Guidelines - -**Frontmatter reference**: [DOCS-FRONTMATTER.md](../DOCS-FRONTMATTER.md) -**Shortcodes reference**: [DOCS-SHORTCODES.md](../DOCS-SHORTCODES.md) -**Working examples**: [content/example.md](./example.md) - -**For complete content editing workflow**, see -[content-editing skill](../.agents/skills/content-editing/SKILL.md) which covers: - -- Creating and editing content with CLI tools -- Shared content management and testing -- Fact-checking with MCP server -- Complete validation workflows - -### CLI Tools for Content Workflow - -The unified `docs` CLI provides tools for content creation and editing. -For decision guidance on when to use CLI vs direct editing, see -[docs-cli-workflow skill](../.agents/skills/docs-cli-workflow/SKILL.md). - -#### Creating New Content - -Use `docs create` for AI-assisted scaffolding: - -```bash -# Create from draft -docs create drafts/feature.md --products influxdb3_core - -# Create and open files in editor (non-blocking) -docs create drafts/feature.md --products influxdb3_core --open - -# Create and open, wait for editor (blocking) -docs create drafts/feature.md --products influxdb3_core --open --wait -``` - -#### Editing Existing Content - -Use `docs edit` to quickly find and open content files: - -```bash -# Find and list files (no editor) -docs edit /influxdb3/core/admin/databases/ --list - -# Open in editor (non-blocking, exits immediately) -docs edit /influxdb3/core/admin/databases/ - -# Open and wait for editor (blocking, interactive) -docs edit /influxdb3/core/admin/databases/ --wait - -# Use specific editor -docs edit /influxdb3/core/admin/databases/ --editor nano -``` - -**Options:** - -- Both commands are **non-blocking by default** (agent-friendly) -- Use `--wait` for interactive editing sessions -- Use `--list` with `docs edit` to see files without opening -- Accepts both product keys (`influxdb3_core`) and paths (`/influxdb3/core`) - -#### Other CLI Commands - -```bash -# Add placeholder syntax to code blocks -docs placeholders <file.md> - -# Audit documentation coverage -docs audit --products influxdb3_core - -# Generate release notes -docs release-notes v3.1.0 v3.2.0 --products influxdb3_core -``` - -For complete CLI reference, run `docs --help`. - -### Shared Content Management - -When editing files with `source:` frontmatter (shared content): - -- **Recommended**: Use `docs edit <url>` - automatically finds and opens all - related files -- **Manual**: If editing directly, remember to touch sourcing files to trigger - Hugo rebuild - -For complete shared content workflow, see -[content-editing skill](../.agents/skills/content-editing/SKILL.md). - -### Required for All Content Files - -Every content file needs: - -```yaml -title: # Page h1 heading -description: # SEO meta description -menu: - product_menu_key: # Identifies the Hugo menu specific to the current product - name: # Navigation link text - parent: # Parent menu item (if nested) -weight: # Sort order (1-99, 101-199, 201-299...) -``` - -### Testing After Content Changes - -```bash -# 1. Verify Hugo build -npx hugo --quiet - -# 2. Validate links (build first; see DOCS-TESTING.md) -link-checker map content/path/*.md | xargs link-checker check - -# 3. Test code blocks (if applicable) -yarn test:codeblocks:all -``` - -For comprehensive testing workflows, see -[content-editing skill](../.agents/skills/content-editing/SKILL.md). - -#### Line protocol fences - -Use `lp` for InfluxDB line protocol examples. -The code-block linter validates `lp` fences and blocks malformed syntax in CI. -Qualified field keys use `family::field`; only the first `::` identifies the -family delimiter, so later `::` sequences remain part of the field name. -For an intentionally invalid example, add `{lint="false"}` to the fence. - -### Style Guidelines - -- Use semantic line feeds (one sentence per line) -- Test all code examples before committing -- Use appropriate shortcodes for UI elements -- Follow Google Developer Documentation Style Guide -- Use active voice, present tense, second person -- Use data-ownership framing: when writing import/write/load guidance, point the - verb at the resource the user owns ("import your data into a database or - table"), not at the product ("import data into InfluxDB"). The user owns their - data in their own object storage; InfluxDB reads and writes it but doesn't take - custody of it. -- Phrase recommendations in first-person plural: "We recommend...", not - third-party attributions such as "The Telegraf project recommends...". - Docs speak with InfluxData's voice, even when a recommendation originates - in an upstream project's guidance. -- Set `weight` at the page level (top-level frontmatter), not on the menu - entry. Menu items inherit the page weight, and page-level weight keeps - sorting consistent outside menu contexts, such as `children` shortcode - listings. (Hugo sorts unweighted pages after weighted ones, so mixing the - two placements within a section breaks list ordering.) - -### Most Common Shortcodes - -**Callouts**: - -```markdown -> [!Note] -> [!Warning] -> [!Caution] -> [!Important] -> [!Tip] -``` - -**Required elements**: - -```markdown -{{< req >}} -{{< req type="key" >}} -``` - -**Code placeholders**: - -````markdown -```sh { placeholders="DATABASE_NAME|API_TOKEN" } -curl -X POST https://cloud2.influxdata.com/api/v2/write?bucket=DATABASE_NAME -``` -```` - -Replace the following: - -- {{% code-placeholder-key %}}`DATABASE_NAME`{{% /code-placeholder-key %}}: - your database name - -**Tabbed content**: - -```markdown -{{< tabs-wrapper >}} -{{% tabs %}} -[Tab 1](#) -[Tab 2](#) -{{% /tabs %}} -{{% tab-content %}} -Content for tab 1 -{{% /tab-content %}} -{{% tab-content %}} -Content for tab 2 -{{% /tab-content %}} -{{< /tabs-wrapper >}} -``` - -For complete shortcodes reference, see -[DOCS-SHORTCODES.md](../DOCS-SHORTCODES.md). - -### Related Resources - -- **Complete workflow**: [content-editing skill](../.agents/skills/content-editing/SKILL.md) -- **CLI decision guidance**: - [docs-cli-workflow skill](../.agents/skills/docs-cli-workflow/SKILL.md) -- **Frontmatter**: [DOCS-FRONTMATTER.md](../DOCS-FRONTMATTER.md) -- **Shortcodes**: [DOCS-SHORTCODES.md](../DOCS-SHORTCODES.md) -- **Contributing**: [DOCS-CONTRIBUTING.md](../DOCS-CONTRIBUTING.md) +## Content files + +Use [DOCS-FRONTMATTER.md](../DOCS-FRONTMATTER.md) and +[DOCS-SHORTCODES.md](../DOCS-SHORTCODES.md) as the syntax authority. +Use the [content-editing skill](../.agents/skills/content-editing/SKILL.md) for workflow +and the [docs-cli-workflow skill](../.agents/skills/docs-cli-workflow/SKILL.md) to select +the `docs` CLI or direct editing. + +### Requirements + +- Page frontmatter supplies `title`, `description`, appropriate `menu`, and + page-level `weight` when it appears in navigation. +- Do not add a body h1. Use semantic line feeds, active present-tense second + person, long CLI options, and `python` rather than `py` fences. +- Use `lp` for line protocol. Add `{lint="false"}` only to intentional invalid + examples. +- Do not hardcode production docs URLs when a relative link or `relref` works. +- Shared files have no frontmatter. A stub's `source:` must begin `/shared/`. + Direct shared edits require touching every source stub; `docs edit` finds them. +- Use resource-ownership language for import/write/load guidance and write + recommendations in InfluxData's first-person plural voice. + +Run `yarn verify:changed -- <files>` to select manual checks. See +[content/example.md](./example.md) for working shortcode examples. ## Content Review Criteria diff --git a/helper-scripts/__tests__/agent-instruction-limits.test.mjs b/helper-scripts/__tests__/agent-instruction-limits.test.mjs new file mode 100644 index 0000000000..69583df76e --- /dev/null +++ b/helper-scripts/__tests__/agent-instruction-limits.test.mjs @@ -0,0 +1,34 @@ +import assert from 'assert/strict'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import test from 'node:test'; +import { + INSTRUCTION_LIMIT, + lineLimitError, + ROOT_AGENTS_LIMIT, + SKILL_LIMIT, +} from '../agent-instruction-limits.js'; + +function fixture(lines) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'agent-limits-')); + const file = path.join(directory, 'fixture.md'); + fs.writeFileSync( + file, + `${Array.from({ length: lines }, () => 'line').join('\n')}\n` + ); + return { directory, file }; +} +test('reports each canonical line-limit class', () => { + for (const limit of [ROOT_AGENTS_LIMIT, INSTRUCTION_LIMIT, SKILL_LIMIT]) { + const { directory, file } = fixture(limit + 1); + assert.match( + lineLimitError(file, limit, directory), + new RegExp(`has ${limit + 1} lines; limit is ${limit}`) + ); + } +}); +test('does not apply entrypoint limits to a reference file', () => { + const { directory, file } = fixture(SKILL_LIMIT + 50); + assert.equal(lineLimitError(file, Infinity, directory), null); +}); diff --git a/helper-scripts/agent-instruction-limits.js b/helper-scripts/agent-instruction-limits.js new file mode 100644 index 0000000000..f9c311ac46 --- /dev/null +++ b/helper-scripts/agent-instruction-limits.js @@ -0,0 +1,21 @@ +import fs from 'fs'; +import path from 'path'; + +export const ROOT_AGENTS_LIMIT = 120; +export const INSTRUCTION_LIMIT = 100; +export const SKILL_LIMIT = 160; + +export function lineCount(filePath) { + const text = fs.readFileSync(filePath, 'utf8'); + if (!text) return 0; + return text.endsWith('\n') + ? text.split('\n').length - 1 + : text.split('\n').length; +} + +export function lineLimitError(filePath, limit, root = process.cwd()) { + const count = lineCount(filePath); + return count > limit + ? `${path.relative(root, filePath)} has ${count} lines; limit is ${limit}` + : null; +} diff --git a/helper-scripts/validate-agent-instructions.js b/helper-scripts/validate-agent-instructions.js index ba9b6cf9e6..2503bb63e4 100644 --- a/helper-scripts/validate-agent-instructions.js +++ b/helper-scripts/validate-agent-instructions.js @@ -8,6 +8,12 @@ import path from 'path'; import process from 'process'; import { execSync } from 'child_process'; import matter from 'gray-matter'; +import { + INSTRUCTION_LIMIT, + lineLimitError, + ROOT_AGENTS_LIMIT, + SKILL_LIMIT, +} from './agent-instruction-limits.js'; import { buildAgentInstructionAdapters, buildPlatformReference, @@ -25,6 +31,7 @@ const errors = []; validateInstructions(); validateSkills(); +validateLineLimits(); validateClaudeSkillsSymlink(); await validateGeneratedAdapters(); @@ -36,6 +43,27 @@ if (errors.length > 0) { process.exit(1); } +function validateLineLimits() { + validateLineLimit(path.join(PROJECT_ROOT, 'AGENTS.md'), ROOT_AGENTS_LIMIT); + if (fs.existsSync(INSTRUCTIONS_DIR)) { + for (const file of fs.readdirSync(INSTRUCTIONS_DIR).sort()) { + if (file.endsWith('.md')) + validateLineLimit(path.join(INSTRUCTIONS_DIR, file), INSTRUCTION_LIMIT); + } + } + if (fs.existsSync(SKILLS_DIR)) { + for (const entry of fs.readdirSync(SKILLS_DIR).sort()) { + const skillPath = path.join(SKILLS_DIR, entry, 'SKILL.md'); + if (fs.existsSync(skillPath)) validateLineLimit(skillPath, SKILL_LIMIT); + } + } +} + +function validateLineLimit(filePath, limit) { + const error = lineLimitError(filePath, limit, PROJECT_ROOT); + if (error) errors.push(error); +} + console.log('✅ Agent instructions and skills are valid'); function validateInstructions() { diff --git a/layouts/AGENTS.md b/layouts/AGENTS.md index 5d65f7a471..0d07220f1d 100644 --- a/layouts/AGENTS.md +++ b/layouts/AGENTS.md @@ -6,131 +6,23 @@ These instructions apply when working in `layouts/`. -## Layout and Shortcode Implementation Guidelines - -**Shortcodes reference**: [DOCS-SHORTCODES.md](../DOCS-SHORTCODES.md) -**Test examples**: [content/example.md](../content/example.md) - -**For detailed Hugo template development workflow**, see -[hugo-template-dev skill](../.agents/skills/hugo-template-dev/SKILL.md) which covers: - -- Hugo template syntax and data access patterns -- Build-time vs runtime testing strategies -- Shortcode implementation best practices -- Complete TDD workflow for Hugo templates - -### No Magic Values in Template Logic - -Templates operate on data and stay ignorant of the values in that data. -A product name, version segment, or `data/products.yml` key must never appear -as a string literal in template logic. -Nobody should have to edit a template because a product was renamed or added. - -Never write any of these in `layouts/**`: - -- A slice of product names or version segments used in a condition, such as a - list of the versions that count as current or the products that support Flux. -- A single hardcoded product comparison that branches behavior, such as testing - whether the first path segment equals a specific product. -- Deriving a `data/products.yml` key by matching the URL path when the page - already declares one. - -This file is generated into `layouts/AGENTS.md`, and Hugo parses every file -under `layouts/` as a template, so it carries no Go template examples. -For the annotated before and after, see the -[hugo-template-dev skill](../.agents/skills/hugo-template-dev/SKILL.md). - -Do this instead: - -1. Put the fact in `data/products.yml` as a per-product field — a boolean such - as `supports_flux`, `has_support_contract`, or `search_includes_resources` — - and read it with a `| default` that covers products that don't set it. -2. Resolve the product with `partial "product/get-data.html"` or - `partial "product/get-context.html"`, which read the page's cascade `product` - param. - Every product section declares `product` and `version` by cascade in its - section `_index.md`, so the key is stated rather than guessed. -3. When two templates need the same decision, extract it into one partial so - the two can't drift. - `layouts/partials/product/is-latest.html` is the worked example. - -The one exception is a value that must match an external system rather than a -product fact. -The Algolia search tag in `layouts/partials/header/search-attributes.html` -stays path-derived because Algolia indexed every record under the crawled URL. -Comment any such case in the template so the next reader doesn't "fix" it. - -For the before/after example and the incident behind this rule, see -[hugo-template-dev skill](../.agents/skills/hugo-template-dev/SKILL.md). - -### Implementing Shortcodes - -When creating or modifying Hugo layouts and shortcodes: - -1. Use test-driven development using `/cypress/` -2. Use Hugo template syntax and functions -3. Follow existing patterns in `/layouts/shortcodes/` -4. Test in [content/example.md](../content/example.md) -5. Document new shortcodes in [DOCS-SHORTCODES.md](../DOCS-SHORTCODES.md) - -### Shortcode Pattern - -```html -<!-- layouts/shortcodes/example.html --> -{{ $param := .Get 0 }} -{{ $namedParam := .Get "name" }} - -<div class="example"> - {{ .Inner | markdownify }} -</div> -``` - -### Testing - -**IMPORTANT:** Use test-driven development with Cypress. - -Add shortcode usage examples to `content/example.md` to verify: - -- Rendering in browser -- Hugo build succeeds -- No console errors -- JavaScript functionality works as expected (check browser console for errors) -- Interactive elements behave correctly (click links, buttons, etc.) - -#### TDD Workflow - -1. Add Cypress tests (high-level to start). -2. Run tests and make sure they fail. -3. Implement code changes -4. Run tests and make sure they pass. -5. Add and refine tests. -6. Repeat. - -#### Manual Testing Workflow - -1. Make changes to shortcode/layout files -2. Wait for Hugo to rebuild (check terminal output) -3. Get the server URL from the log -4. Open browser DevTools console (F12) -5. Test the functionality and check for JavaScript errors -6. Verify the feature works as intended before marking complete - -See [DOCS-SHORTCODES.md](../DOCS-SHORTCODES.md) for complete shortcode -documentation. - -#### Line protocol render hook - -`layouts/_default/_markup/render-codeblock-lp.html` renders `lp` code fences. -Keep its output Chroma-compatible (`.highlight > pre.chroma > code.language-lp`) -and HTML-escape source text before marking generated markup safe. -For malformed source, render escaped plain text rather than partial highlighting. - -### Related Resources - -- **Complete Hugo template workflow**: - [hugo-template-dev skill](../.agents/skills/hugo-template-dev/SKILL.md) -- **Shortcodes reference**: [DOCS-SHORTCODES.md](../DOCS-SHORTCODES.md) -- **Test examples**: [content/example.md](../content/example.md) -- **Article-level page actions** (buttons/links next to the page title — when - to use, how to add a new one): - [DOCS-PAGE-ACTIONS.md](../DOCS-PAGE-ACTIONS.md) +## Hugo layouts and shortcodes + +Use the [hugo-template-dev skill](../.agents/skills/hugo-template-dev/SKILL.md) for +implementation and runtime verification. Follow existing shortcode patterns and +document user-facing shortcodes in [DOCS-SHORTCODES.md](../DOCS-SHORTCODES.md). + +### Requirements + +- Do not encode product names, versions, URL segments, or product-key guesses + in template branching. Put product facts in `data/products.yml`, resolve page + context through product partials, and share repeated decisions in a partial. +- The sole exception is an externally mandated path-derived value; explain it + in a template comment. +- Add behavior coverage in Cypress and an example in + [content/example.md](../content/example.md) when appropriate. A Hugo build + alone is insufficient for runtime behavior. +- Keep the line-protocol render hook Chroma-compatible and HTML-escape source; + malformed input renders as escaped plain text. + +Run `yarn verify:changed -- <files>` to identify manual checks. diff --git a/package.json b/package.json index cabb194c73..6d16391943 100644 --- a/package.json +++ b/package.json @@ -90,6 +90,9 @@ "build:pytest:image": "docker build -t influxdata/docs-pytest:latest -f Dockerfile.pytest .", "build:agent:instructions": "node ./helper-scripts/build-agent-instructions.js", "validate:agent-instructions": "node ./helper-scripts/validate-agent-instructions.js", + "verify:changed": "node scripts/verify-changed.mjs", + "test:verify-changed": "node --test scripts/__tests__/verify-changed.test.mjs", + "test:agent-instruction-limits": "node --test helper-scripts/__tests__/agent-instruction-limits.test.mjs", "build:ts": "tsc --project tsconfig.json --outDir dist", "build:ts:watch": "tsc --project tsconfig.json --outDir dist --watch", "build:md": "node scripts/build-llm-markdown.js", diff --git a/scripts/__tests__/verify-changed.test.mjs b/scripts/__tests__/verify-changed.test.mjs new file mode 100644 index 0000000000..c1bbbaa1b4 --- /dev/null +++ b/scripts/__tests__/verify-changed.test.mjs @@ -0,0 +1,68 @@ +import assert from 'assert/strict'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import test from 'node:test'; +import { classify, parseArgs, plan, runCommand } from '../verify-changed.mjs'; + +test('validates explicit paths and staged selection', () => { + assert.deepEqual(parseArgs(['--run', 'content/a.md']), { + paths: ['content/a.md'], + run: true, + staged: false, + }); + assert.deepEqual(parseArgs(['--staged']), { + paths: [], + run: false, + staged: true, + }); + assert.throws(() => parseArgs([]), /Provide one/); + assert.throws(() => parseArgs(['--staged', 'content/a.md']), /not both/); +}); +test('classifies supported and deferred paths', () => { + const result = classify(['content/a.md', 'AGENTS.md', 'assets/js/app.ts']); + assert.deepEqual(result.content, ['content/a.md']); + assert.deepEqual(result.agent, ['AGENTS.md']); + assert.deepEqual(result.unsupported, ['assets/js/app.ts']); +}); +test('discovers shared consumers and plans checks', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'verify-changed-')); + fs.mkdirSync(path.join(root, 'content', 'shared'), { recursive: true }); + fs.mkdirSync(path.join(root, 'content', 'influxdb3', 'core'), { + recursive: true, + }); + fs.writeFileSync(path.join(root, 'content/shared/page.md'), 'Body\n'); + fs.writeFileSync( + path.join(root, 'content/influxdb3/core/page.md'), + '---\nsource: /shared/page.md\n---\n' + ); + const result = plan(['content/shared/page.md'], root); + assert.deepEqual(result.consumers, ['content/influxdb3/core/page.md']); + assert.deepEqual(result.commands[1], [ + 'sh', + [ + '-c', + 'link-checker map "$@" | xargs link-checker check', + 'verify:changed', + 'content/shared/page.md', + 'content/influxdb3/core/page.md', + ], + ]); +}); +test('keeps a failure log and deletes successful logs', () => { + const output = { error: () => {}, log: () => {} }; + const failed = runCommand('node', ['-e', ''], { + output, + spawn: () => ({ status: 1, stdout: '', stderr: 'failure\n' }), + }); + assert.equal(failed.ok, false); + assert.equal(fs.existsSync(failed.log), true); + fs.unlinkSync(failed.log); + assert.deepEqual( + runCommand('node', ['-e', ''], { + output, + spawn: () => ({ status: 0, stdout: 'ok', stderr: '' }), + }), + { ok: true } + ); +}); diff --git a/scripts/verify-changed.mjs b/scripts/verify-changed.mjs new file mode 100644 index 0000000000..756b3aa584 --- /dev/null +++ b/scripts/verify-changed.mjs @@ -0,0 +1,188 @@ +#!/usr/bin/env node + +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import process from 'process'; +import { execFileSync, spawnSync } from 'child_process'; +import { fileURLToPath } from 'url'; + +const ROOT = process.cwd(); +const CONTENT = /^content\/.+\.md$/; +const SHARED = /^content\/shared\/.+\.md$/; +const AGENT = + /^(AGENTS\.md|\.agents\/(instructions\/.*\.md|skills\/[^/]+\/SKILL\.md))$/; + +export function parseArgs(args) { + let run = false; + let staged = false; + const paths = []; + for (const arg of args) { + if (arg === '--run') run = true; + else if (arg === '--staged') staged = true; + else if (arg.startsWith('--')) throw new Error(`Unknown option: ${arg}`); + else paths.push(normalize(arg)); + } + if (staged && paths.length) + throw new Error('Use paths or --staged, not both.'); + if (!staged && !paths.length) + throw new Error('Provide one or more paths, or use --staged.'); + return { paths, run, staged }; +} + +export function normalize(value) { + return value.replace(/^\.\//, '').split(path.sep).join('/'); +} + +export function stagedPaths(root = ROOT) { + return execFileSync( + 'git', + ['diff', '--cached', '--name-only', '--diff-filter=ACMR'], + { + cwd: root, + encoding: 'utf8', + } + ) + .split('\n') + .filter(Boolean) + .map(normalize); +} + +export function sharedConsumers(sharedPath, root = ROOT) { + const source = `/${sharedPath.replace(/^content\//, '')}`; + const result = []; + walk(path.join(root, 'content'), (file) => { + const rel = normalize(path.relative(root, file)); + if (rel === sharedPath || !rel.endsWith('.md')) return; + if ( + fs + .readFileSync(file, 'utf8') + .match(new RegExp(`^source:\\s*${escapeRegex(source)}\\s*$`, 'm')) + ) + result.push(rel); + }); + return result.sort(); +} + +export function classify(paths, root = ROOT) { + const content = paths.filter((file) => CONTENT.test(file)); + const shared = content.filter((file) => SHARED.test(file)); + const agent = paths.filter((file) => AGENT.test(file)); + const unsupported = paths.filter( + (file) => !CONTENT.test(file) && !AGENT.test(file) + ); + const consumers = [ + ...new Set(shared.flatMap((file) => sharedConsumers(file, root))), + ]; + return { agent, content, consumers, shared, unsupported }; +} + +export function plan(paths, root = ROOT) { + const groups = classify(paths, root); + const commands = []; + if (groups.content.length) { + commands.push(['yarn', ['lint-codeblocks', ...groups.content]]); + commands.push([ + 'sh', + [ + '-c', + 'link-checker map "$@" | xargs link-checker check', + 'verify:changed', + ...groups.content, + ...groups.consumers, + ], + ]); + } + if (groups.agent.length) { + commands.push(['yarn', ['build:agent:instructions']]); + commands.push(['yarn', ['validate:agent-instructions']]); + } + return { ...groups, commands }; +} + +export function printPlan(result) { + console.log(`Detected: ${summary(result) || 'no supported files'}.`); + if (result.shared.length) + console.log( + `Shared-content consumers: ${result.consumers.join(', ') || 'none found'}.` + ); + console.log( + 'Commit hooks cover formatting, Vale, source-path checks, and instruction linting; CI covers broader link and render checks.' + ); + if (result.unsupported.length) + console.log( + `Deferred (unsupported in this release): ${result.unsupported.join(', ')}.` + ); + if (!result.commands.length) + console.log('No manual checks are currently selected.'); + for (const [command, args] of result.commands) + console.log(`Would run: ${[command, ...args].join(' ')}`); +} + +export function runCommand( + command, + args, + { root = ROOT, spawn = spawnSync, output = console } = {} +) { + const log = path.join( + os.tmpdir(), + `docs-verify-${Date.now()}-${Math.random().toString(16).slice(2)}.log` + ); + const result = spawn(command, args, { cwd: root, encoding: 'utf8' }); + fs.writeFileSync(log, `${result.stdout || ''}${result.stderr || ''}`); + if (result.status === 0) { + fs.unlinkSync(log); + output.log(`PASS ${[command, ...args].join(' ')}`); + return { ok: true }; + } + const excerpt = fs + .readFileSync(log, 'utf8') + .trim() + .split('\n') + .slice(-12) + .join('\n'); + output.error(`FAIL ${[command, ...args].join(' ')} (log: ${log})`); + if (excerpt) output.error(excerpt); + return { ok: false, log }; +} + +function walk(directory, callback) { + if (!fs.existsSync(directory)) return; + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const file = path.join(directory, entry.name); + if (entry.isDirectory()) walk(file, callback); + else callback(file); + } +} +function escapeRegex(value) { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} +function summary(result) { + return [ + ['content Markdown', result.content.length], + ['shared content', result.shared.length], + ['agent assets', result.agent.length], + ] + .filter(([, count]) => count) + .map(([name, count]) => `${count} ${name}`) + .join(', '); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + try { + const options = parseArgs(process.argv.slice(2)); + const files = options.staged ? stagedPaths() : options.paths; + if (!files.length) throw new Error('The selected target set is empty.'); + const result = plan(files); + printPlan(result); + if (options.run) { + let ok = true; + for (const [command, args] of result.commands) + ok = runCommand(command, args).ok && ok; + if (!ok) process.exitCode = 1; + } + } catch (error) { + console.error(`verify:changed: ${error.message}`); + process.exitCode = 1; + } +} From 07377c6fdbd344ff30674276dc2c5d17b64825de Mon Sep 17 00:00:00 2001 From: Jason Stirnaman <stirnamanj@gmail.com> Date: Tue, 8 Sep 2026 22:30:27 -0500 Subject: [PATCH 8/8] docs(adr): archive stale execution plans What changed: - Removed completed and stale execution plans. - Preserved durable site architecture as three concise ADRs. Why: - Execution plans are not a durable archive, while the retained choices guide future site maintenance. Impact: - Repository decision history now distinguishes architecture from content and rollout history. Verification: - yarn verify:changed -- docs/adr/0005-agent-instruction-adapter-architecture.md docs/adr/0006-product-jsonld-rendering.md docs/adr/0007-release-version-drift-detection.md - .ci/remark/remark.sh docs/adr/0005-agent-instruction-adapter-architecture.md docs/adr/0006-product-jsonld-rendering.md docs/adr/0007-release-version-drift-detection.md --quiet - git diff --check --- ...-agent-instruction-adapter-architecture.md | 13 ++ docs/adr/0006-product-jsonld-rendering.md | 16 ++ .../0007-release-version-drift-detection.md | 15 ++ .../2026-05-13-influxdb3-decision-page.md | 37 ---- ...26-05-14-influxdb3-decision-page-jsonld.md | 37 ---- ...05-15-influxdb3-decision-page-discovery.md | 42 ---- .../2026-05-15-influxdb3-hub-landing.md | 36 ---- .../2026-05-21-robots-named-ai-bots.md | 39 ---- ...-techarticle-softwareapplication-jsonld.md | 89 -------- .../2026-06-29-gh-merge-queue-master.md | 190 ------------------ ...07-21-release-notes-version-drift-check.md | 119 ----------- .../agent-harness-instructions-skills.md | 130 ------------ 12 files changed, 44 insertions(+), 719 deletions(-) create mode 100644 docs/adr/0005-agent-instruction-adapter-architecture.md create mode 100644 docs/adr/0006-product-jsonld-rendering.md create mode 100644 docs/adr/0007-release-version-drift-detection.md delete mode 100644 docs/exec-plans/2026-05-13-influxdb3-decision-page.md delete mode 100644 docs/exec-plans/2026-05-14-influxdb3-decision-page-jsonld.md delete mode 100644 docs/exec-plans/2026-05-15-influxdb3-decision-page-discovery.md delete mode 100644 docs/exec-plans/2026-05-15-influxdb3-hub-landing.md delete mode 100644 docs/exec-plans/2026-05-21-robots-named-ai-bots.md delete mode 100644 docs/exec-plans/2026-05-28-techarticle-softwareapplication-jsonld.md delete mode 100644 docs/exec-plans/2026-06-29-gh-merge-queue-master.md delete mode 100644 docs/exec-plans/2026-07-21-release-notes-version-drift-check.md delete mode 100644 docs/exec-plans/agent-harness-instructions-skills.md diff --git a/docs/adr/0005-agent-instruction-adapter-architecture.md b/docs/adr/0005-agent-instruction-adapter-architecture.md new file mode 100644 index 0000000000..ff0bb31394 --- /dev/null +++ b/docs/adr/0005-agent-instruction-adapter-architecture.md @@ -0,0 +1,13 @@ +# Keep agent sources canonical and harness adapters generated + +Agent guidance has one authored source: `AGENTS.md` for repository-wide rules +and `.agents/` for reusable skills and scoped instructions. Harness-specific +representations are generated files or a symlink, never independent copies. + +This keeps authoring portable while preserving each harness's native discovery +mechanism. Validation treats adapter drift and a broken skill symlink as errors. + +## Consequences + +Update canonical sources, then regenerate and validate the adapters. Do not +edit generated instruction files directly. diff --git a/docs/adr/0006-product-jsonld-rendering.md b/docs/adr/0006-product-jsonld-rendering.md new file mode 100644 index 0000000000..1286d09af9 --- /dev/null +++ b/docs/adr/0006-product-jsonld-rendering.md @@ -0,0 +1,16 @@ +# Render product JSON-LD from the product model + +Structured product metadata is rendered by shared header partials from the +existing product cascade and product data. Eligibility is structural rather +than an opt-in repeated across pages, with a narrow page-level opt-out for +exceptions. + +Each product has one authoritative application entity at its resolved landing +page. Related article entities reference that entity by identifier instead of +repeating incomplete inline objects. + +## Consequences + +Adding a product that participates in the normal product cascade inherits the +rendering model. Changes to the structured-data shape stay in the shared +partials and product-data contract. diff --git a/docs/adr/0007-release-version-drift-detection.md b/docs/adr/0007-release-version-drift-detection.md new file mode 100644 index 0000000000..3d3fdfec96 --- /dev/null +++ b/docs/adr/0007-release-version-drift-detection.md @@ -0,0 +1,15 @@ +# Treat release-version drift checks as advisory CI + +Release-version consistency is evaluated by an explicit, validated mapping and +edition-aware parsing. The check reports actionable reminders but does not +block unrelated pull requests; rendered-link validation remains in the +existing link-check workflow. + +This separates release-specific diagnosis from general site validation and +avoids a fragile blocking gate based on incomplete external availability +signals. + +## Consequences + +New tracked release sources require an explicit mapping and parser coverage. +The check is a safety net, not the authoritative source for version data. diff --git a/docs/exec-plans/2026-05-13-influxdb3-decision-page.md b/docs/exec-plans/2026-05-13-influxdb3-decision-page.md deleted file mode 100644 index ce2c3ec1ca..0000000000 --- a/docs/exec-plans/2026-05-13-influxdb3-decision-page.md +++ /dev/null @@ -1,37 +0,0 @@ -# "Which InfluxDB 3 should I use?" decision page (page only) - -**Status:** Merged 2026-05-15 — PR [#7214](https://github.com/influxdata/docs-v2/pull/7214) -**Refs:** [#7219](https://github.com/influxdata/docs-v2/issues/7219) (tracking) -**Parent:** [#7230](https://github.com/influxdata/docs-v2/issues/7230) (AI visibility — Phase 0) - -## Goal - -Ship the canonical decision page at `/influxdb3/which-influxdb-3/` — the single URL that answers "which InfluxDB 3 product fits my workload?" Renders 7 FAQ Q\&As with deep-linkable anchors, a decision table, per-product sections, and migration links. PR 1 of 4 in the Phase 0 decision-page rollout. - -## Why now - -Phase 0 of the AI visibility epic (#7230) requires a canonical landing for the "which v3?" question. The v3 product family (Core, Enterprise, Cloud Dedicated, Cloud Serverless, Clustered) has grown beyond what users can disambiguate from product index pages alone; LLM-aware tooling needs a citable URL to point at. - -## Decisions - -- **FAQ content in a Hugo data file** (`data/faqs/which-influxdb-3.yml`), not embedded markdown. One source feeds the rendering shortcode now and the JSON-LD partial in PR #7220 — no drift possible. -- **`{{< faq >}}` emits semantic `<h2 id="…">` + `<div class="faq-answer">`**, not `<dl>` or `<details>`. Deep-linkable anchors per question, harvestable by Google's FAQ rich result and by LLM extractors. -- **No `menu:` frontmatter.** The page is cross-cutting; existing product menus are product-scoped, and sidebar placement under any one product would imply false ownership. Discovery comes via direct URL + llms.txt + per-product callouts in PR #7229. -- **`faq_data` + `faq_canonical: true` frontmatter scheme.** Prepared ahead so PR #7220's JSON-LD partial can gate emission per page — single content source, multiple consumers, decision lives on the page. -- **Title-template fix bundled.** Two-segment `/influxdb3/<page>/` URLs previously emitted `<title><nil> Documentation` because `data/products.yml` has no bare `influxdb3` key (only namespaced `influxdb3_*`). Added the case + a question-shaped page title for SEO and LLM extraction. Same review context, same partial. -- **PR-stacked rollout (1 of 4).** Reviewers see one concern per PR instead of a 990-line drop. Each PR's behavior verifiable end-to-end before the next stacks on. - -## Explicitly out of scope - -- FAQPage JSON-LD ([#7220](https://github.com/influxdata/docs-v2/pull/7220)) -- `/influxdb3/` hub landing ([#7228](https://github.com/influxdata/docs-v2/pull/7228)) -- Cross-link callouts, llms.txt, platform FAQ pointer ([#7229](https://github.com/influxdata/docs-v2/pull/7229)) - -## How to update - -Edit `data/faqs/which-influxdb-3.yml` to add or change FAQ Q\&As. The shortcode regenerates HTML on every Hugo build; the JSON-LD partial (PR #7220) regenerates structured data from the same file. Decision-page prose lives in `content/shared/influxdb3/which-influxdb-3.md`. - -## Verification - -- `node cypress/support/run-e2e-specs.js --spec "cypress/e2e/content/which-influxdb-3.cy.js" --no-mapping` → 8/8 pass: H1, 7 FAQ Q\&As as semantic HTML with stable anchors, decision table, migration links, self-canonical, no raw markdown leakage -- `npx hugo --quiet && grep -c "What's the difference between InfluxDB 1" public/influxdb3/which-influxdb-3/index.html` → ≥ 1 diff --git a/docs/exec-plans/2026-05-14-influxdb3-decision-page-jsonld.md b/docs/exec-plans/2026-05-14-influxdb3-decision-page-jsonld.md deleted file mode 100644 index 9e83583f49..0000000000 --- a/docs/exec-plans/2026-05-14-influxdb3-decision-page-jsonld.md +++ /dev/null @@ -1,37 +0,0 @@ -# FAQPage JSON-LD for the InfluxDB 3 decision page - -**Status:** Merged 2026-05-15 — PR [#7220](https://github.com/influxdata/docs-v2/pull/7220) -**Refs:** [#7219](https://github.com/influxdata/docs-v2/issues/7219) (tracking) -**Parent:** [#7230](https://github.com/influxdata/docs-v2/issues/7230) (AI visibility — Phase 0) - -## Goal - -Emit schema.org `FAQPage` JSON-LD on `/influxdb3/which-influxdb-3/` so the page is eligible for Google's FAQ rich result and so LLM-aware tooling can consume structured Q\&A data. PR 2 of 4 in the decision-page rollout. - -## Why now - -PR #7214 shipped the rendered FAQ HTML; structured data is the second half of the AI/SEO value. Landing it as a separate PR isolates the JSON-LD shape and the gate logic for independent review. - -## Decisions - -- **Partial reads the same `data/faqs/.yml` file the `{{< faq >}}` shortcode consumes.** Single source of truth; HTML and structured data can't drift. -- **Gated on `faq_data` AND `faq_canonical: true`** (both required). Pages that transclude the same shared FAQ body — like the `/influxdb3/` hub coming in PR #7228 — won't double-emit. Canonical equity stays on the slug URL. -- **`safeJS` wrapper around `jsonify`.** Go's `html/template` applies JS-context escaping inside `