diff --git a/docs/examples/neo4j/autoscaler/compute/neo4j-compute-autoscaler.yaml b/docs/examples/neo4j/autoscaler/compute/neo4j-compute-autoscaler.yaml new file mode 100644 index 0000000000..a2a2f0465f --- /dev/null +++ b/docs/examples/neo4j/autoscaler/compute/neo4j-compute-autoscaler.yaml @@ -0,0 +1,27 @@ +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: Neo4jAutoscaler +metadata: + name: neo4j-compute-autoscaler + namespace: demo +spec: + databaseRef: + name: neo4j-autoscale + opsRequestOptions: + apply: IfReady + timeout: 10m + maxRetries: 3 + compute: + neo4j: + trigger: "On" + podLifeTimeThreshold: 5m + resourceDiffPercentage: 20 + minAllowed: + cpu: 600m + memory: 2500Mi + maxAllowed: + cpu: "2" + memory: 4Gi + controlledResources: + - cpu + - memory + containerControlledValues: RequestsAndLimits diff --git a/docs/examples/neo4j/autoscaler/neo4j.yaml b/docs/examples/neo4j/autoscaler/neo4j.yaml new file mode 100644 index 0000000000..6d671d4b8b --- /dev/null +++ b/docs/examples/neo4j/autoscaler/neo4j.yaml @@ -0,0 +1,28 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: neo4j-autoscale + namespace: demo +spec: + version: "2025.12.1" + replicas: 3 + storageType: Durable + storage: + storageClassName: longhorn + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + podTemplate: + spec: + containers: + - name: neo4j + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: 500m + memory: 2Gi + deletionPolicy: WipeOut diff --git a/docs/examples/neo4j/autoscaler/storage/neo4j-storage-autoscaler.yaml b/docs/examples/neo4j/autoscaler/storage/neo4j-storage-autoscaler.yaml new file mode 100644 index 0000000000..13472d64d6 --- /dev/null +++ b/docs/examples/neo4j/autoscaler/storage/neo4j-storage-autoscaler.yaml @@ -0,0 +1,18 @@ +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: Neo4jAutoscaler +metadata: + name: neo4j-storage-autoscaler + namespace: demo +spec: + databaseRef: + name: neo4j-autoscale + opsRequestOptions: + apply: IfReady + timeout: 10m + maxRetries: 3 + storage: + neo4j: + trigger: "On" + usageThreshold: 40 + scalingThreshold: 50 + expansionMode: Online diff --git a/docs/guides/neo4j/README.md b/docs/guides/neo4j/README.md index fab4c488eb..3c6398f3d6 100644 --- a/docs/guides/neo4j/README.md +++ b/docs/guides/neo4j/README.md @@ -91,6 +91,7 @@ spec: ### Day-2 Operations +- [Point-in-time Recovery](/docs/guides/neo4j/pitr/archiver.md) — archive full and differential backups, then recover a graph to a timestamp before an accidental change. - [TLS — How It Works](/docs/guides/neo4j/tls/overview/) — how KubeDB provisions TLS certificates via cert-manager. - [Configure TLS](/docs/guides/neo4j/tls/configure/) — enable TLS on a new or existing cluster. - [Reconfigure — How It Works](/docs/guides/neo4j/reconfigure/overview.md) — how KubeDB applies config changes internally. @@ -108,4 +109,4 @@ spec: - [Vertical Scaling](/docs/guides/neo4j/scaling/vertical-scaling/scale-vertically/) — resize CPU and memory for Neo4j pods. - [Volume Expansion — How It Works](/docs/guides/neo4j/volume-expansion/overview.md) — how KubeDB expands PVCs. - [Volume Expansion](/docs/guides/neo4j/volume-expansion/volume-expansion.md) — increase persistent storage size online or offline. -- [StorageClass Migration](/docs/guides/neo4j/migration/storagemigration.md) — migrate Neo4j data to a different StorageClass. \ No newline at end of file +- [StorageClass Migration](/docs/guides/neo4j/migration/storagemigration.md) — migrate Neo4j data to a different StorageClass. diff --git a/docs/guides/neo4j/autoscaler/_index.md b/docs/guides/neo4j/autoscaler/_index.md new file mode 100644 index 0000000000..fdd9a0f4ae --- /dev/null +++ b/docs/guides/neo4j/autoscaler/_index.md @@ -0,0 +1,10 @@ +--- +title: Autoscaling +menu: + docs_{{ .version }}: + identifier: neo4j-autoscaling + name: Autoscaling + parent: neo4j-guides + weight: 90 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/neo4j/autoscaler/compute/_index.md b/docs/guides/neo4j/autoscaler/compute/_index.md new file mode 100644 index 0000000000..537e2446ce --- /dev/null +++ b/docs/guides/neo4j/autoscaler/compute/_index.md @@ -0,0 +1,10 @@ +--- +title: Compute Autoscaling +menu: + docs_{{ .version }}: + identifier: neo4j-compute-autoscaling + name: Compute Autoscaling + parent: neo4j-autoscaling + weight: 10 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/neo4j/autoscaler/compute/autoscale.md b/docs/guides/neo4j/autoscaler/compute/autoscale.md new file mode 100644 index 0000000000..ad65515b8a --- /dev/null +++ b/docs/guides/neo4j/autoscaler/compute/autoscale.md @@ -0,0 +1,265 @@ +--- +title: Autoscale Neo4j Compute Resources +menu: + docs_{{ .version }}: + identifier: neo4j-compute-autoscaling-guide + name: Autoscale Compute Resources + parent: neo4j-compute-autoscaling + weight: 20 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Autoscale Neo4j Compute Resources + +This guide deploys a Neo4j cluster, loads a small social graph, and configures KubeDB to adjust CPU and memory automatically. After scaling, we query the graph again to verify that the data remains available. + +## Before You Begin + +| Requirement | Details | +|---|---| +| KubeDB | Provisioner, Ops Manager, and Autoscaler operators must be installed. | +| Metrics Server | Install [Kubernetes Metrics Server](https://github.com/kubernetes-sigs/metrics-server#installation) so the recommender can observe pod usage. | +| Storage | This example uses a `longhorn` StorageClass; substitute another available class if necessary. | +| Tools | `kubectl`, `jq`, and `base64` must be available locally. | + +See [Neo4jAutoscaler](/docs/guides/neo4j/concepts/autoscaler.md) and the [compute autoscaling overview](/docs/guides/neo4j/autoscaler/compute/overview.md) for background. + +## Deploy Neo4j + +Create an isolated namespace. The examples in this guide use the `longhorn` StorageClass. If your cluster uses a different StorageClass, change `spec.storage.storageClassName` before applying the manifest. + +```bash +$ kubectl create namespace demo +namespace/demo created +``` + +The following manifest creates a three-member Neo4j cluster. Neo4j requires at least `2Gi` of storage per member, so each pod receives its own `2Gi` persistent volume: + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: neo4j-autoscale + namespace: demo +spec: + version: "2025.12.1" + replicas: 3 + storageType: Durable + storage: + storageClassName: longhorn + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + podTemplate: + spec: + containers: + - name: neo4j + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: 500m + memory: 2Gi + deletionPolicy: WipeOut +``` + +Here, `spec.version` selects an installed `Neo4jVersion`, `replicas: 3` creates a fault-tolerant cluster, and `storageType: Durable` preserves data across pod restarts. `deletionPolicy: WipeOut` removes the database-owned PVCs and credentials when the Neo4j resource is deleted, so use a safer deletion policy when retention is required. + +Apply the same manifest from the examples directory: + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/neo4j/autoscaler/neo4j.yaml +neo4j.kubedb.com/neo4j-autoscale created +``` + +Wait for the cluster to become ready: + +```bash +$ kubectl get neo4j -n demo neo4j-autoscale -w +NAME VERSION STATUS AGE +neo4j-autoscale 2025.12.1 Ready 3m +``` + +The Neo4j container initially requests and limits `500m` CPU and `2Gi` memory. Neo4j needs enough memory for the JVM, page cache, and native allocations; limits that are too small can cause the process to be OOM-killed. + +```bash +$ kubectl get pod -n demo neo4j-autoscale-0 \ + -o jsonpath='{.spec.containers[?(@.name=="neo4j")].resources}' | jq . +{ + "limits": {"cpu": "500m", "memory": "2Gi"}, + "requests": {"cpu": "500m", "memory": "2Gi"} +} +``` + +## Create a Sample Graph + +Read the generated admin password, create an application database, and wait until it is online: + +```bash +$ PASS=$(kubectl get secret -n demo neo4j-autoscale-auth \ + -o jsonpath='{.data.password}' | base64 -d) + +$ kubectl exec -n demo neo4j-autoscale-0 -- \ + cypher-shell -u neo4j -p "$PASS" \ + "CREATE DATABASE appdb IF NOT EXISTS WAIT" +``` + +Create a uniqueness constraint, then use `MERGE` to load users and `FOLLOWS` relationships. These commands are safe to repeat because they do not create duplicate users or relationships: + +```bash +$ kubectl exec -n demo neo4j-autoscale-0 -- \ + cypher-shell -d appdb -u neo4j -p "$PASS" \ + "CREATE CONSTRAINT user_id IF NOT EXISTS + FOR (u:User) REQUIRE u.id IS UNIQUE" + +$ kubectl exec -n demo neo4j-autoscale-0 -- \ + cypher-shell -d appdb -u neo4j -p "$PASS" \ + "UNWIND range(1,10000) AS i + MERGE (u:User {id: i}) + SET u.name = 'user-' + toString(i)" + +$ kubectl exec -n demo neo4j-autoscale-0 -- \ + cypher-shell -d appdb -u neo4j -p "$PASS" \ + "UNWIND range(1,9999) AS i + MATCH (a:User {id: i}), (b:User {id: i + 1}) + MERGE (a)-[:FOLLOWS]->(b)" +``` + +Verify the initial graph: + +```bash +$ kubectl exec -n demo neo4j-autoscale-0 -- \ + cypher-shell -d appdb -u neo4j -p "$PASS" \ + "MATCH (u:User) OPTIONAL MATCH (u)-[r:FOLLOWS]->() + RETURN count(DISTINCT u) AS users, count(r) AS follows" +users, follows +10000, 9999 +``` + +## Create the Neo4jAutoscaler + +The example policy permits recommendations from `600m` to `2` CPU and from `2500Mi` to `4Gi` memory: + +```yaml +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: Neo4jAutoscaler +metadata: + name: neo4j-compute-autoscaler + namespace: demo +spec: + databaseRef: + name: neo4j-autoscale + opsRequestOptions: + apply: IfReady + timeout: 10m + maxRetries: 3 + compute: + neo4j: + trigger: "On" + podLifeTimeThreshold: 5m + resourceDiffPercentage: 20 + minAllowed: + cpu: 600m + memory: 2500Mi + maxAllowed: + cpu: "2" + memory: 4Gi + controlledResources: + - cpu + - memory + containerControlledValues: RequestsAndLimits +``` + +Apply it: + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/neo4j/autoscaler/compute/neo4j-compute-autoscaler.yaml +neo4jautoscaler.autoscaling.kubedb.com/neo4j-compute-autoscaler created +``` + +The minimum values are deliberately higher than the initial allocation, making this tutorial reproducible. In production, choose bounds based on workload requirements and capacity. + +## Observe the Recommendation and Scaling + +Run a read workload while the recommender gathers samples: + +```bash +$ for i in $(seq 1 100); do + kubectl exec -n demo neo4j-autoscale-0 -- \ + cypher-shell -d appdb -u neo4j -p "$PASS" \ + "MATCH (u:User)-[:FOLLOWS*1..3]->(v:User) + RETURN count(v)" >/dev/null 2>&1 || true + done +``` + +The Autoscaler may start the rolling resize while the loop is still running. The `|| true` allows this workload generator to continue past a temporary connection failure while a pod is being replaced. Application clients should use bounded retries with backoff for the same condition. + +After the `podLifeTimeThreshold` has passed, inspect the recommendation: + +```bash +$ kubectl get neo4jautoscaler -n demo neo4j-compute-autoscaler \ + -o jsonpath='{.status.vpas[*].recommendation.containerRecommendations}' | jq . +[ + { + "containerName": "neo4j", + "lowerBound": {"cpu": "600m", "memory": "2500Mi"}, + "target": {"cpu": "716m", "memory": "2500Mi"}, + "upperBound": {"cpu": "2", "memory": "4Gi"} + } +] +``` + +KubeDB creates a `Neo4jOpsRequest` when the recommendation differs sufficiently from the current resources: + +```bash +$ kubectl get neo4jopsrequest -n demo -w +NAME TYPE STATUS AGE +neoops-neo4j-autoscale-xxxxxx VerticalScaling Successful 2m +``` + +Verify that the pod allocation is now within the configured bounds: + +```bash +$ kubectl get pod -n demo neo4j-autoscale-0 \ + -o jsonpath='{.spec.containers[?(@.name=="neo4j")].resources}' | jq . +{ + "limits": {"cpu": "600m", "memory": "2500Mi"}, + "requests": {"cpu": "600m", "memory": "2500Mi"} +} +``` + +The precise recommendation and completion time depend on observed usage and the available samples. The applied allocation will stay between `600m` and `2` CPU and between `2500Mi` and `4Gi` memory. + +## Verify the Graph + +Confirm that the database still contains the users and relationships after the scaling rollout: + +```bash +$ kubectl exec -n demo neo4j-autoscale-0 -- \ + cypher-shell -d appdb -u neo4j -p "$PASS" \ + "MATCH (u:User) OPTIONAL MATCH (u)-[r:FOLLOWS]->() + RETURN count(DISTINCT u) AS users, count(r) AS follows" +users, follows +10000, 9999 +``` + +## Troubleshooting + +- If no recommendation appears, verify Metrics Server with `kubectl top pod -n demo` and wait for more samples. +- Check the Autoscaler's conditions with `kubectl describe neo4jautoscaler -n demo neo4j-compute-autoscaler` before changing its thresholds. +- If no OpsRequest is created, check `podLifeTimeThreshold`, `resourceDiffPercentage`, and the Autoscaler conditions. +- If an operation remains pending, describe it with `kubectl describe neo4jopsrequest -n demo ` and check Ops Manager logs. + +## Cleaning Up + +```bash +$ kubectl delete neo4jautoscaler -n demo neo4j-compute-autoscaler +$ kubectl delete neo4j -n demo neo4j-autoscale +$ kubectl delete namespace demo +``` diff --git a/docs/guides/neo4j/autoscaler/compute/overview.md b/docs/guides/neo4j/autoscaler/compute/overview.md new file mode 100644 index 0000000000..f3e862fbbc --- /dev/null +++ b/docs/guides/neo4j/autoscaler/compute/overview.md @@ -0,0 +1,44 @@ +--- +title: Neo4j Compute Autoscaling Overview +menu: + docs_{{ .version }}: + identifier: neo4j-compute-autoscaling-overview + name: Overview + parent: neo4j-compute-autoscaling + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Neo4j Compute Autoscaling + +KubeDB can automatically adjust the CPU and memory assigned to Neo4j pods. A `Neo4jAutoscaler` observes real resource usage, generates recommendations, and creates a `Neo4jOpsRequest` when the recommended resources differ sufficiently from the current allocation. + +## Before You Begin + +You should be familiar with: + +- [Neo4j](/docs/guides/neo4j/concepts/neo4j.md) +- [Neo4jAutoscaler](/docs/guides/neo4j/concepts/autoscaler.md) +- [Neo4jOpsRequest](/docs/guides/neo4j/concepts/opsrequest.md) +- [Neo4j vertical scaling](/docs/guides/neo4j/scaling/vertical-scaling/overview.md) + +Install [Kubernetes Metrics Server](https://github.com/kubernetes-sigs/metrics-server#installation) before enabling compute autoscaling. The recommender uses resource metrics collected from the Neo4j pods. + +## How Compute Autoscaling Works + +
+ Compute autoscaling process for Neo4j +
Fig: Neo4j compute autoscaling process
+
+ +1. The user creates a KubeDB `Neo4j` resource. +2. The Provisioner creates the Neo4j cluster and its supporting Kubernetes resources. +3. The user creates a `Neo4jAutoscaler` with a `spec.compute.neo4j` policy. +4. The Autoscaler creates and watches a Vertical Pod Autoscaler recommendation for the Neo4j container. +5. After the pod lifetime and resource-difference thresholds are satisfied, the Autoscaler creates a `Neo4jOpsRequest` of type `VerticalScaling`. +6. Ops Manager applies the recommendation and updates the Neo4j pods within the configured minimum and maximum bounds. + +The next guide demonstrates this workflow end to end. diff --git a/docs/guides/neo4j/autoscaler/storage/_index.md b/docs/guides/neo4j/autoscaler/storage/_index.md new file mode 100644 index 0000000000..626a1b0ea2 --- /dev/null +++ b/docs/guides/neo4j/autoscaler/storage/_index.md @@ -0,0 +1,10 @@ +--- +title: Storage Autoscaling +menu: + docs_{{ .version }}: + identifier: neo4j-storage-autoscaling + name: Storage Autoscaling + parent: neo4j-autoscaling + weight: 20 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/neo4j/autoscaler/storage/autoscale.md b/docs/guides/neo4j/autoscaler/storage/autoscale.md new file mode 100644 index 0000000000..ba4f9b373c --- /dev/null +++ b/docs/guides/neo4j/autoscaler/storage/autoscale.md @@ -0,0 +1,251 @@ +--- +title: Autoscale Neo4j Storage +menu: + docs_{{ .version }}: + identifier: neo4j-storage-autoscaling-guide + name: Autoscale Storage + parent: neo4j-storage-autoscaling + weight: 20 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Autoscale Neo4j Storage + +This guide configures KubeDB to expand Neo4j data volumes automatically. We insert payload-bearing graph records until the demonstration threshold is crossed, observe the generated volume-expansion operation, and verify both the new capacity and the stored data. + +## Before You Begin + +| Requirement | Details | +|---|---| +| KubeDB | Provisioner, Ops Manager, and Autoscaler operators must be installed. | +| Storage metrics | Install KubeDB with `--set kubedb-autoscaler.storage-metrics-server.enabled=true`. | +| Expandable storage | This example uses `longhorn`; the selected StorageClass must report `ALLOWVOLUMEEXPANSION=true`. | +| Tools | `kubectl`, `base64`, and a POSIX-compatible shell must be available locally. | + +See [Neo4jAutoscaler](/docs/guides/neo4j/concepts/autoscaler.md) and the [storage autoscaling overview](/docs/guides/neo4j/autoscaler/storage/overview.md) for background. + +Verify storage expansion and the custom metrics API before continuing: + +```bash +$ kubectl get storageclass longhorn +NAME PROVISIONER RECLAIMPOLICY ALLOWVOLUMEEXPANSION +longhorn driver.longhorn.io Delete true + +$ kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1 | head +``` + +## Deploy Neo4j + +Create an isolated namespace: + +```bash +$ kubectl create namespace demo +namespace/demo created +``` + +The following manifest creates a three-member Neo4j cluster with the minimum `2Gi` of storage per member. The selected StorageClass must support volume expansion: + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: neo4j-autoscale + namespace: demo +spec: + version: "2025.12.1" + replicas: 3 + storageType: Durable + storage: + storageClassName: longhorn + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + podTemplate: + spec: + containers: + - name: neo4j + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: 500m + memory: 2Gi + deletionPolicy: WipeOut +``` + +Here, `spec.version` selects an installed `Neo4jVersion`, `replicas: 3` creates a fault-tolerant cluster, and `storageType: Durable` provisions one PVC per member. `deletionPolicy: WipeOut` is convenient for a disposable tutorial but also removes the database-owned PVCs and credentials when the Neo4j resource is deleted. + +Apply the same manifest from the examples directory and wait for Neo4j to become ready: + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/neo4j/autoscaler/neo4j.yaml +neo4j.kubedb.com/neo4j-autoscale created + +$ kubectl get neo4j -n demo neo4j-autoscale -w +NAME VERSION STATUS AGE +neo4j-autoscale 2025.12.1 Ready 3m +``` + +Confirm the initial PVC capacities: + +```bash +$ kubectl get pvc -n demo -l app.kubernetes.io/instance=neo4j-autoscale \ + -o custom-columns=NAME:.metadata.name,CAPACITY:.status.capacity.storage +NAME CAPACITY +data-neo4j-autoscale-0 2Gi +data-neo4j-autoscale-1 2Gi +data-neo4j-autoscale-2 2Gi +``` + +## Create the Neo4jAutoscaler + +The following policy triggers when a data volume reaches `40%` usage and increases its current size by `50%`: + +```yaml +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: Neo4jAutoscaler +metadata: + name: neo4j-storage-autoscaler + namespace: demo +spec: + databaseRef: + name: neo4j-autoscale + opsRequestOptions: + apply: IfReady + timeout: 10m + maxRetries: 3 + storage: + neo4j: + trigger: "On" + usageThreshold: 40 + scalingThreshold: 50 + expansionMode: Online +``` + +Apply it and confirm that it is active: + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/examples/neo4j/autoscaler/storage/neo4j-storage-autoscaler.yaml +neo4jautoscaler.autoscaling.kubedb.com/neo4j-storage-autoscaler created + +$ kubectl get neo4jautoscaler -n demo neo4j-storage-autoscaler +NAME AGE +neo4j-storage-autoscaler 10s +``` + +> The `40%` threshold is intentionally low so the tutorial completes quickly. It also remains above the observed usage after the first expansion, preventing this sample workload from immediately triggering another operation. Use a higher threshold, such as `80%`, for a production policy and leave sufficient headroom for traffic spikes and expansion time. + +## Insert Graph Data + +Retrieve the admin password, create an application database, and add a uniqueness constraint so a batch can be retried safely: + +```bash +$ PASS=$(kubectl get secret -n demo neo4j-autoscale-auth \ + -o jsonpath='{.data.password}' | base64 -d) + +$ kubectl exec -n demo neo4j-autoscale-0 -- \ + cypher-shell -u neo4j -p "$PASS" \ + "CREATE DATABASE appdb IF NOT EXISTS WAIT" + +$ kubectl exec -n demo neo4j-autoscale-0 -- \ + cypher-shell -d appdb -u neo4j -p "$PASS" \ + "CREATE CONSTRAINT event_id IF NOT EXISTS + FOR (e:Event) REQUIRE e.id IS UNIQUE" +``` + +Insert events in bounded transactions. Every event contains a payload of approximately 1 KiB, so this creates real Neo4j store and transaction-log growth without writing unrelated files into the volume: + +```bash +$ START_BATCH=0 +$ for batch in $(seq "$START_BATCH" "$((START_BATCH + 99))"); do + kubectl exec -n demo neo4j-autoscale-0 -- \ + cypher-shell -d appdb -u neo4j -p "$PASS" \ + "UNWIND range(1,2500) AS i + MERGE (e:Event {id: $((batch * 2500)) + i}) + ON CREATE SET + e.source = 'storage-autoscaling-demo', + e.payload = reduce(s = '', n IN range(1,32) | s + randomUUID())" + done +``` + +Check the data and filesystem usage: + +```bash +$ kubectl exec -n demo neo4j-autoscale-0 -- \ + cypher-shell -d appdb -u neo4j -p "$PASS" \ + "MATCH (e:Event) RETURN count(e) AS events" +events +250000 + +$ kubectl exec -n demo neo4j-autoscale-0 -- df -h /data +Filesystem Size Used Avail Use% Mounted on +/dev/longhorn 2.0G 962M 955M 51% /data +``` + +Actual usage varies because Neo4j store files and the storage backend have their own overhead. If usage is still below `40%`, set `START_BATCH=100` and run another batch. Increase it by `100` for each additional run, and stop inserting once the threshold is crossed. + +## Observe Volume Expansion + +The Autoscaler creates a `Neo4jOpsRequest` of type `VolumeExpansion` after the storage metric reaches the threshold: + +```bash +$ kubectl get neo4jopsrequest -n demo -w +NAME TYPE STATUS AGE +neoops-neo4j-autoscale-xxxxxx VolumeExpansion Progressing 20s +neoops-neo4j-autoscale-xxxxxx VolumeExpansion Successful 2m +``` + +Inspect the operation to see the calculated target and completed steps: + +```bash +$ kubectl describe neo4jopsrequest -n demo neoops-neo4j-autoscale-xxxxxx +``` + +With a `50%` scaling threshold, the target is approximately 50% larger than the usable capacity reported by the storage metrics. A nominal `2Gi` PVC produced `2920Mi` PVCs in this test; the exact value can differ slightly by filesystem and storage backend: + +```bash +$ kubectl get pvc -n demo -l app.kubernetes.io/instance=neo4j-autoscale \ + -o custom-columns=NAME:.metadata.name,CAPACITY:.status.capacity.storage +NAME CAPACITY +data-neo4j-autoscale-0 2920Mi +data-neo4j-autoscale-1 2920Mi +data-neo4j-autoscale-2 2920Mi +``` + +Verify the expanded filesystem and the application data: + +```bash +$ kubectl exec -n demo neo4j-autoscale-0 -- df -h /data +Filesystem Size Used Avail Use% Mounted on +/dev/longhorn 2.8G 962M 1.8G 35% /data + +$ kubectl exec -n demo neo4j-autoscale-0 -- \ + cypher-shell -d appdb -u neo4j -p "$PASS" \ + "MATCH (e:Event) RETURN count(e) AS events" +events +250000 +``` + +## Troubleshooting + +- If the custom metrics endpoint is unavailable, verify that the KubeDB storage metrics server is enabled and healthy. +- If no OpsRequest appears, describe the Autoscaler and confirm that `/data` usage is above `usageThreshold`. +- If a PVC remains at its old capacity, confirm `allowVolumeExpansion: true` and inspect PVC events. +- Check the storage backend's health before testing expansion. For example, every Longhorn volume must have enough schedulable replicas; a degraded volume can reject resize requests. +- If a PVC reports `FileSystemResizePending`, the CSI driver requires the volume to be remounted before the filesystem sees the new capacity. Use `expansionMode: Offline` for such drivers; Neo4j pods will be restarted during expansion. + +## Cleaning Up + +Deleting the example database with `deletionPolicy: WipeOut` also deletes its data volumes: + +```bash +$ kubectl delete neo4jautoscaler -n demo neo4j-storage-autoscaler +$ kubectl delete neo4j -n demo neo4j-autoscale +$ kubectl delete namespace demo +``` diff --git a/docs/guides/neo4j/autoscaler/storage/overview.md b/docs/guides/neo4j/autoscaler/storage/overview.md new file mode 100644 index 0000000000..abc4c28627 --- /dev/null +++ b/docs/guides/neo4j/autoscaler/storage/overview.md @@ -0,0 +1,44 @@ +--- +title: Neo4j Storage Autoscaling Overview +menu: + docs_{{ .version }}: + identifier: neo4j-storage-autoscaling-overview + name: Overview + parent: neo4j-storage-autoscaling + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Neo4j Storage Autoscaling + +KubeDB can automatically expand Neo4j persistent volumes as graph data grows. The Autoscaler watches volume usage and creates a volume-expansion operation before the disks become full. + +## Before You Begin + +You should be familiar with: + +- [Neo4j](/docs/guides/neo4j/concepts/neo4j.md) +- [Neo4jAutoscaler](/docs/guides/neo4j/concepts/autoscaler.md) +- [Neo4jOpsRequest](/docs/guides/neo4j/concepts/opsrequest.md) +- [Neo4j volume expansion](/docs/guides/neo4j/volume-expansion/overview.md) + +## How Storage Autoscaling Works + +
+ Storage autoscaling process for Neo4j +
Fig: Neo4j storage autoscaling process
+
+ +1. The user creates a KubeDB `Neo4j` resource with durable storage. +2. The Provisioner creates a persistent volume for every Neo4j pod. +3. The user creates a `Neo4jAutoscaler` with a `spec.storage.neo4j` policy. +4. The Autoscaler reads PVC usage from the KubeDB storage metrics API. +5. When usage reaches `usageThreshold`, the Autoscaler calculates a larger size and creates a `Neo4jOpsRequest` of type `VolumeExpansion`. +6. Ops Manager expands the PVCs using the configured online or offline mode. + +> Volume expansion requires a StorageClass with `allowVolumeExpansion: true`. Kubernetes does not support shrinking a PVC after it has been expanded. + +The next guide demonstrates this workflow using actual Neo4j graph data. diff --git a/docs/guides/neo4j/backup/kubestash/logical/_index.md b/docs/guides/neo4j/backup/kubestash/logical/_index.md new file mode 100644 index 0000000000..dd0d220614 --- /dev/null +++ b/docs/guides/neo4j/backup/kubestash/logical/_index.md @@ -0,0 +1,10 @@ +--- +title: Logical Backup & Restore of Neo4j | KubeStash +menu: + docs_{{ .version }}: + identifier: guides-neo4j-logical-backup-stashv2 + name: Logical Backup + parent: guides-neo4j-backup-stashv2 + weight: 20 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/backupconfiguration.yaml b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/backupconfiguration.yaml new file mode 100644 index 0000000000..73cbc526da --- /dev/null +++ b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/backupconfiguration.yaml @@ -0,0 +1,33 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: source-neo4j-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Neo4j + namespace: demo + name: source-neo4j + backends: + - name: s3-backend + storageRef: + namespace: demo + name: s3-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: s3-neo4j-composite-repo + backend: s3-backend + directory: /neo4j-composite + addon: + name: neo4j-addon + tasks: + - name: logical-backup diff --git a/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/backupstorage.yaml b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/backupstorage.yaml new file mode 100644 index 0000000000..41e0daca04 --- /dev/null +++ b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/backupstorage.yaml @@ -0,0 +1,19 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: s3-storage + namespace: demo +spec: + storage: + provider: s3 + s3: + bucket: + region: + endpoint: + secretName: s3-secret + prefix: demo + usagePolicy: + allowedNamespaces: + from: All + default: false + deletionPolicy: Delete diff --git a/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/remote-alias-credentials.yaml b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/remote-alias-credentials.yaml new file mode 100644 index 0000000000..c41582cf1e --- /dev/null +++ b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/remote-alias-credentials.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: neo4j-remote-alias-credentials + namespace: demo +type: Opaque +stringData: + credentials.yaml: | + media.reviews: + password: "" + reviews-remote: + password: "" diff --git a/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/remote-neo4j.yaml b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/remote-neo4j.yaml new file mode 100644 index 0000000000..abe0f9306b --- /dev/null +++ b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/remote-neo4j.yaml @@ -0,0 +1,16 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: remote-neo4j + namespace: demo +spec: + version: 2025.12.1 + replicas: 3 + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + deletionPolicy: WipeOut diff --git a/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/restored-neo4j.yaml b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/restored-neo4j.yaml new file mode 100644 index 0000000000..89c000dc22 --- /dev/null +++ b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/restored-neo4j.yaml @@ -0,0 +1,25 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: restored-neo4j + namespace: demo +spec: + version: 2025.12.1 + replicas: 3 + configuration: + remoteAliasKeystore: + keystoreRef: + name: neo4j-remote-alias-keystore + key: aes + passwordRef: + name: neo4j-remote-alias-keystore + key: password + keyName: neo + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + deletionPolicy: WipeOut diff --git a/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/restoresession.yaml b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/restoresession.yaml new file mode 100644 index 0000000000..b291e2fb96 --- /dev/null +++ b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/restoresession.yaml @@ -0,0 +1,34 @@ +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: source-neo4j-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Neo4j + namespace: demo + name: restored-neo4j + dataSource: + repository: s3-neo4j-composite-repo + snapshot: latest + addon: + name: neo4j-addon + tasks: + - name: logical-backup-restore + params: + seedServerName: "restored-neo4j-0" + remoteAliasCredentialsSecret: neo4j-remote-alias-credentials + jobTemplate: + spec: + volumes: + - name: data + persistentVolumeClaim: + claimName: data-restored-neo4j-0 + volumeMounts: + - mountPath: /data + name: data + subPath: data + securityContext: + runAsNonRoot: true + runAsUser: 7474 diff --git a/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/retentionpolicy.yaml b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/retentionpolicy.yaml new file mode 100644 index 0000000000..9e2758c41f --- /dev/null +++ b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/retentionpolicy.yaml @@ -0,0 +1,15 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All diff --git a/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/source-neo4j.yaml b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/source-neo4j.yaml new file mode 100644 index 0000000000..bf8574fa91 --- /dev/null +++ b/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/source-neo4j.yaml @@ -0,0 +1,25 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: source-neo4j + namespace: demo +spec: + version: 2025.12.1 + replicas: 3 + configuration: + remoteAliasKeystore: + keystoreRef: + name: neo4j-remote-alias-keystore + key: aes + passwordRef: + name: neo4j-remote-alias-keystore + key: password + keyName: neo + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + deletionPolicy: WipeOut diff --git a/docs/guides/neo4j/backup/kubestash/logical/composite-database/index.md b/docs/guides/neo4j/backup/kubestash/logical/composite-database/index.md new file mode 100644 index 0000000000..79d80a681c --- /dev/null +++ b/docs/guides/neo4j/backup/kubestash/logical/composite-database/index.md @@ -0,0 +1,587 @@ +--- +title: Backup and Restore Neo4j Composite Databases and Aliases +description: Backup and restore Neo4j composite databases and aliases using KubeStash +menu: + docs_{{ .version }}: + identifier: guides-neo4j-composite-database-logical-backup-stashv2 + name: Composite Databases and Aliases + parent: guides-neo4j-logical-backup-stashv2 + weight: 30 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +# Backup and Restore Neo4j Composite Databases and Aliases + +This guide shows you how to back up and restore physical databases, composite databases, composite aliases, and standalone aliases with KubeStash. It also explains the separate credential step required to restore stored-native remote aliases. + +## Before You Begin + +- Prepare a Kubernetes cluster and configure `kubectl` to access it. +- Install [KubeDB](/docs/setup/README.md), [KubeStash](https://kubestash.com/docs/latest/setup/install/kubestash), and the [KubeStash kubectl plugin](https://kubestash.com/docs/latest/setup/install/kubectl-plugin/). +- Use a KubeDB-supported Neo4j Enterprise version that supports composite databases. This guide uses `2025.12.1`. +- Prepare an S3-compatible bucket and its access credentials. +- Install a Java Development Kit that provides `keytool` on the machine where you run the setup commands. +- Make sure the source and restore-target Neo4j deployments can reach the remote Neo4j Bolt endpoint. +- Read the [Neo4j backup and restore overview](/docs/guides/neo4j/backup/kubestash/overview/index.md) if you are new to this process. + +Create the namespace used throughout this guide: + +```bash +$ kubectl create namespace demo +``` + +> **Note:** Replace every placeholder in the example manifests before applying them. + +## Architecture and What Gets Backed Up + +This walkthrough uses three distinct Neo4j deployments: + +- `source-neo4j` owns the `movies` physical database, the `media` composite database, and the alias catalog that KubeStash backs up. +- `remote-neo4j` owns the `reviews` physical database. `source-neo4j` reaches it through a remote alias. Its data is outside this backup's scope. +- `restored-neo4j` receives the physical database backup and reconstructed catalog. + +KubeStash discovers the catalog through Neo4j's `system` database. A backup contains physical database backup artifacts and metadata for composite database definitions, composite aliases, and standalone local and remote aliases. Depending on the alias, this metadata includes its location, target database, URL, username, credential type, driver settings, and properties. + +The password for a stored-native remote alias is intentionally **not** stored in backup metadata. You must supply it separately during restore. + +The database selector has the following behavior: + +- The default `*` selector backs up all visible physical databases and captures all composite and standalone alias definitions. +- Selecting a composite database explicitly automatically includes physical databases referenced by its local aliases. +- A remote alias definition is captured, but the data in its remote target database is not backed up. +- A local alias target must be selected and restored successfully before its composite database and alias can be restored. +- Database and alias exclusions are honored. Do not exclude a physical database required by a selected local alias. +- The `system` database is used for catalog discovery; it is not restored as a normal physical database. + +During restore, KubeStash performs these operations in dependency order: + +1. Restore the selected physical databases. +2. Wait for local alias target databases to become visible and online. +3. Create or replace composite databases. +4. Restore local and remote composite aliases. +5. Restore standalone aliases. +6. Verify the reconstructed composite and alias catalog. + +Restore preflight checks catalog conflicts and required remote credentials before destructive restore work begins. Without overwrite, a conflicting physical database, composite database, or alias causes the restore to fail. With overwrite enabled, KubeStash replaces catalog definitions where supported. It may temporarily detach aliases that depend on a physical database being replaced and recreate them after that database is restored. + +## Deploy the Source and Remote Neo4j Instances + +### Generate the Remote Alias Encryption Key + +Neo4j reversibly encrypts stored-native remote alias credentials in the `system` database. Before creating such an alias, generate a 256-bit AES key and store it in a password-protected PKCS12 keystore: + +```bash +$ KEYSTORE_PASSWORD=$(openssl rand -base64 32) +$ keytool -genseckey -keyalg AES -keysize 256 -storetype PKCS12 \ + -keystore neo4j-remote-alias-keystore.p12 -alias neo \ + -storepass "$KEYSTORE_PASSWORD" +``` + +Create a Kubernetes Secret containing the keystore and its password. The Secret must exist before you create either Neo4j resource that references it: + +```bash +$ kubectl create secret generic neo4j-remote-alias-keystore -n demo \ + --from-file=aes=neo4j-remote-alias-keystore.p12 \ + --from-literal=password="$KEYSTORE_PASSWORD" +``` + +The `source-neo4j` and `restored-neo4j` manifests reference the Secret through this configuration: + +```yaml +spec: + configuration: + remoteAliasKeystore: + keystoreRef: + name: neo4j-remote-alias-keystore + key: aes + passwordRef: + name: neo4j-remote-alias-keystore + key: password + keyName: neo +``` + +Here, `aes` is the Secret data key containing the PKCS12 file, `password` contains its password, and `keyName` must match the alias passed to `keytool` (`neo` in this example). Keep the keystore and password secure. Do not commit either one to source control. + +### Create the Neo4j Instances + +Create the source instance: + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: source-neo4j + namespace: demo +spec: + version: 2025.12.1 + replicas: 3 + configuration: + remoteAliasKeystore: + keystoreRef: + name: neo4j-remote-alias-keystore + key: aes + passwordRef: + name: neo4j-remote-alias-keystore + key: password + keyName: neo + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + deletionPolicy: WipeOut +``` + +Apply the manifest: + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/source-neo4j.yaml +``` + +Create the separate instance that will host the remote database: + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: remote-neo4j + namespace: demo +spec: + version: 2025.12.1 + replicas: 3 + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + deletionPolicy: WipeOut +``` + +Apply the manifest: + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/remote-neo4j.yaml +``` + +Both manifests request three Neo4j replicas and durable `2Gi` storage, and use `WipeOut` as the deletion policy. The source manifest also configures the remote alias keystore. Wait until both databases are ready: + +```bash +$ kubectl wait --for=jsonpath='{.status.phase}'=Ready neo4j/source-neo4j neo4j/remote-neo4j -n demo --timeout=10m +``` + +Store the generated administrator passwords in shell variables. These commands do not print the values: + +```bash +$ SOURCE_PASS=$(kubectl get secret source-neo4j-auth -n demo -o jsonpath='{.data.password}' | base64 -d) +$ REMOTE_PASS=$(kubectl get secret remote-neo4j-auth -n demo -o jsonpath='{.data.password}' | base64 -d) +``` + +## Create Sample Physical Databases + +Administrative commands run against the `system` database. Create `movies` on the source and `reviews` on the remote instance: + +```bash +$ kubectl exec -n demo source-neo4j-0 -- cypher-shell -d system -u neo4j -p "$SOURCE_PASS" \ + "CREATE DATABASE movies WAIT;" + +$ kubectl exec -n demo remote-neo4j-0 -- cypher-shell -d system -u neo4j -p "$REMOTE_PASS" \ + "CREATE DATABASE reviews WAIT;" +``` + +Add data to each physical database: + +```bash +$ kubectl exec -n demo source-neo4j-0 -- cypher-shell -d movies -u neo4j -p "$SOURCE_PASS" \ + "CREATE (:Movie {title: 'Example Movie'});" + +$ kubectl exec -n demo remote-neo4j-0 -- cypher-shell -d reviews -u neo4j -p "$REMOTE_PASS" \ + "CREATE (:Review {summary: 'Example Review'});" +``` + +## Create a Composite Database + +Create the `media` composite database on the source instance: + +```bash +$ kubectl exec -n demo source-neo4j-0 -- cypher-shell -d system -u neo4j -p "$SOURCE_PASS" \ + "CREATE COMPOSITE DATABASE media WAIT;" +``` + +## Create a Local Alias + +Create `media.movies` as a constituent local alias. Also create `movies-local` to demonstrate a standalone local alias: + +```bash +$ kubectl exec -n demo source-neo4j-0 -- cypher-shell -d system -u neo4j -p "$SOURCE_PASS" \ + "CREATE ALIAS media.movies FOR DATABASE movies; + CREATE ALIAS `movies-local` FOR DATABASE movies;" +``` + +The `media` prefix makes `media.movies` a constituent of the composite database. `movies-local` has no composite namespace and is therefore standalone. + +## Create a Stored-Native Remote Alias + +With `spec.configuration.remoteAliasKeystore` configured, create `media.reviews` and a standalone alias named `reviews-remote`. The service URL addresses the separate `remote-neo4j` deployment. This in-cluster example uses the `neo4j://` scheme, so its driver settings explicitly disable TLS enforcement. Use a secure `neo4j+s://` URL in production. + +```bash +$ kubectl exec -n demo source-neo4j-0 -- cypher-shell -d system -u neo4j -p "$SOURCE_PASS" \ + "CREATE ALIAS media.reviews FOR DATABASE reviews + AT 'neo4j://remote-neo4j.demo.svc:7687' + USER neo4j PASSWORD '$REMOTE_PASS' + DRIVER {ssl_enforced: false} + PROPERTIES {purpose: 'reviews'}; + CREATE ALIAS `reviews-remote` FOR DATABASE reviews + AT 'neo4j://remote-neo4j.demo.svc:7687' + USER neo4j PASSWORD '$REMOTE_PASS' + DRIVER {ssl_enforced: false};" +``` + +Do not place the real password in a manifest, documentation, shell history, or source control. The shell expands `REMOTE_PASS` only when this command runs. + +## Verify the Source Catalog + +Use the `system` database for catalog queries: + +```bash +$ kubectl exec -n demo source-neo4j-0 -- cypher-shell -d system -u neo4j -p "$SOURCE_PASS" \ + "SHOW DATABASES + YIELD name, type, currentStatus + RETURN name, type, currentStatus + ORDER BY name;" + +$ kubectl exec -n demo source-neo4j-0 -- cypher-shell -d system -u neo4j -p "$SOURCE_PASS" \ + "SHOW ALIASES FOR DATABASE + YIELD name, composite, database, location, url, user + RETURN name, composite, database, location, url, user + ORDER BY name;" +``` + +Confirm that `movies` is `standard` and online, `media` is `composite`, both `media.*` aliases belong to `media`, and the two standalone aliases have a null `composite` value. The local aliases must target `movies`; the remote aliases must show location `remote`, database `reviews`, the configured URL, and user `neo4j`. + +Verify that both composite constituents can be queried: + +```bash +$ kubectl exec -n demo source-neo4j-0 -- cypher-shell -d media -u neo4j -p "$SOURCE_PASS" \ + "USE media.movies MATCH (m:Movie) RETURN m.title;" + +$ kubectl exec -n demo source-neo4j-0 -- cypher-shell -d media -u neo4j -p "$SOURCE_PASS" \ + "USE media.reviews MATCH (r:Review) RETURN r.summary;" +``` + +## Configure BackupStorage and RetentionPolicy + +Create the S3 credential Secret as described in the [basic logical backup guide](/docs/guides/neo4j/backup/kubestash/logical/standalone-and-ha/#prepare-backend). Then replace the placeholders in the following `BackupStorage` manifest: + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: s3-storage + namespace: demo +spec: + storage: + provider: s3 + s3: + bucket: + region: + endpoint: + secretName: s3-secret + prefix: demo + usagePolicy: + allowedNamespaces: + from: All + default: false + deletionPolicy: Delete +``` + +Apply the manifest after replacing the placeholders: + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/backupstorage.yaml +``` + +The provider block selects S3 and supplies its bucket, region, endpoint, credential Secret, and object prefix. `usagePolicy` permits repositories in all namespaces to use this storage. `deletionPolicy: Delete` removes stored backup data when the `BackupStorage` is deleted. + +Create the retention policy: + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: demo-retention + namespace: demo +spec: + default: true + failedSnapshots: + last: 2 + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + usagePolicy: + allowedNamespaces: + from: All +``` + +Apply the manifest: + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/retentionpolicy.yaml +``` + +This policy retains the last five successful and two failed snapshots for at most two months. Its usage policy allows all namespaces. + +## Create BackupConfiguration + +Apply the backup configuration: + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: BackupConfiguration +metadata: + name: source-neo4j-backup + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Neo4j + namespace: demo + name: source-neo4j + backends: + - name: s3-backend + storageRef: + namespace: demo + name: s3-storage + retentionPolicy: + name: demo-retention + namespace: demo + sessions: + - name: frequent-backup + scheduler: + schedule: "*/5 * * * *" + jobTemplate: + backoffLimit: 1 + repositories: + - name: s3-neo4j-composite-repo + backend: s3-backend + directory: /neo4j-composite + addon: + name: neo4j-addon + tasks: + - name: logical-backup +``` + +Apply the manifest: + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/backupconfiguration.yaml +``` + +The target is `source-neo4j`. The backend refers to `s3-storage` and `demo-retention`. The session runs every five minutes and writes to repository `s3-neo4j-composite-repo` under `/neo4j-composite`. The `logical-backup` task uses its default `*` selector, so it backs up all visible physical databases and captures every composite and standalone alias definition. + +To select only `media`, set `databases: "media"` in the task parameters. KubeStash will also select `movies` because `media.movies` depends on it. Do not exclude `movies` from that backup. + +## Verify the Backup and Snapshot + +Wait for the configuration and its first backup to succeed: + +```bash +$ kubectl get backupconfiguration -n demo source-neo4j-backup +$ kubectl get backupsession -n demo -w +``` + +Then inspect the repository and snapshot: + +```bash +$ kubectl get repository -n demo s3-neo4j-composite-repo +$ kubectl get snapshot -n demo -l kubestash.com/repo-name=s3-neo4j-composite-repo +``` + +The `BackupConfiguration` and repository should become `Ready`, and the `BackupSession` and snapshot should reach `Succeeded`. Snapshot names and timings are generated, so this guide does not show fabricated output. Save the snapshot name if you prefer to restore a fixed snapshot instead of `latest`. + +## Deploy the Restore Target + +Create the empty restore target: + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: restored-neo4j + namespace: demo +spec: + version: 2025.12.1 + replicas: 3 + configuration: + remoteAliasKeystore: + keystoreRef: + name: neo4j-remote-alias-keystore + key: aes + passwordRef: + name: neo4j-remote-alias-keystore + key: password + keyName: neo + storageType: Durable + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + deletionPolicy: WipeOut +``` + +Apply the manifest: + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/restored-neo4j.yaml +``` + +The target uses the same Neo4j version, topology, and remote alias keystore configuration. The restore Job will seed `restored-neo4j-0` through its PVC. The keystore configuration lets the restored Neo4j instance encrypt credentials while KubeStash recreates stored-native remote aliases. + +## Create the Remote Alias Credential Secret + +Stored-native alias passwords are never included in backup metadata. Create this Secret in the same namespace as the `RestoreSession` before starting the restore: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: neo4j-remote-alias-credentials + namespace: demo +type: Opaque +stringData: + credentials.yaml: | + media.reviews: + password: "" + reviews-remote: + password: "" +``` + +Save this manifest as `remote-alias-credentials.yaml`, replace the placeholders locally, and apply it. Do not commit real values: + +```bash +$ kubectl apply -f remote-alias-credentials.yaml +``` + +The Secret must contain a `credentials.yaml` key. Its value is a map keyed by the exact, complete alias name returned by `SHOW ALIASES FOR DATABASE`. Every selected stored-native remote alias needs a non-empty `password`. The username, URL, driver settings, and properties come from backup metadata; only the password comes from this Secret. The remote endpoint must be reachable from `restored-neo4j`. OIDC credential-forwarding aliases do not use stored passwords and do not need entries. + +## Create RestoreSession + +Create the restore session after the credential Secret exists: + +```yaml +apiVersion: core.kubestash.com/v1alpha1 +kind: RestoreSession +metadata: + name: source-neo4j-restore + namespace: demo +spec: + target: + apiGroup: kubedb.com + kind: Neo4j + namespace: demo + name: restored-neo4j + dataSource: + repository: s3-neo4j-composite-repo + snapshot: latest + addon: + name: neo4j-addon + tasks: + - name: logical-backup-restore + params: + seedServerName: "restored-neo4j-0" + remoteAliasCredentialsSecret: neo4j-remote-alias-credentials + jobTemplate: + spec: + volumes: + - name: data + persistentVolumeClaim: + claimName: data-restored-neo4j-0 + volumeMounts: + - mountPath: /data + name: data + subPath: data + securityContext: + runAsNonRoot: true + runAsUser: 7474 +``` + +Apply the manifest: + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/neo4j/backup/kubestash/logical/composite-database/examples/restoresession.yaml +``` + +The target is `restored-neo4j`, and the data source selects the latest snapshot from `s3-neo4j-composite-repo`. The `logical-backup-restore` task uses `restored-neo4j-0` as the seed server. `remoteAliasCredentialsSecret` identifies the Secret that supplies stored-native remote alias passwords. The restore Job mounts `data-restored-neo4j-0` at `/data` with subpath `data` and runs as the Neo4j user `7474`, ensuring correct ownership of restored files. + +The addon's default restore arguments enable overwrite. If overwrite is disabled, any conflicting physical database, composite database, or alias fails preflight. Review the target catalog before restoring into a non-empty deployment. + +Watch the restore: + +```bash +$ kubectl get restoresession -n demo source-neo4j-restore -w +``` + +Continue only after its phase is `Succeeded` and `restored-neo4j` is `Ready`. + +## Verify the Restored Physical Databases, Composite Database, and Aliases + +Load the restore-target password without printing it: + +```bash +$ RESTORED_PASS=$(kubectl get secret restored-neo4j-auth -n demo -o jsonpath='{.data.password}' | base64 -d) +``` + +Verify database type and status: + +```bash +$ kubectl exec -n demo restored-neo4j-0 -- cypher-shell -d system -u neo4j -p "$RESTORED_PASS" \ + "SHOW DATABASES + YIELD name, type, currentStatus + RETURN name, type, currentStatus + ORDER BY name;" +``` + +Confirm that `movies` is online and that `media` has type `composite`. Next, compare alias metadata with the source catalog: + +```bash +$ kubectl exec -n demo restored-neo4j-0 -- cypher-shell -d system -u neo4j -p "$RESTORED_PASS" \ + "SHOW ALIASES FOR DATABASE + YIELD name, composite, database, location, url, user + RETURN name, composite, database, location, url, user + ORDER BY name;" +``` + +Confirm the expected composite membership, target, location, URL, and username for all four aliases. Finally, query both composite constituents: + +```bash +$ kubectl exec -n demo restored-neo4j-0 -- cypher-shell -d media -u neo4j -p "$RESTORED_PASS" \ + "USE media.movies MATCH (m:Movie) RETURN m.title;" + +$ kubectl exec -n demo restored-neo4j-0 -- cypher-shell -d media -u neo4j -p "$RESTORED_PASS" \ + "USE media.reviews MATCH (r:Review) RETURN r.summary;" + +$ kubectl get restoresession -n demo source-neo4j-restore +``` + +The local query reads restored `movies` data. The remote query reads the live `reviews` database on `remote-neo4j`; it does not prove that remote data was part of the backup. The final command must report the `RestoreSession` phase as `Succeeded`. + +## Cleanup + +Delete the tutorial resources when you no longer need them: + +```bash +$ kubectl delete restoresession -n demo source-neo4j-restore +$ kubectl delete backupconfiguration -n demo source-neo4j-backup +$ kubectl delete backupstorage -n demo s3-storage +$ kubectl delete retentionpolicy -n demo demo-retention +$ kubectl delete secret -n demo neo4j-remote-alias-credentials neo4j-remote-alias-keystore s3-secret +$ kubectl delete neo4j -n demo restored-neo4j source-neo4j remote-neo4j +``` + +Because these Neo4j resources use `deletionPolicy: WipeOut`, deleting them also removes their database storage. The `BackupStorage` uses `deletionPolicy: Delete`, so deleting it removes its stored backup data. diff --git a/docs/guides/neo4j/backup/kubestash/logical/index.md b/docs/guides/neo4j/backup/kubestash/logical/standalone-and-ha/index.md similarity index 98% rename from docs/guides/neo4j/backup/kubestash/logical/index.md rename to docs/guides/neo4j/backup/kubestash/logical/standalone-and-ha/index.md index a37d3d3c14..6344b6b06c 100644 --- a/docs/guides/neo4j/backup/kubestash/logical/index.md +++ b/docs/guides/neo4j/backup/kubestash/logical/standalone-and-ha/index.md @@ -3,9 +3,9 @@ title: Backup & Restore Neo4j | KubeStash description: Backup and Restore Neo4j database using KubeStash menu: docs_{{ .version }}: - identifier: guides-neo4j-logical-backup-stashv2 - name: Logical Backup - parent: guides-neo4j-backup-stashv2 + identifier: guides-neo4j-standalone-ha-logical-backup-stashv2 + name: Standalone and HA Cluster + parent: guides-neo4j-logical-backup-stashv2 weight: 20 menu_name: docs_{{ .version }} section_menu_id: guides @@ -17,6 +17,8 @@ KubeStash allows you to backup and restore `Neo4j` databases. It supports backup This guide will give you an overview how you can take backup and restore your `Neo4j` databases using `KubeStash`. +For composite databases and local, standalone, or remote aliases, see [Backup and Restore Neo4j Composite Databases and Aliases](/docs/guides/neo4j/backup/kubestash/logical/composite-database/). + ## Before You Begin - At first, you need to have a Kubernetes cluster, and the `kubectl` command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using `Minikube` or `Kind`. diff --git a/docs/guides/neo4j/concepts/autoscaler.md b/docs/guides/neo4j/concepts/autoscaler.md new file mode 100644 index 0000000000..7fe03560f4 --- /dev/null +++ b/docs/guides/neo4j/concepts/autoscaler.md @@ -0,0 +1,143 @@ +--- +title: Neo4jAutoscaler CRD +menu: + docs_{{ .version }}: + identifier: neo4j-autoscaler-concepts + name: Neo4jAutoscaler + parent: neo4j-concepts + weight: 30 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Neo4jAutoscaler + +## What is Neo4jAutoscaler? + +`Neo4jAutoscaler` is a Kubernetes custom resource that declares how KubeDB should automatically scale the compute resources and persistent storage of a Neo4j cluster. The Autoscaler operator translates its recommendations into [Neo4jOpsRequest](/docs/guides/neo4j/concepts/opsrequest.md) resources, which are executed by Ops Manager. + +The following example enables both compute and storage autoscaling: + +```yaml +apiVersion: autoscaling.kubedb.com/v1alpha1 +kind: Neo4jAutoscaler +metadata: + name: neo4j-autoscaler + namespace: demo +spec: + databaseRef: + name: neo4j-autoscale + opsRequestOptions: + apply: IfReady + timeout: 10m + maxRetries: 3 + compute: + neo4j: + trigger: "On" + podLifeTimeThreshold: 5m + resourceDiffPercentage: 20 + minAllowed: + cpu: 600m + memory: 1200Mi + maxAllowed: + cpu: "2" + memory: 2Gi + controlledResources: + - cpu + - memory + containerControlledValues: RequestsAndLimits + storage: + neo4j: + trigger: "On" + usageThreshold: 80 + scalingThreshold: 50 + expansionMode: Online +``` + +## Specification + +Like other Kubernetes resources, `Neo4jAutoscaler` contains `apiVersion`, `kind`, `metadata`, `spec`, and `status`. Users declare the policy in `spec`; the operator reports observed state in `status`. + +### `spec.databaseRef` + +`spec.databaseRef` is required and identifies the `Neo4j` resource in the same namespace. + +| Field | Description | +|---|---| +| `name` | Name of the target KubeDB `Neo4j` resource. | + +### `spec.opsRequestOptions` + +These options are copied to every `Neo4jOpsRequest` created by the Autoscaler. + +| Field | Description | +|---|---| +| `apply` | `IfReady` creates operations only while the database is ready; `Always` permits creation regardless of readiness. Defaults to `IfReady`. | +| `timeout` | Maximum duration allowed for each operation step. | +| `maxRetries` | Maximum retries for a failed operation. Defaults to `1`. | + +### `spec.compute` + +`spec.compute.neo4j` controls CPU and memory recommendations for the `neo4j` container. + +| Field | Description | +|---|---| +| `trigger` | Enables autoscaling when set to `On`; use `Off` to disable it without deleting the resource. | +| `minAllowed` | Lower CPU and memory bounds for recommendations. | +| `maxAllowed` | Upper CPU and memory bounds for recommendations. | +| `controlledResources` | Resources controlled by the Autoscaler, normally `cpu` and `memory`. | +| `containerControlledValues` | `RequestsAndLimits` updates both values; `RequestsOnly` updates only requests. | +| `resourceDiffPercentage` | Minimum percentage difference between the current allocation and a recommendation before an update is applied. Defaults to `50`. | +| `podLifeTimeThreshold` | Minimum pod lifetime considered when deciding whether to apply a recommendation. Defaults to `15m`. | + +`spec.compute.nodeTopology` is optional. When set, the Autoscaler selects resources from the named `NodeTopology` instead of applying an arbitrary recommendation. `scaleUpDiffPercentage` and `scaleDownDiffPercentage` control when it moves between topology entries; their defaults are `15` and `25`, respectively. + +### `spec.storage` + +`spec.storage.neo4j` controls expansion of the Neo4j data volumes. + +| Field | Description | +|---|---| +| `trigger` | Enables storage autoscaling when set to `On`. | +| `usageThreshold` | Used-capacity percentage at which expansion is triggered. Defaults to `80`. | +| `scalingThreshold` | Percentage by which the current volume is increased. Defaults to `50`. | +| `scalingRules` | Optional size-dependent rules. Each rule has an `appliesUpto` capacity and a `threshold` percentage or absolute quantity. | +| `upperBound` | Optional maximum volume size. | +| `expansionMode` | Required expansion strategy: `Online` or `Offline`. | + +For example, the following rules grow smaller volumes proportionally and larger volumes by a fixed amount: + +```yaml +storage: + neo4j: + trigger: "On" + usageThreshold: 80 + expansionMode: Online + upperBound: 2Ti + scalingRules: + - appliesUpto: 500Gi + threshold: 30pc + - appliesUpto: 1Ti + threshold: 20pc + - appliesUpto: "" + threshold: 100Gi +``` + +### `status` + +The status is managed by KubeDB and should not be edited. Important fields include: + +| Field | Description | +|---|---| +| `phase` | Current Autoscaler phase, such as `InProgress`, `Current`, or `Failed`. | +| `observedGeneration` | Most recent resource generation processed by the operator. | +| `conditions` | Events and outcomes reported by the Autoscaler controller. | +| `vpas` | Current compute recommendations and their conditions. | +| `checkpoints` | Historical CPU and memory samples used by the recommender. | + +## Next Steps + +- [Autoscale Neo4j compute resources](/docs/guides/neo4j/autoscaler/compute/autoscale.md) +- [Autoscale Neo4j storage](/docs/guides/neo4j/autoscaler/storage/autoscale.md) diff --git a/docs/guides/neo4j/pitr/_index.md b/docs/guides/neo4j/pitr/_index.md new file mode 100644 index 0000000000..5dcf6dca8c --- /dev/null +++ b/docs/guides/neo4j/pitr/_index.md @@ -0,0 +1,10 @@ +--- +title: Continuous Archiving and Point-in-time Recovery +menu: + docs_{{ .version }}: + identifier: pitr-neo4j + name: Point-in-time Recovery + parent: neo4j-guides + weight: 70 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/neo4j/pitr/archiver.md b/docs/guides/neo4j/pitr/archiver.md new file mode 100644 index 0000000000..d470b244f2 --- /dev/null +++ b/docs/guides/neo4j/pitr/archiver.md @@ -0,0 +1,559 @@ +--- +title: Continuous Archiving and Point-in-time Recovery for Neo4j +description: Archive Neo4j full and differential backups with KubeDB and recover a graph to a timestamp before an accidental change. +menu: + docs_{{ .version }}: + identifier: pitr-neo4j-archiver + name: Overview + parent: pitr-neo4j + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Start with the [Neo4j quickstart](/docs/guides/neo4j/quickstart/quickstart.md). + +# Continuous Archiving and Point-in-time Recovery for Neo4j + +An accidental write can leave a database healthy while its data is wrong. Point-in-time recovery (PITR) lets you recover the graph as it existed before that write, using a full backup and the transaction logs in subsequent differential backups. + +In this tutorial, you will deploy a three-member Neo4j source, enable continuous archiving with `Neo4jArchiver`, and restore into a separate standalone Neo4j instance. The recovery exercise changes a person's properties, deletes another person and their relationship, and adds a new person. You will then verify that the restored graph contains only the state committed before your chosen timestamp. + +The examples use namespace `demo`. Kubernetes resources have the prefix `neo4j-pitr-`, and backup data uses an isolated object-storage prefix, `neo4j-pitr-demo`. + +## Before You Begin + +You need: + +- A Kubernetes cluster and `kubectl` configured to use it. +- [KubeDB](/docs/setup/README.md), KubeStash, and the Sidekick controller, with support for `Neo4jArchiver` and `Neo4j.spec.init.archiver`. +- A Neo4j Enterprise version and matching backup addon that support full and differential backups. This example uses `2026.06.0`. +- An S3-compatible bucket reachable from the database and backup pods. This example uses an existing MinIO service, `minio.demo.svc.cluster.local:80`, and bucket `kubestash`. +- A provisioner for persistent volumes. The example uses `local-path` and requests `2Gi` per database pod; choose a storage class appropriate for your environment. +- Bash, `jq`, and OpenSSL for the commands below. + +Check the installed API and addon before continuing: + +```bash +kubectl explain neo4jarchiver.spec +kubectl explain neo4j.spec.init.archiver +kubectl get neo4jversions.catalog.kubedb.com 2026.06.0 +kubectl get addons.addons.kubestash.com neo4j-addon +kubectl get deployments -n kubedb +kubectl get deployments -n kubestash +``` + +Create the namespace if it does not already exist: + +```bash +kubectl get namespace demo >/dev/null 2>&1 || kubectl create namespace demo +``` + +The manifests are in [docs/guides/neo4j/pitr/yamls](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/guides/neo4j/pitr/yamls). Download that directory and run the commands from it. Review the bucket, endpoint, storage class, and credentials before applying the files. Apply each manifest at the step shown; the restore manifest is used only after choosing a recoverable timestamp. + +## How the Archive Chain Works + +`Neo4jArchiver` describes the backup policy for selected KubeDB `Neo4j` resources. The database label and explicit archiver reference in this example connect `neo4j-pitr-source` to `neo4j-pitr-archiver`. + +| Component | Responsibility in this example | +| --- | --- | +| KubeDB | Creates the backup configuration and Sidekick for the selected source; coordinates initialization of the restore target. | +| KubeStash full-backup session | Runs `neo4j-admin` through the `Neo4jAdmin` driver and backs up database manifests. | +| Sidekick | Runs differential backups every five minutes, adding transaction-log artifacts to the full backup's archive chain. | +| Manifest-backup session | Backs up Kubernetes manifests and referenced resources on a separate schedule. | +| Full and manifest repositories | Record the locations and snapshots used during recovery. | +| Restore sessions | Restore requested manifests and recover database files into the target's seed pod volume. | + +Neo4j stores its native `.backup` artifacts in S3-compatible storage. The data-backup driver is `Neo4jAdmin`. + +## Prepare Backup Storage + +### Storage Credentials + +Edit `storage-secret.yaml` with your bucket credentials: + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret + namespace: demo +type: Opaque +stringData: + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: "" +``` + +```bash +kubectl apply -f storage-secret.yaml +``` + +If `minio-secret` already contains the correct credentials, reuse it and skip this apply. Do not replace an existing Secret with placeholder values or commit populated credentials. + +### BackupStorage + +The storage object uses a dedicated prefix, so this tutorial's archives are separate from other backups in the bucket. + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: neo4j-pitr-storage + namespace: demo +spec: + storage: + provider: s3 + s3: + bucket: kubestash + endpoint: http://minio.demo.svc.cluster.local:80 + region: us-east-1 + prefix: neo4j-pitr-demo + secretName: minio-secret + usagePolicy: + allowedNamespaces: + from: Same + default: false + deletionPolicy: Delete +``` + +```bash +kubectl apply -f backupstorage.yaml +kubectl get backupstorage neo4j-pitr-storage -n demo +``` + +Wait for `PHASE` to become `Ready`. The example's HTTP endpoint is specific to its internal MinIO service; use the appropriate endpoint and transport security for your storage backend. + +### Retention and Encryption Secret + +`retention-policy.yaml` keeps up to five successful snapshots and two failed snapshots, subject to a maximum retention period of two months: + +```yaml +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: neo4j-pitr-retention + namespace: demo +spec: + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + failedSnapshots: + last: 2 + usagePolicy: + allowedNamespaces: + from: Same +``` + +```bash +kubectl apply -f retention-policy.yaml +kubectl create secret generic neo4j-pitr-encryption -n demo \ + --from-literal=RESTIC_PASSWORD="$(openssl rand -hex 24)" +``` + +The generated password is not printed. Store it securely for recovery. Alternatively, replace the placeholder in `encryption-secret.yaml` and apply that file instead of running `create secret`. + +The snapshot count is also a retention constraint: `2mo` does not mean that every point within two months is guaranteed recoverable. A recovery point needs its full backup and the continuous differential chain that covers it. The archiver's successful/failed log history limits control monitoring history, not the desired recovery window. + +## Enable Archiving + +### Create the Neo4jArchiver + +```yaml +apiVersion: archiver.kubedb.com/v1alpha1 +kind: Neo4jArchiver +metadata: + name: neo4j-pitr-archiver + namespace: demo +spec: + pause: false + databases: + namespaces: + from: Same + selector: + matchLabels: + archiver: neo4j-pitr + backupStorage: + ref: + name: neo4j-pitr-storage + namespace: demo + subDir: /neo4j-backup + retentionPolicy: + name: neo4j-pitr-retention + namespace: demo + encryptionSecret: + name: neo4j-pitr-encryption + namespace: demo + fullBackup: + driver: Neo4jAdmin + scheduler: + schedule: "0 1 * * *" + sessionHistoryLimit: 3 + timeout: 1h + differentialBackup: + backupInterval: 5m + successfulLogHistoryLimit: 5 + failedLogHistoryLimit: 5 + manifestBackup: + scheduler: + schedule: "*/30 * * * *" + sessionHistoryLimit: 3 + timeout: 15m + deletionPolicy: Delete +``` + +```bash +kubectl apply -f neo4jarchiver.yaml +``` + +The full-backup schedule runs daily at `01:00` in the scheduler's timezone; the manifest schedule runs every thirty minutes. KubeStash also triggers an initial backup when the generated backup configuration becomes ready. The test cluster uses UTC for its schedules. The differential interval is independent of those CronJob schedules. + +### Deploy the Source + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: neo4j-pitr-source + namespace: demo + labels: + archiver: neo4j-pitr +spec: + version: "2026.06.0" + replicas: 3 + archiver: + ref: + name: neo4j-pitr-archiver + namespace: demo + storageType: Durable + storage: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + deletionPolicy: WipeOut +``` + +```bash +kubectl apply -f neo4j.yaml +kubectl wait neo4j/neo4j-pitr-source -n demo \ + --for=condition=Ready --timeout=10m +kubectl get pods -n demo -l app.kubernetes.io/instance=neo4j-pitr-source +``` + +The live run produced three ready source pods: + +```text +NAME READY STATUS RESTARTS AGE +neo4j-pitr-source-0 1/1 Running 0 47s +neo4j-pitr-source-1 1/1 Running 0 42s +neo4j-pitr-source-2 1/1 Running 0 37s +``` + +> **Deletion policy:** `WipeOut` makes deleting these disposable Neo4j resources destructive to their database storage. Do not use the cleanup commands against a database you intend to keep. + +### Verify the First Full Backup + +KubeDB creates `neo4j-pitr-source-archiver`; you do not create a separate `BackupConfiguration` for this workflow. + +```bash +kubectl get backupconfiguration neo4j-pitr-source-archiver -n demo +kubectl get backupsessions -n demo +kubectl get repositories neo4j-pitr-source-full neo4j-pitr-source-manifest -n demo +kubectl get snapshots.storage.kubestash.com -n demo \ + -l kubestash.com/app-ref-name=neo4j-pitr-source +``` + +Wait for the configuration and repositories to be `Ready` and the initial full and manifest backup sessions to be `Succeeded`. Use the fully qualified Snapshot resource name: a cluster may also have Longhorn resources named `snapshots`. + +This is the initial full-backup artifact recorded in the test: + +```text +BackupSession: neo4j-pitr-source-archiver-full-backup-1789046588 +Phase: Succeeded +Database: neo4j +Artifact: neo4j-2026-09-10T13-23-42.backup +Artifact time: 2026-09-10T13:23:42 UTC +``` + +The graph used below is created after this full backup. Recovering it therefore requires the differential chain, not just restoring the full backup. + +## Recover a Graph to a Middle Timestamp + +The sequence is: + +```text +Full backup → Create Alice and Bob → Differential backup + ↓ + Record recovery cutoff + ↓ + Change Alice, delete Bob, add Charlie + ↓ + Later differential backup + ↓ + Restore only transactions before cutoff +``` + +### Connect Without Printing Credentials + +Define a helper in your Bash session. It reads the credentials from the Secret already mounted in the database pod and passes them to `cypher-shell` through environment variables: + +```bash +source_cypher() { + kubectl exec -n demo neo4j-pitr-source-0 -- sh -c ' + export NEO4J_USERNAME="$(cat /config/neo4j-auth/username)" + export NEO4J_PASSWORD="$(cat /config/neo4j-auth/password)" + exec cypher-shell -a neo4j://neo4j-pitr-source.demo.svc:7687 \ + -d neo4j --format plain "$1" + ' sh "$1" +} +``` + +The `neo4j://` address allows the client to route writes to the database leader. The source pod must be running and the mounted credentials must be available. + +### Create the Baseline Graph + +Run this once against the fresh source: + +```bash +source_cypher "CREATE (a:PITRPerson {name: 'Alice', age: 30}), + (b:PITRPerson {name: 'Bob', age: 25}), + (a)-[:KNOWS]->(b);" + +source_cypher "MATCH (p:PITRPerson) + RETURN p.name AS name, p.age AS age ORDER BY name;" +source_cypher "MATCH (:PITRPerson)-[r:KNOWS]->(:PITRPerson) + RETURN count(r) AS relationships;" +``` + +```text +name, age +"Alice", 30 +"Bob", 25 +relationships +1 +``` + +Wait for a successful differential cycle after these writes. Inspect the Sidekick logs and monitoring history: + +```bash +kubectl logs -n demo neo4j-pitr-source-sidekick --tail=60 +kubectl get snapshots.storage.kubestash.com \ + neo4j-pitr-source-differential-snapshot -n demo -o json | + jq '.status.components.log.logStats' +``` + +A completed cycle should report `Differential backup cycle completed`, and `lastSucceededStats` should contain a completion time later than the baseline writes. A running pod alone does not prove that the writes have been archived. + +### Record the Cutoff, Then Change the Data + +After the baseline differential backup succeeds, capture a whole-second UTC timestamp from Neo4j: + +```bash +RECOVERY_TIMESTAMP=$(source_cypher \ + "RETURN toString(datetime.truncate('second', datetime({timezone: '+00:00'}))) AS recoveryTimestamp;" | + tail -n 1 | tr -d '"\r') +printf '%s\n' "$RECOVERY_TIMESTAMP" +``` + +The live run returned: + +```text +2026-09-10T13:32:30Z +``` + +Keep this value for the restore manifest. Allow at least two seconds before the next writes so that the cutoff is clearly between the transactions: + +```bash +sleep 2 +source_cypher "MATCH (a:PITRPerson {name: 'Alice'}) SET a.age = 99; + MATCH (b:PITRPerson {name: 'Bob'}) DETACH DELETE b; + CREATE (:PITRPerson {name: 'Charlie', age: 40});" + +source_cypher "MATCH (p:PITRPerson) + RETURN p.name AS name, p.age AS age ORDER BY name;" +source_cypher "MATCH (:PITRPerson)-[r:KNOWS]->(:PITRPerson) + RETURN count(r) AS relationships;" +``` + +The live source then returned: + +```text +name, age +"Alice", 99 +"Charlie", 40 +relationships +0 +``` + +The changes had committed by `2026-09-10T13:32:54.085Z`, after the cutoff. These mutations are deliberately limited to the tutorial's `PITRPerson` nodes in its disposable source database. + +### Wait for a Backup After the Changes + +Wait for another successful differential cycle. Check the history and logs again, making sure the cycle started after the destructive writes and completed successfully. + +For a recovery target between backups, the archive chain must include transactions beyond that target. Neo4j's restore command replays differential transaction logs and stops before the specified UTC time. See [Neo4j's timestamp recovery semantics](https://neo4j.com/docs/operations-manual/current/backup-restore/restore-backup/#restore-data-up-to-a-specific-date). + +Do not use a future timestamp to demonstrate PITR. In the tested plugin, a requested time newer than every available artifact emits a warning and restores the latest available state. That can succeed without proving historical recovery. + +## Restore into a New Neo4j Instance + +The restore uses `spec.init.archiver` with the source's full and manifest repositories and the same encryption Secret. KubeDB creates the required `RestoreSession` resources, selects backup data, and supplies the target seed pod and PVC to the data-restore job. You do not need to hand-create those sessions in a matching release. + +Use the same Neo4j version for this recovery exercise. The target intentionally has one replica so the example validates recovery separately from subsequent cluster expansion. It has no source archiver selector label. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: neo4j-pitr-restored + namespace: demo +spec: + version: "2026.06.0" + replicas: 1 + init: + archiver: + fullDBRepository: + name: neo4j-pitr-source-full + namespace: demo + manifestRepository: + name: neo4j-pitr-source-manifest + namespace: demo + encryptionSecret: + name: neo4j-pitr-encryption + namespace: demo + # Replace with the UTC cutoff recorded in your own run. + recoveryTimestamp: "2026-09-10T13:32:30Z" + storageType: Durable + storage: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + deletionPolicy: WipeOut +``` + +Edit `restored-neo4j.yaml` and set `spec.init.archiver.recoveryTimestamp` to the value of `$RECOVERY_TIMESTAMP` from your run. The checked-in timestamp records this tutorial's test; it is not valid for a newly created archive chain. + +```bash +kubectl apply -f restored-neo4j.yaml +kubectl get restoresessions -n demo +``` + +Wait for every generated restore session to report `Succeeded`, then verify the database-specific restore condition. KubeDB selects a successful full-backup Snapshot with a `dump` component; the restore addon follows its archive metadata to the differential artifact whose chain covers the cutoff. + +```bash +kubectl wait neo4j/neo4j-pitr-restored -n demo \ + --for=condition=SuccessfullyDataRestored --timeout=10m +kubectl wait neo4j/neo4j-pitr-restored -n demo \ + --for=condition=Ready --timeout=10m +kubectl get neo4j neo4j-pitr-source neo4j-pitr-restored -n demo +``` + +`Ready` by itself is insufficient: a target can accept connections during provisioning before data recovery finishes. Check the successful restore sessions and `SuccessfullyDataRestored` before validating the graph. + +The live resources reached these final states: + +```text +NAME VERSION STATUS AGE +neo4j-pitr-source 2026.06.0 Ready 15h +neo4j-pitr-restored 2026.06.0 Ready 14h + +NAME REPOSITORY PHASE DURATION +neo4j-pitr-restored-data-backup-restorer neo4j-pitr-source-full Succeeded 25s +``` + +The verified restore used this chain: + +```text +Full artifact neo4j-2026-09-10T13-23-42.backup transactions 1-3 +First differential neo4j-2026-09-10T13-30-51.backup transactions 4-8 +Second differential neo4j-2026-09-10T13-35-50.backup transactions 9-11 +Recovery cutoff 2026-09-10T13:32:30Z +Recovered checkpoint transaction 8 +RestoreSession neo4j-pitr-restored-data-backup-restorer: Succeeded (25s) +``` + +The restore command retained `--restore-until=2026-09-10 13:32:30`, merged all three artifacts, and checkpointed at transaction 8. Transactions 9-11 contained the later changes and were not applied. + +### Verify the Restored Graph + +Define the corresponding helper for the target: + +```bash +restored_cypher() { + kubectl exec -n demo neo4j-pitr-restored-0 -- sh -c ' + export NEO4J_USERNAME="$(cat /config/neo4j-auth/username)" + export NEO4J_PASSWORD="$(cat /config/neo4j-auth/password)" + exec cypher-shell -a neo4j://neo4j-pitr-restored.demo.svc:7687 \ + -d neo4j --format plain "$1" + ' sh "$1" +} + +restored_cypher "MATCH (p:PITRPerson) + RETURN p.name AS name, p.age AS age ORDER BY name;" +restored_cypher "MATCH (a:PITRPerson)-[:KNOWS]->(b:PITRPerson) + RETURN a.name AS from, b.name AS to;" +restored_cypher "MATCH (p:PITRPerson {name: 'Charlie'}) + RETURN count(p) AS charlieCount;" +``` + +The live restored database returned: + +```text +name, age +"Alice", 30 +"Bob", 25 +source, target +"Alice", "Bob" +charlieCount +0 +``` + +This proves that recovery returned the graph to the middle timestamp: Alice has her original age, Bob and the `KNOWS` relationship exist, and Charlie does not. + +## Troubleshooting + +| Symptom | What to check | +| --- | --- | +| No backup configuration appears | Check the source's selector label, `spec.archiver.ref`, namespace selection, installed CRDs, and controller logs. | +| BackupStorage is not `Ready` | Check that the bucket exists, the credential Secret is in the expected namespace, and the endpoint is reachable from backup pods. | +| Sidekick restarts with a missing required argument | Verify that the KubeDB controller and Neo4j backup-plugin images come from a compatible release. | +| Sidekick runs but archives are stale | Inspect its logs and the latest successful differential history; do not infer archive health from pod readiness. | +| Snapshot list is empty | Use `snapshots.storage.kubestash.com` explicitly to avoid querying another API group's Snapshot resource. | +| RestoreSession is `Invalid` with `Component dump not exist` | Verify that the controller selected a successful full-backup Snapshot containing `status.components.dump`, and install matching controller and addon builds. | +| Restore reports no continuous chain covering the target | Check that a full backup predates the cutoff and differential artifacts cover it; verify that retention or manual object deletion has not removed required artifacts. | +| Restore warns that the target is newer than the latest backup | Wait for an archive that covers the desired time and restore into a new target; a latest-state fallback is not evidence of PITR. | +| Database is ready but expected data is absent | Check both restore sessions and `SuccessfullyDataRestored`, then query the correct database and target service. | + +Inspect the restore job's output when a recovery fails or its result is unexpected: + +```bash +kubectl describe restoresession neo4j-pitr-restored-data-backup-restorer -n demo +kubectl logs -n demo job/neo4j-pitr-restored-data-backup-restorer +kubectl get events -n demo --sort-by=.lastTimestamp +``` + +Use `status.components.log.logStats` and the archive/restore logs as evidence of current differential-backup health; do not confuse the monitoring object with a full-backup Snapshot containing the `dump` component. + +## Cleanup + +Keep the source, restored database, repositories, and encryption Secret until you have finished verifying recovery. The live tutorial resources were left in place for inspection. + +When you explicitly want to remove the disposable databases: + +```bash +kubectl delete neo4j neo4j-pitr-source neo4j-pitr-restored -n demo +kubectl delete neo4jarchiver neo4j-pitr-archiver -n demo +``` + +The database manifests use `WipeOut`, so this removes their database storage. Inspect the remaining repositories and snapshots before deleting backup-related resources. The example uses `Delete` rather than `WipeOut` for archive storage; do not assume that deleting Kubernetes objects purges the archived objects from the bucket. Preserve the encryption Secret while any retained repository still needs it. Do not delete the shared MinIO credential Secret or the entire `demo` namespace. + +## Next Steps + +- [Backup and restore standalone and HA Neo4j](/docs/guides/neo4j/backup/kubestash/logical/standalone-and-ha/) for the explicit `BackupConfiguration`/`RestoreSession` workflow. +- [Customize Neo4j backup and restore](/docs/guides/neo4j/backup/kubestash/customization/index.md) for database selection, resource settings, and restore parameters. +- [Back up composite databases and aliases](/docs/guides/neo4j/backup/kubestash/logical/composite-database/) for catalog and alias-specific considerations beyond this single-database recovery exercise. diff --git a/docs/guides/neo4j/pitr/yamls/backupstorage.yaml b/docs/guides/neo4j/pitr/yamls/backupstorage.yaml new file mode 100644 index 0000000000..44ed86d588 --- /dev/null +++ b/docs/guides/neo4j/pitr/yamls/backupstorage.yaml @@ -0,0 +1,19 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: BackupStorage +metadata: + name: neo4j-pitr-storage + namespace: demo +spec: + storage: + provider: s3 + s3: + bucket: kubestash + endpoint: http://minio.demo.svc.cluster.local:80 + region: us-east-1 + prefix: neo4j-pitr-demo + secretName: minio-secret + usagePolicy: + allowedNamespaces: + from: Same + default: false + deletionPolicy: Delete diff --git a/docs/guides/neo4j/pitr/yamls/encryption-secret.yaml b/docs/guides/neo4j/pitr/yamls/encryption-secret.yaml new file mode 100644 index 0000000000..226c02d80f --- /dev/null +++ b/docs/guides/neo4j/pitr/yamls/encryption-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: neo4j-pitr-encryption + namespace: demo +type: Opaque +stringData: + RESTIC_PASSWORD: "" diff --git a/docs/guides/neo4j/pitr/yamls/neo4j.yaml b/docs/guides/neo4j/pitr/yamls/neo4j.yaml new file mode 100644 index 0000000000..4c54093330 --- /dev/null +++ b/docs/guides/neo4j/pitr/yamls/neo4j.yaml @@ -0,0 +1,23 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: neo4j-pitr-source + namespace: demo + labels: + archiver: neo4j-pitr +spec: + version: "2026.06.0" + replicas: 3 + archiver: + ref: + name: neo4j-pitr-archiver + namespace: demo + storageType: Durable + storage: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + deletionPolicy: WipeOut diff --git a/docs/guides/neo4j/pitr/yamls/neo4jarchiver.yaml b/docs/guides/neo4j/pitr/yamls/neo4jarchiver.yaml new file mode 100644 index 0000000000..4bb2323d77 --- /dev/null +++ b/docs/guides/neo4j/pitr/yamls/neo4jarchiver.yaml @@ -0,0 +1,40 @@ +apiVersion: archiver.kubedb.com/v1alpha1 +kind: Neo4jArchiver +metadata: + name: neo4j-pitr-archiver + namespace: demo +spec: + pause: false + databases: + namespaces: + from: Same + selector: + matchLabels: + archiver: neo4j-pitr + backupStorage: + ref: + name: neo4j-pitr-storage + namespace: demo + subDir: /neo4j-backup + retentionPolicy: + name: neo4j-pitr-retention + namespace: demo + encryptionSecret: + name: neo4j-pitr-encryption + namespace: demo + fullBackup: + driver: Neo4jAdmin + scheduler: + schedule: "0 1 * * *" + sessionHistoryLimit: 3 + timeout: 1h + differentialBackup: + backupInterval: 5m + successfulLogHistoryLimit: 5 + failedLogHistoryLimit: 5 + manifestBackup: + scheduler: + schedule: "*/30 * * * *" + sessionHistoryLimit: 3 + timeout: 15m + deletionPolicy: Delete diff --git a/docs/guides/neo4j/pitr/yamls/restored-neo4j.yaml b/docs/guides/neo4j/pitr/yamls/restored-neo4j.yaml new file mode 100644 index 0000000000..1058c2a070 --- /dev/null +++ b/docs/guides/neo4j/pitr/yamls/restored-neo4j.yaml @@ -0,0 +1,30 @@ +apiVersion: kubedb.com/v1alpha2 +kind: Neo4j +metadata: + name: neo4j-pitr-restored + namespace: demo +spec: + version: "2026.06.0" + replicas: 1 + init: + archiver: + fullDBRepository: + name: neo4j-pitr-source-full + namespace: demo + manifestRepository: + name: neo4j-pitr-source-manifest + namespace: demo + encryptionSecret: + name: neo4j-pitr-encryption + namespace: demo + # Replace with the UTC cutoff recorded in your own run. + recoveryTimestamp: "2026-09-10T13:32:30Z" + storageType: Durable + storage: + storageClassName: local-path + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + deletionPolicy: WipeOut diff --git a/docs/guides/neo4j/pitr/yamls/retention-policy.yaml b/docs/guides/neo4j/pitr/yamls/retention-policy.yaml new file mode 100644 index 0000000000..d0cad0f316 --- /dev/null +++ b/docs/guides/neo4j/pitr/yamls/retention-policy.yaml @@ -0,0 +1,14 @@ +apiVersion: storage.kubestash.com/v1alpha1 +kind: RetentionPolicy +metadata: + name: neo4j-pitr-retention + namespace: demo +spec: + maxRetentionPeriod: 2mo + successfulSnapshots: + last: 5 + failedSnapshots: + last: 2 + usagePolicy: + allowedNamespaces: + from: Same diff --git a/docs/guides/neo4j/pitr/yamls/storage-secret.yaml b/docs/guides/neo4j/pitr/yamls/storage-secret.yaml new file mode 100644 index 0000000000..81dcf8cbea --- /dev/null +++ b/docs/guides/neo4j/pitr/yamls/storage-secret.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Secret +metadata: + name: minio-secret + namespace: demo +type: Opaque +stringData: + AWS_ACCESS_KEY_ID: "" + AWS_SECRET_ACCESS_KEY: ""