Operationalize Production Infrastructure: Multi-Region, Monitoring, Chaos Testing (#117)

* Initial plan

* Add production infrastructure with multi-region deployment, monitoring, and chaos testing

Co-authored-by: Steake <530040+Steake@users.noreply.github.com>

* Add infrastructure validation script and testing documentation

Co-authored-by: Steake <530040+Steake@users.noreply.github.com>

* Address code review feedback: security improvements and configuration fixes

Co-authored-by: Steake <530040+Steake@users.noreply.github.com>

* Add comprehensive implementation summary and finalize documentation

Co-authored-by: Steake <530040+Steake@users.noreply.github.com>

* Fix PR review feedback: improve Python code quality and require Grafana password

- Remove unused imports (random, Optional) from chaos_test.py
- Fix type hint: change 'any' to 'Any'
- Replace bare except clauses with specific exception types
- Remove unused variable nodes_group_a
- Require GRAFANA_ADMIN_PASSWORD (no default) for security
- Document env var substitution requirements in alertmanager.yml
- Add preprocessing notes to kubernetes deployment.yaml
- Update all documentation to reflect required password

Co-authored-by: Steake <530040+Steake@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: Steake <530040+Steake@users.noreply.github.com>

Author: Copilot

.gitignore

@@ -35,5 +35,11 @@ criterion/
 help_output.txt
 target/
 
+# Infrastructure data
+/infra/docker/prometheus-data/
+/infra/docker/grafana-data/
+/infra/docker/alertmanager-data/
+/backups/
+
 # SDK compiled bytecode
 sdk/**/*.bin

crates/bitcell-node/src/monitoring/metrics.rs

@@ -1,5 +1,9 @@
 //! Metrics collection and export
 
+use std::sync::Arc;
+use tokio::net::TcpListener;
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
 pub use super::MetricsRegistry;
 
 /// HTTP server for Prometheus metrics endpoint
@@ -22,8 +26,78 @@ impl MetricsServer {
         self.registry.export_prometheus()
     }
 
-    // Future: Actual HTTP server implementation would go here
-    // For now, just expose the metrics getter
+    /// Get health check status
+    pub fn get_health(&self) -> String {
+        // Basic health check - node is up if we can respond
+        let chain_height = self.registry.get_chain_height();
+        let peer_count = self.registry.get_peer_count();
+        
+        format!(
+            r#"{{"status":"ok","chain_height":{},"peer_count":{}}}"#,
+            chain_height, peer_count
+        )
+    }
+    
+    /// Start HTTP server for metrics and health endpoints
+    pub async fn serve(self) -> Result<(), std::io::Error> {
+        let addr = format!("0.0.0.0:{}", self.port);
+        let listener = TcpListener::bind(&addr).await?;
+        let registry = Arc::new(self.registry);
+        
+        tracing::info!("Metrics server listening on {}", addr);
+        
+        loop {
+            match listener.accept().await {
+                Ok((mut socket, _)) => {
+                    let registry_clone = Arc::clone(&registry);
+                    
+                    tokio::spawn(async move {
+                        let mut buffer = [0; 1024];
+                        
+                        match socket.read(&mut buffer).await {
+                            Ok(n) if n > 0 => {
+                                let request = String::from_utf8_lossy(&buffer[..n]);
+                                
+                                let response = if request.starts_with("GET /health") {
+                                    // Health check endpoint
+                                    let chain_height = registry_clone.get_chain_height();
+                                    let peer_count = registry_clone.get_peer_count();
+                                    let body = format!(
+                                        r#"{{"status":"ok","chain_height":{},"peer_count":{}}}"#,
+                                        chain_height, peer_count
+                                    );
+                                    format!(
+                                        "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{}",
+                                        body.len(), body
+                                    )
+                                } else if request.starts_with("GET /metrics") {
+                                    // Prometheus metrics endpoint
+                                    let body = registry_clone.export_prometheus();
+                                    format!(
+                                        "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\nContent-Length: {}\r\n\r\n{}",
+                                        body.len(), body
+                                    )
+                                } else {
+                                    // 404 for other paths
+                                    let body = "Not Found";
+                                    format!(
+                                        "HTTP/1.1 404 Not Found\r\nContent-Length: {}\r\n\r\n{}",
+                                        body.len(), body
+                                    )
+                                };
+                                
+                                let _ = socket.write_all(response.as_bytes()).await;
+                            }
+                            _ => {}
+                        }
+                    });
+                }
+                Err(e) => {
+                    tracing::error!("Failed to accept connection: {}", e);
+                }
+            }
+        }
+    }
 }
 
 #[cfg(test)]

infra/.gitignore

@@ -0,0 +1,24 @@
+# Temporary data directories
+/tmp/bitcell/
+
+# Docker volumes data
+*.tar.gz
+
+# Backup files
+/backups/
+
+# Log files
+*.log
+
+# Environment files with secrets
+.env.production
+.env.local
+
+# Grafana data
+grafana-data/
+
+# Prometheus data
+prometheus-data/
+
+# Alertmanager data
+alertmanager-data/