diff --git a/backend/services/deploy-mining.sh b/backend/services/deploy-mining.sh new file mode 100644 index 00000000..33e85229 --- /dev/null +++ b/backend/services/deploy-mining.sh @@ -0,0 +1,655 @@ +#!/bin/bash +# +# RWA Mining Ecosystem 2.0 - Deployment & Management Script +# ========================================================== +# +# This script manages the Mining 2.0 ecosystem independently from the 1.0 system. +# The 2.0 system is completely isolated and can be reset at any time without +# affecting the 1.0 system. +# +# Usage: +# ./deploy-mining.sh up # Start all 2.0 services +# ./deploy-mining.sh down # Stop all 2.0 services +# ./deploy-mining.sh restart # Restart all 2.0 services +# ./deploy-mining.sh status # Show 2.0 service status +# ./deploy-mining.sh logs [service] # View logs (optional: specific service) +# ./deploy-mining.sh build # Rebuild all 2.0 images +# +# Database Management: +# ./deploy-mining.sh db-create # Create 2.0 databases +# ./deploy-mining.sh db-migrate # Run Prisma migrations +# ./deploy-mining.sh db-reset # Drop and recreate 2.0 databases (DANGEROUS!) +# ./deploy-mining.sh db-status # Show database status +# +# CDC & Sync: +# ./deploy-mining.sh sync-reset # Reset CDC consumer offsets to beginning +# ./deploy-mining.sh sync-status # Show CDC consumer group status +# ./deploy-mining.sh sync-start # Start CDC sync from current position +# +# Full Reset (for development/testing): +# ./deploy-mining.sh full-reset # Complete reset: stop services, drop DBs, recreate, resync +# +# Health & Monitoring: +# ./deploy-mining.sh health # Check health of all 2.0 services +# ./deploy-mining.sh stats # Show system statistics +# + +set -e + +# =========================================================================== +# Configuration +# =========================================================================== +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="$SCRIPT_DIR/.env" + +# 2.0 Services +MINING_SERVICES=( + "contribution-service" + "mining-service" + "trading-service" + "mining-admin-service" +) + +# 2.0 Databases +MINING_DATABASES=( + "rwa_contribution" + "rwa_mining" + "rwa_trading" + "rwa_mining_admin" +) + +# 2.0 Ports +declare -A SERVICE_PORTS=( + ["contribution-service"]="3020" + ["mining-service"]="3021" + ["trading-service"]="3022" + ["mining-admin-service"]="3023" +) + +# CDC Consumer Group +CDC_CONSUMER_GROUP="contribution-service-cdc-group" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +MAGENTA='\033[0;35m' +NC='\033[0m' +BOLD='\033[1m' + +# =========================================================================== +# Logging Functions +# =========================================================================== +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } +log_step() { echo -e "${BLUE}[STEP]${NC} $1"; } +log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } + +print_header() { + echo "" + echo -e "${CYAN}╔════════════════════════════════════════════════════════════╗${NC}" + echo -e "${CYAN}║${NC} ${BOLD}RWA Mining Ecosystem 2.0 - Management Script${NC} ${CYAN}║${NC}" + echo -e "${CYAN}╚════════════════════════════════════════════════════════════╝${NC}" + echo "" +} + +print_section() { + echo "" + echo -e "${MAGENTA}━━━ $1 ━━━${NC}" + echo "" +} + +# =========================================================================== +# Environment Loading +# =========================================================================== +load_env() { + if [ -f "$ENV_FILE" ]; then + set -a + source "$ENV_FILE" + set +a + else + log_warn "No .env file found, using defaults" + fi + + # Set defaults + POSTGRES_HOST="${POSTGRES_HOST:-localhost}" + POSTGRES_PORT="${POSTGRES_PORT:-5432}" + POSTGRES_USER="${POSTGRES_USER:-postgres}" + POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-postgres}" + KAFKA_BROKERS="${KAFKA_BROKERS:-localhost:9092}" + REDIS_HOST="${REDIS_HOST:-localhost}" + REDIS_PORT="${REDIS_PORT:-6379}" +} + +# =========================================================================== +# Database Functions +# =========================================================================== +db_create() { + print_section "Creating 2.0 Databases" + + for db in "${MINING_DATABASES[@]}"; do + log_step "Creating database: $db" + PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d postgres -c "CREATE DATABASE $db;" 2>/dev/null || { + log_warn "Database $db already exists or creation failed" + } + done + + log_success "Database creation completed" +} + +db_drop() { + print_section "Dropping 2.0 Databases" + + for db in "${MINING_DATABASES[@]}"; do + log_step "Dropping database: $db" + PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d postgres -c "DROP DATABASE IF EXISTS $db WITH (FORCE);" 2>/dev/null || { + log_warn "Failed to drop database $db" + } + done + + log_success "Database drop completed" +} + +db_migrate() { + print_section "Running Prisma Migrations" + + for service in "${MINING_SERVICES[@]}"; do + service_dir="$SCRIPT_DIR/$service" + if [ -d "$service_dir/prisma" ]; then + log_step "Migrating: $service" + cd "$service_dir" + npx prisma migrate deploy 2>/dev/null || npx prisma db push --accept-data-loss + cd "$SCRIPT_DIR" + fi + done + + log_success "Migrations completed" +} + +db_status() { + print_section "2.0 Database Status" + + echo -e "${BOLD}Database${NC}\t\t${BOLD}Status${NC}\t\t${BOLD}Tables${NC}" + echo "────────────────────────────────────────────────────" + + for db in "${MINING_DATABASES[@]}"; do + if PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$db" -c "SELECT 1" &>/dev/null; then + table_count=$(PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$db" -t -c "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = 'public';" 2>/dev/null | tr -d ' ') + echo -e "${GREEN}$db${NC}\t${GREEN}UP${NC}\t\t$table_count tables" + else + echo -e "${RED}$db${NC}\t\t${RED}DOWN${NC}\t\t-" + fi + done +} + +db_reset() { + print_section "Resetting 2.0 Databases" + + echo -e "${RED}${BOLD}WARNING: This will DELETE ALL 2.0 DATA!${NC}" + echo "Affected databases:" + for db in "${MINING_DATABASES[@]}"; do + echo " - $db" + done + echo "" + read -p "Are you sure? Type 'yes' to confirm: " confirm + + if [ "$confirm" != "yes" ]; then + log_warn "Aborted" + return 1 + fi + + db_drop + db_create + db_migrate + + log_success "Database reset completed" +} + +# =========================================================================== +# Service Functions +# =========================================================================== +services_up() { + print_section "Starting 2.0 Services" + + for service in "${MINING_SERVICES[@]}"; do + log_step "Starting: $service" + docker-compose -f "$SCRIPT_DIR/docker-compose.yml" up -d "$service" 2>/dev/null || { + log_warn "$service not in docker-compose, trying npm start" + service_dir="$SCRIPT_DIR/$service" + if [ -d "$service_dir" ]; then + cd "$service_dir" + nohup npm run start:prod > /tmp/$service.log 2>&1 & + echo $! > /tmp/$service.pid + cd "$SCRIPT_DIR" + fi + } + done + + log_success "Services started" +} + +services_down() { + print_section "Stopping 2.0 Services" + + for service in "${MINING_SERVICES[@]}"; do + log_step "Stopping: $service" + docker-compose -f "$SCRIPT_DIR/docker-compose.yml" stop "$service" 2>/dev/null || { + if [ -f "/tmp/$service.pid" ]; then + kill $(cat /tmp/$service.pid) 2>/dev/null || true + rm -f /tmp/$service.pid + fi + } + done + + log_success "Services stopped" +} + +services_restart() { + services_down + sleep 2 + services_up +} + +services_status() { + print_section "2.0 Service Status" + + echo -e "${BOLD}Service${NC}\t\t\t${BOLD}Port${NC}\t${BOLD}Status${NC}\t\t${BOLD}Health${NC}" + echo "────────────────────────────────────────────────────────────────" + + for service in "${MINING_SERVICES[@]}"; do + port="${SERVICE_PORTS[$service]}" + + # Check if port is listening + if nc -z localhost "$port" 2>/dev/null; then + status="${GREEN}RUNNING${NC}" + + # Check health endpoint + health_response=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:$port/health" 2>/dev/null || echo "000") + if [ "$health_response" = "200" ]; then + health="${GREEN}HEALTHY${NC}" + else + health="${YELLOW}UNKNOWN${NC}" + fi + else + status="${RED}STOPPED${NC}" + health="${RED}-${NC}" + fi + + echo -e "$service\t$port\t$status\t$health" + done +} + +services_logs() { + local service="$1" + + if [ -n "$service" ]; then + docker-compose -f "$SCRIPT_DIR/docker-compose.yml" logs -f "$service" 2>/dev/null || { + if [ -f "/tmp/$service.log" ]; then + tail -f "/tmp/$service.log" + else + log_error "No logs found for $service" + fi + } + else + for svc in "${MINING_SERVICES[@]}"; do + docker-compose -f "$SCRIPT_DIR/docker-compose.yml" logs --tail=50 "$svc" 2>/dev/null || true + done + fi +} + +services_build() { + print_section "Building 2.0 Services" + + for service in "${MINING_SERVICES[@]}"; do + service_dir="$SCRIPT_DIR/$service" + if [ -d "$service_dir" ]; then + log_step "Building: $service" + cd "$service_dir" + npm run build + cd "$SCRIPT_DIR" + fi + done + + log_success "Build completed" +} + +# =========================================================================== +# CDC / Sync Functions +# =========================================================================== +sync_reset() { + print_section "Resetting CDC Consumer Offsets" + + echo -e "${YELLOW}This will reset the CDC consumer to read from the beginning.${NC}" + echo "Consumer Group: $CDC_CONSUMER_GROUP" + echo "" + read -p "Continue? (y/n): " confirm + + if [ "$confirm" != "y" ]; then + log_warn "Aborted" + return 1 + fi + + # Stop contribution-service first + log_step "Stopping contribution-service" + docker-compose -f "$SCRIPT_DIR/docker-compose.yml" stop contribution-service 2>/dev/null || true + + # Reset offsets + log_step "Resetting consumer group offsets" + kafka-consumer-groups.sh --bootstrap-server "$KAFKA_BROKERS" \ + --group "$CDC_CONSUMER_GROUP" \ + --reset-offsets \ + --to-earliest \ + --all-topics \ + --execute 2>/dev/null || { + # Try with docker + docker exec -it kafka kafka-consumer-groups.sh --bootstrap-server localhost:9092 \ + --group "$CDC_CONSUMER_GROUP" \ + --reset-offsets \ + --to-earliest \ + --all-topics \ + --execute 2>/dev/null || log_warn "Could not reset offsets automatically" + } + + log_success "CDC consumer offsets reset to beginning" + log_info "Start contribution-service to begin syncing from the beginning" +} + +sync_status() { + print_section "CDC Sync Status" + + echo -e "${BOLD}Consumer Group:${NC} $CDC_CONSUMER_GROUP" + echo "" + + kafka-consumer-groups.sh --bootstrap-server "$KAFKA_BROKERS" \ + --group "$CDC_CONSUMER_GROUP" \ + --describe 2>/dev/null || { + docker exec -it kafka kafka-consumer-groups.sh --bootstrap-server localhost:9092 \ + --group "$CDC_CONSUMER_GROUP" \ + --describe 2>/dev/null || log_warn "Could not get consumer group status" + } +} + +# =========================================================================== +# Full Reset Function +# =========================================================================== +full_reset() { + print_section "Full 2.0 System Reset" + + echo -e "${RED}${BOLD}╔════════════════════════════════════════════════════════════╗${NC}" + echo -e "${RED}${BOLD}║ WARNING: This will completely reset the 2.0 system! ║${NC}" + echo -e "${RED}${BOLD}║ ║${NC}" + echo -e "${RED}${BOLD}║ - Stop all 2.0 services ║${NC}" + echo -e "${RED}${BOLD}║ - Drop all 2.0 databases ║${NC}" + echo -e "${RED}${BOLD}║ - Recreate databases ║${NC}" + echo -e "${RED}${BOLD}║ - Run migrations ║${NC}" + echo -e "${RED}${BOLD}║ - Reset CDC consumer offsets ║${NC}" + echo -e "${RED}${BOLD}║ - Restart services (will sync from 1.0) ║${NC}" + echo -e "${RED}${BOLD}║ ║${NC}" + echo -e "${RED}${BOLD}║ This will NOT affect the 1.0 system in any way. ║${NC}" + echo -e "${RED}${BOLD}╚════════════════════════════════════════════════════════════╝${NC}" + echo "" + read -p "Type 'RESET' to confirm: " confirm + + if [ "$confirm" != "RESET" ]; then + log_warn "Aborted" + return 1 + fi + + echo "" + log_step "Step 1/6: Stopping 2.0 services..." + services_down + + log_step "Step 2/6: Dropping 2.0 databases..." + db_drop + + log_step "Step 3/6: Creating 2.0 databases..." + db_create + + log_step "Step 4/6: Running migrations..." + db_migrate + + log_step "Step 5/6: Resetting CDC consumer offsets..." + kafka-consumer-groups.sh --bootstrap-server "$KAFKA_BROKERS" \ + --group "$CDC_CONSUMER_GROUP" \ + --reset-offsets \ + --to-earliest \ + --all-topics \ + --execute 2>/dev/null || { + docker exec kafka kafka-consumer-groups.sh --bootstrap-server localhost:9092 \ + --group "$CDC_CONSUMER_GROUP" \ + --reset-offsets \ + --to-earliest \ + --all-topics \ + --execute 2>/dev/null || log_warn "Could not reset offsets, may need manual reset" + } + + log_step "Step 6/6: Starting 2.0 services..." + services_up + + echo "" + echo -e "${GREEN}${BOLD}╔════════════════════════════════════════════════════════════╗${NC}" + echo -e "${GREEN}${BOLD}║ Full reset completed successfully! ║${NC}" + echo -e "${GREEN}${BOLD}║ ║${NC}" + echo -e "${GREEN}${BOLD}║ The 2.0 system will now sync all data from 1.0 via CDC. ║${NC}" + echo -e "${GREEN}${BOLD}║ Monitor with: ./deploy-mining.sh logs contribution-service║${NC}" + echo -e "${GREEN}${BOLD}╚════════════════════════════════════════════════════════════╝${NC}" +} + +# =========================================================================== +# Health Check Function +# =========================================================================== +health_check() { + print_section "2.0 System Health Check" + + local all_healthy=true + + # Check databases + echo -e "${BOLD}Databases:${NC}" + for db in "${MINING_DATABASES[@]}"; do + if PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$db" -c "SELECT 1" &>/dev/null; then + echo -e " ${GREEN}✓${NC} $db" + else + echo -e " ${RED}✗${NC} $db" + all_healthy=false + fi + done + + echo "" + echo -e "${BOLD}Services:${NC}" + for service in "${MINING_SERVICES[@]}"; do + port="${SERVICE_PORTS[$service]}" + health_response=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:$port/health" 2>/dev/null || echo "000") + if [ "$health_response" = "200" ]; then + echo -e " ${GREEN}✓${NC} $service (port $port)" + else + echo -e " ${RED}✗${NC} $service (port $port)" + all_healthy=false + fi + done + + echo "" + echo -e "${BOLD}Infrastructure:${NC}" + # Kafka + if nc -z ${KAFKA_BROKERS%%:*} ${KAFKA_BROKERS##*:} 2>/dev/null; then + echo -e " ${GREEN}✓${NC} Kafka ($KAFKA_BROKERS)" + else + echo -e " ${RED}✗${NC} Kafka ($KAFKA_BROKERS)" + all_healthy=false + fi + + # Redis + if nc -z "$REDIS_HOST" "$REDIS_PORT" 2>/dev/null; then + echo -e " ${GREEN}✓${NC} Redis ($REDIS_HOST:$REDIS_PORT)" + else + echo -e " ${RED}✗${NC} Redis ($REDIS_HOST:$REDIS_PORT)" + all_healthy=false + fi + + echo "" + if [ "$all_healthy" = true ]; then + echo -e "${GREEN}${BOLD}All systems healthy!${NC}" + else + echo -e "${RED}${BOLD}Some systems are unhealthy!${NC}" + return 1 + fi +} + +# =========================================================================== +# Statistics Function +# =========================================================================== +show_stats() { + print_section "2.0 System Statistics" + + for db in "${MINING_DATABASES[@]}"; do + echo -e "${BOLD}Database: $db${NC}" + + if PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$db" -c "SELECT 1" &>/dev/null; then + # Get table row counts + PGPASSWORD="$POSTGRES_PASSWORD" psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$db" -t -c " + SELECT tablename AS table, + pg_size_pretty(pg_total_relation_size(schemaname || '.' || tablename)) AS size + FROM pg_tables + WHERE schemaname = 'public' + ORDER BY pg_total_relation_size(schemaname || '.' || tablename) DESC + LIMIT 10; + " 2>/dev/null || echo " Could not get table stats" + else + echo " Database not available" + fi + echo "" + done +} + +# =========================================================================== +# Help Function +# =========================================================================== +show_help() { + print_header + + echo "Usage: $0 [options]" + echo "" + echo -e "${BOLD}Service Management:${NC}" + echo " up Start all 2.0 services" + echo " down Stop all 2.0 services" + echo " restart Restart all 2.0 services" + echo " status Show service status" + echo " logs [service] View logs (optionally for specific service)" + echo " build Build all 2.0 services" + echo "" + echo -e "${BOLD}Database Management:${NC}" + echo " db-create Create 2.0 databases" + echo " db-migrate Run Prisma migrations" + echo " db-reset Drop and recreate databases ${RED}(DANGEROUS!)${NC}" + echo " db-status Show database status" + echo "" + echo -e "${BOLD}CDC / Sync Management:${NC}" + echo " sync-reset Reset CDC consumer to read from beginning" + echo " sync-status Show CDC consumer group status" + echo "" + echo -e "${BOLD}Full Reset:${NC}" + echo " full-reset Complete system reset ${RED}(DANGEROUS!)${NC}" + echo " Drops DBs, resets CDC, restarts services" + echo "" + echo -e "${BOLD}Health & Monitoring:${NC}" + echo " health Check health of all 2.0 components" + echo " stats Show system statistics" + echo "" + echo -e "${BOLD}2.0 Services:${NC}" + for service in "${MINING_SERVICES[@]}"; do + echo " - $service (port ${SERVICE_PORTS[$service]})" + done + echo "" + echo -e "${BOLD}2.0 Databases:${NC}" + for db in "${MINING_DATABASES[@]}"; do + echo " - $db" + done + echo "" + echo -e "${YELLOW}Note: The 2.0 system is completely isolated from 1.0.${NC}" + echo -e "${YELLOW}Any reset operation will NOT affect the 1.0 system.${NC}" +} + +# =========================================================================== +# Main +# =========================================================================== +main() { + load_env + + case "${1:-}" in + # Service commands + up) + services_up + ;; + down) + services_down + ;; + restart) + services_restart + ;; + status) + print_header + services_status + ;; + logs) + services_logs "$2" + ;; + build) + services_build + ;; + + # Database commands + db-create) + db_create + ;; + db-migrate) + db_migrate + ;; + db-reset) + db_reset + ;; + db-status) + print_header + db_status + ;; + + # Sync commands + sync-reset) + sync_reset + ;; + sync-status) + sync_status + ;; + + # Full reset + full-reset) + print_header + full_reset + ;; + + # Health & monitoring + health) + print_header + health_check + ;; + stats) + print_header + show_stats + ;; + + # Help + help|--help|-h|"") + show_help + ;; + + *) + log_error "Unknown command: $1" + echo "" + show_help + exit 1 + ;; + esac +} + +main "$@"