diff --git a/backend/services/deploy-mining.sh b/backend/services/deploy-mining.sh index 3cc6237e..6af8821a 100755 --- a/backend/services/deploy-mining.sh +++ b/backend/services/deploy-mining.sh @@ -779,36 +779,73 @@ cdc_resnapshot() { # For each CDC Postgres connector, save config, delete, and recreate log_step "Re-creating CDC Postgres connectors..." + local scripts_dir="$SCRIPT_DIR/scripts/debezium" + for connector in "${CDC_POSTGRES_CONNECTORS[@]}"; do log_info "Processing connector: $connector" - # Get current config + # Get current config from running connector local config config=$(curl -s "$connect_url/connectors/$connector/config" 2>/dev/null) + local config_file="" + local use_file_config=false + + # If connector doesn't exist, try to find config file if [ -z "$config" ] || echo "$config" | grep -q "error_code"; then - log_warn "Connector $connector not found or error getting config, skipping" - continue + log_warn "Connector $connector not found, looking for config file..." + + # Map connector name to config file + case "$connector" in + "identity-postgres-connector") + config_file="$scripts_dir/identity-connector.json" + ;; + "referral-postgres-connector") + config_file="$scripts_dir/referral-connector.json" + ;; + "planting-postgres-connector") + config_file="$scripts_dir/planting-connector.json" + ;; + esac + + if [ -n "$config_file" ] && [ -f "$config_file" ]; then + log_info "Found config file: $config_file" + use_file_config=true + else + log_error "No config available for $connector, skipping" + continue + fi + else + # Delete existing connector + log_info "Deleting connector: $connector" + curl -s -X DELETE "$connect_url/connectors/$connector" &>/dev/null + sleep 2 fi - # Delete connector - log_info "Deleting connector: $connector" - curl -s -X DELETE "$connect_url/connectors/$connector" &>/dev/null - - # Wait for deletion - sleep 2 - - # Recreate with the same config (Debezium will re-snapshot due to snapshot.mode=initial) - log_info "Recreating connector: $connector" + # Create connector + log_info "Creating connector: $connector with snapshot.mode=always" local result - result=$(curl -s -X POST "$connect_url/connectors" \ - -H "Content-Type: application/json" \ - -d "{\"name\":\"$connector\",\"config\":$config}" 2>/dev/null) + + if [ "$use_file_config" = true ]; then + # Use config file, replace snapshot.mode with always + local json_config + json_config=$(cat "$config_file" | envsubst | sed 's/"snapshot.mode": "initial"/"snapshot.mode": "always"/') + result=$(echo "$json_config" | curl -s -X POST "$connect_url/connectors" \ + -H "Content-Type: application/json" \ + -d @- 2>/dev/null) + else + # Use config from running connector, but change snapshot.mode to always + local modified_config + modified_config=$(echo "$config" | sed 's/"snapshot.mode":"initial"/"snapshot.mode":"always"/' | sed 's/"snapshot.mode": "initial"/"snapshot.mode": "always"/') + result=$(curl -s -X POST "$connect_url/connectors" \ + -H "Content-Type: application/json" \ + -d "{\"name\":\"$connector\",\"config\":$modified_config}" 2>/dev/null) + fi if echo "$result" | grep -q '"name"'; then - log_success "Recreated connector: $connector" + log_success "Created connector: $connector" else - log_error "Failed to recreate connector $connector: $result" + log_error "Failed to create connector $connector: $result" fi # Wait between connectors diff --git a/backend/services/scripts/debezium/identity-connector.json b/backend/services/scripts/debezium/identity-connector.json index e8888606..1fde3417 100644 --- a/backend/services/scripts/debezium/identity-connector.json +++ b/backend/services/scripts/debezium/identity-connector.json @@ -4,7 +4,7 @@ "connector.class": "io.debezium.connector.postgresql.PostgresConnector", "tasks.max": "1", - "database.hostname": "postgres", + "database.hostname": "rwa-postgres", "database.port": "5432", "database.user": "${POSTGRES_USER:-rwa_user}", "database.password": "${POSTGRES_PASSWORD:-rwa_secure_password}", diff --git a/backend/services/scripts/debezium/planting-connector.json b/backend/services/scripts/debezium/planting-connector.json index 6b1eda01..e2b92827 100644 --- a/backend/services/scripts/debezium/planting-connector.json +++ b/backend/services/scripts/debezium/planting-connector.json @@ -4,7 +4,7 @@ "connector.class": "io.debezium.connector.postgresql.PostgresConnector", "tasks.max": "1", - "database.hostname": "postgres", + "database.hostname": "rwa-postgres", "database.port": "5432", "database.user": "${POSTGRES_USER:-rwa_user}", "database.password": "${POSTGRES_PASSWORD:-rwa_secure_password}", diff --git a/backend/services/scripts/debezium/referral-connector.json b/backend/services/scripts/debezium/referral-connector.json index d8bd7d9d..c3b08270 100644 --- a/backend/services/scripts/debezium/referral-connector.json +++ b/backend/services/scripts/debezium/referral-connector.json @@ -4,7 +4,7 @@ "connector.class": "io.debezium.connector.postgresql.PostgresConnector", "tasks.max": "1", - "database.hostname": "postgres", + "database.hostname": "rwa-postgres", "database.port": "5432", "database.user": "${POSTGRES_USER:-rwa_user}", "database.password": "${POSTGRES_PASSWORD:-rwa_secure_password}",