diff --git a/.drone.yml b/.drone.yml index 53e1d1b..5d23017 100644 --- a/.drone.yml +++ b/.drone.yml @@ -305,3 +305,276 @@ steps: depends_on: - check-trigger - text-matcher-contracts +--- +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🚀 Pipeline 4: Full Release +# Trigger: Tag on main branch WITHOUT "contracts_only" or "deploy_only" +# Purpose: Full release cycle - contracts → images → optional deploy +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +kind: pipeline +type: kubernetes +name: full-release + +metadata: + namespace: musk-drone + +trigger: + ref: + - refs/tags/* + event: + - tag + branch: + - main + +clone: + disable: true + +volumes: + - name: nuget-cache + temp: {} + +steps: + - name: clone + image: alpine/git + commands: + - git clone https://gitea.musk.fun/nocr/flea + - cd flea + - git checkout $DRONE_TAG + - git submodule update --init --recursive + + - name: check-trigger + image: alpine/git + commands: + - cd flea + - COMMIT_MSG=$(git log -1 --pretty=%B) + - echo "Commit message - $COMMIT_MSG" + - | + if echo "$COMMIT_MSG" | grep -qE "contracts_only:|deploy_only:"; then + echo "⏭️ contracts_only or deploy_only detected, skipping full release..." + exit 78 + else + echo "✅ Full release triggered" + exit 0 + fi + depends_on: + - clone + + # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + # STAGE 1: Publish NuGet Contracts (sequental) + # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + - name: telegram-listener-nuget + image: mcr.microsoft.com/dotnet/sdk:8.0 + volumes: + - name: nuget-cache + path: /root/.nuget/packages + environment: + NUGETAPIKEY: + from_secret: nuget_musk_api_key + commands: + - cd flea + - dotnet nuget add source --name musk https://gitea.musk.fun/api/packages/nocr/nuget/index.json + - dotnet pack telegram-listener/Nocr.TelegramListener.sln -o ./bin -p:PackageVersion=${DRONE_TAG} + - dotnet nuget push ./bin/*Contract*.nupkg --api-key $NUGETAPIKEY --source musk --skip-duplicate + depends_on: + - check-trigger + + - name: text-matcher-nuget + image: mcr.microsoft.com/dotnet/sdk:8.0 + volumes: + - name: nuget-cache + path: /root/.nuget/packages + environment: + NUGETAPIKEY: + from_secret: nuget_musk_api_key + commands: + - cd flea + - dotnet nuget add source --name musk https://gitea.musk.fun/api/packages/nocr/nuget/index.json + - dotnet pack text-matcher/Nocr.TextMatcher.sln -o ./bin -p:PackageVersion=${DRONE_TAG} + - dotnet nuget push ./bin/*Contract*.nupkg --api-key $NUGETAPIKEY --source musk --skip-duplicate + depends_on: + - check-trigger + - telegram-listener-nuget + + - name: users-nuget + image: mcr.microsoft.com/dotnet/sdk:8.0 + volumes: + - name: nuget-cache + path: /root/.nuget/packages + environment: + NUGETAPIKEY: + from_secret: nuget_musk_api_key + commands: + - cd flea + - dotnet nuget add source --name musk https://gitea.musk.fun/api/packages/nocr/nuget/index.json + - dotnet pack users/Nocr.Users.sln -o ./bin -p:PackageVersion=${DRONE_TAG} + - dotnet nuget push ./bin/*Contract*.nupkg --api-key $NUGETAPIKEY --source musk --skip-duplicate + depends_on: + - check-trigger + - text-matcher-nuget + + # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + # STAGE 2: Build Docker Images with Kaniko (3 parallel streams) + # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + - name: telegram-listener-build-push + image: gcr.io/kaniko-project/executor:debug + environment: + HUB_USERNAME: + from_secret: hub_username + HUB_PASSWORD: + from_secret: hub_password + commands: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"hub.musk.fun\":{\"username\":\"$HUB_USERNAME\",\"password\":\"$HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json + - cd flea/telegram-listener + - /kaniko/executor + --context=. + --dockerfile=src/Nocr.TelegramListener.Host/Dockerfile + --destination=hub.musk.fun/k8s/nocr/telegram_listener:${DRONE_COMMIT_SHA:0:7} + --destination=hub.musk.fun/k8s/nocr/telegram_listener:${DRONE_TAG} + --destination=hub.musk.fun/k8s/nocr/telegram_listener:latest + --cache=true + --cache-repo=hub.musk.fun/k8s/cache/nocr-telegram-listener + --compressed-caching=true + depends_on: + - telegram-listener-nuget + - text-matcher-nuget + - users-nuget + + - name: telegram-client-build-push + image: gcr.io/kaniko-project/executor:debug + environment: + HUB_USERNAME: + from_secret: hub_username + HUB_PASSWORD: + from_secret: hub_password + commands: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"hub.musk.fun\":{\"username\":\"$HUB_USERNAME\",\"password\":\"$HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json + - cd flea/telegram-client + - /kaniko/executor + --context=. + --dockerfile=src/Nocr.TelegramClient.Host/Dockerfile + --destination=hub.musk.fun/k8s/nocr/telegram_client:${DRONE_COMMIT_SHA:0:7} + --destination=hub.musk.fun/k8s/nocr/telegram_client:${DRONE_TAG} + --destination=hub.musk.fun/k8s/nocr/telegram_client:latest + --cache=true + --cache-repo=hub.musk.fun/k8s/cache/nocr-telegram-client + --compressed-caching=true + depends_on: + - telegram-listener-build-push + + - name: text-matcher-build-push + image: gcr.io/kaniko-project/executor:debug + environment: + HUB_USERNAME: + from_secret: hub_username + HUB_PASSWORD: + from_secret: hub_password + commands: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"hub.musk.fun\":{\"username\":\"$HUB_USERNAME\",\"password\":\"$HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json + - cd flea/text-matcher + - /kaniko/executor + --context=. + --dockerfile=src/Nocr.TextMatcher.Host/Dockerfile + --destination=hub.musk.fun/k8s/nocr/text_matcher:${DRONE_COMMIT_SHA:0:7} + --destination=hub.musk.fun/k8s/nocr/text_matcher:${DRONE_TAG} + --destination=hub.musk.fun/k8s/nocr/text_matcher:latest + --cache=true + --cache-repo=hub.musk.fun/k8s/cache/nocr-text-matcher + --compressed-caching=true + depends_on: + - telegram-listener-nuget + - text-matcher-nuget + - users-nuget + + - name: text-matcher-migrator-build-push + image: gcr.io/kaniko-project/executor:debug + environment: + HUB_USERNAME: + from_secret: hub_username + HUB_PASSWORD: + from_secret: hub_password + commands: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"hub.musk.fun\":{\"username\":\"$HUB_USERNAME\",\"password\":\"$HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json + - cd flea/text-matcher + - /kaniko/executor + --context=. + --dockerfile=src/Nocr.TextMatcher.Migrator/Dockerfile + --destination=hub.musk.fun/k8s/nocr/text_matcher_migrator:${DRONE_COMMIT_SHA:0:7} + --destination=hub.musk.fun/k8s/nocr/text_matcher_migrator:${DRONE_TAG} + --destination=hub.musk.fun/k8s/nocr/text_matcher_migrator:latest + --cache=true + --cache-repo=hub.musk.fun/k8s/cache/nocr-text-matcher-migrator + --compressed-caching=true + depends_on: + - text-matcher-build-push + + - name: users-build-push + image: gcr.io/kaniko-project/executor:debug + environment: + HUB_USERNAME: + from_secret: hub_username + HUB_PASSWORD: + from_secret: hub_password + commands: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"hub.musk.fun\":{\"username\":\"$HUB_USERNAME\",\"password\":\"$HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json + - cd flea/users + - /kaniko/executor + --context=. + --dockerfile=src/Nocr.Users.Host/Dockerfile + --destination=hub.musk.fun/k8s/nocr/users:${DRONE_COMMIT_SHA:0:7} + --destination=hub.musk.fun/k8s/nocr/users:${DRONE_TAG} + --destination=hub.musk.fun/k8s/nocr/users:latest + --cache=true + --cache-repo=hub.musk.fun/k8s/cache/nocr-users + --compressed-caching=true + depends_on: + - telegram-listener-nuget + - text-matcher-nuget + - users-nuget + + - name: users-migrator-build-push + image: gcr.io/kaniko-project/executor:debug + environment: + HUB_USERNAME: + from_secret: hub_username + HUB_PASSWORD: + from_secret: hub_password + commands: + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"hub.musk.fun\":{\"username\":\"$HUB_USERNAME\",\"password\":\"$HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json + - cd flea/users + - /kaniko/executor + --context=. + --dockerfile=src/Nocr.Users.Migrator/Dockerfile + --destination=hub.musk.fun/k8s/nocr/users_migrator:${DRONE_COMMIT_SHA:0:7} + --destination=hub.musk.fun/k8s/nocr/users_migrator:${DRONE_TAG} + --destination=hub.musk.fun/k8s/nocr/users_migrator:latest + --cache=true + --cache-repo=hub.musk.fun/k8s/cache/nocr-users-migrator + --compressed-caching=true + depends_on: + - users-build-push + + # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + # STAGE 3: Deploy to Kubernetes (optional, based on tag pattern) + # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + + - name: deploy-to-k8s + image: bitnami/kubectl:latest + commands: + - cd flea/_deploy/scripts + - chmod +x deploy.sh + - ./deploy.sh ${DRONE_TAG} ${DRONE_COMMIT_SHA:0:7} + depends_on: + - telegram-client-build-push + - text-matcher-migrator-build-push + - users-migrator-build-push + when: + ref: + - refs/tags/* diff --git a/_deploy/README.md b/_deploy/README.md new file mode 100644 index 0000000..ab08e5c --- /dev/null +++ b/_deploy/README.md @@ -0,0 +1,301 @@ +# 🚀 Nocr CI/CD Pipeline Documentation + +## 📋 Overview + +The Nocr project uses a modern, multi-pipeline CI/CD setup powered by Drone CI on Kubernetes. This document describes the 5 specialized pipelines and how to use them. + +--- + +## 🎯 Pipeline Architecture + +### Pipeline 1: **Feature Validation** +**Trigger:** Push to `feature/*` or `fix/*` branches +**Purpose:** Fast feedback for developers +**Duration:** ~3-5 minutes + +**What it does:** +- Clones repo with submodules +- Restores all NuGet packages (shared cache) +- Builds all 4 services in Release mode +- Runs unit and integration tests with Testcontainers + +**Example workflow:** +```bash +git checkout -b feature/add-new-filter +# Make changes... +git add . +git commit -m "Add new filter functionality" +git push origin feature/add-new-filter +``` + +Drone automatically runs tests. Check results before creating PR. + +--- + +### Pipeline 2: **Main Validation** +**Trigger:** Push to `main` branch +**Purpose:** Validate main branch after merge +**Duration:** ~3-5 minutes + +**What it does:** +- Same as Feature Validation +- Ensures main branch is always in working state + +**Example workflow:** +```bash +# After PR is merged to main +# Pipeline runs automatically +``` + +--- + +### Pipeline 3: **Contracts-Only Publish** +**Trigger:** Tag with commit message containing `contracts_only:` +**Purpose:** Fast publish of contract packages without building images +**Duration:** ~2 minutes + +**What it does:** +- Packs specified service contracts into NuGet packages +- Publishes to internal NuGet feed +- Skips Docker image builds + +**Example workflow:** +```bash +# Update telegram-listener contracts +cd telegram-listener +# Make changes to Async.Api.Contracts... +git add . +git commit -m "contracts_only:telegram_listener - Add MessageEdited event" +git push origin main + +# Create tag +git tag v1.2.4-contracts +git push origin v1.2.4-contracts +``` + +**Supported markers:** +- `contracts_only:telegram_listener` +- `contracts_only:text_matcher` +- `contracts_only:users` + +--- + +### Pipeline 4: **Full Release** +**Trigger:** Tag on main WITHOUT `contracts_only` or `deploy_only` in commit message +**Purpose:** Complete release cycle +**Duration:** ~8-10 minutes + +**What it does:** +1. **Stage 1:** Publish all contracts to NuGet (parallel) +2. **Stage 2:** Build all Docker images with Kaniko (3 parallel streams) +3. **Stage 3:** Deploy to Kubernetes (only for tags matching `v*`) + +**Example workflow:** +```bash +# Ready to release +git tag v1.3.0 +git push origin v1.3.0 + +# Drone will: +# 1. Publish contracts +# 2. Build images (tagged with v1.3.0, commit SHA, and latest) +# 3. Deploy to k8s (if tag starts with 'v') +``` + +**Image tags created:** +- `hub.musk.fun/k8s/nocr/telegram_listener:v1.3.0` +- `hub.musk.fun/k8s/nocr/telegram_listener:abc1234` (commit SHA) +- `hub.musk.fun/k8s/nocr/telegram_listener:latest` + +--- + +### Pipeline 5: **Deploy-Only** +**Trigger:** Tag with commit message containing `deploy_only:` +**Purpose:** Fast deploy of already-built images +**Duration:** ~1 minute + +**What it does:** +- Skips building +- Deploys specified images to Kubernetes +- Useful for rolling back or promoting existing images + +**Example workflow:** +```bash +# Deploy existing images +git commit --allow-empty -m "deploy_only: Deploy v1.2.9" +git tag v1.2.9-deploy +git push origin v1.2.9-deploy +``` + +--- + +## 🛠️ Deployment Scripts + +All deployment scripts are located in `_deploy/scripts/`: + +### `deploy.sh` +**Purpose:** Deploy services to Kubernetes +**Usage:** +```bash +./deploy.sh +./deploy.sh v1.3.0 abc1234 +``` + +**Features:** +- Updates deployment manifests with new image tags +- Applies manifests to cluster +- Waits for rollouts to complete with timeout +- Runs health checks after deployment +- Shows pod status + +### `rollback.sh` +**Purpose:** Rollback deployments to previous version +**Usage:** +```bash +# Rollback single service +./rollback.sh telegram-listener + +# Rollback all services +./rollback.sh all +``` + +**Features:** +- Shows revision history +- Performs kubectl rollout undo +- Waits for rollback to complete +- Runs health checks after rollback + +### `health-check.sh` +**Purpose:** Check health of all Nocr services +**Usage:** +```bash +./health-check.sh +``` + +**Checks:** +- Pod status (Running/Ready) +- Health endpoints (/health) +- Recent events for failed pods + +--- + +## 📦 Optimizations + +### Shared NuGet Cache +All pipelines use a shared temp volume for NuGet packages: +- First `dotnet restore` downloads packages +- Subsequent builds reuse cached packages +- **~60% faster** than individual restores per service + +### Parallel Execution +- Contract publishing: 3 services in parallel +- Docker builds: 3 parallel streams +- Independent operations never block each other + +### Kaniko Caching +All Kaniko builds use: +- `--cache=true` - Layer caching enabled +- `--cache-repo=hub.musk.fun/k8s/cache/*` - Shared cache repo +- `--compressed-caching=true` - Faster cache transfer + +--- + +## 🧪 Testcontainers Support + +Feature and Main validation pipelines include Docker-in-Docker service for Testcontainers: + +```yaml +services: + - name: docker + image: docker:27-dind + privileged: true +``` + +Tests can use Testcontainers to spin up real databases, message queues, etc. + +--- + +## 🔒 Required Secrets + +Configure these in Drone: + +- `hub_username` - Docker registry username +- `hub_password` - Docker registry password +- `nuget_musk_api_key` - NuGet feed API key + +--- + +## 📊 Pipeline Decision Tree + +``` +Push to feature/* → Feature Validation (build + test) +Push to main → Main Validation (build + test) + +Tag + "contracts_only:" → Contracts Publish +Tag + "deploy_only:" → Deploy Only +Tag (no markers) → Full Release (contracts → images → deploy) +``` + +--- + +## 🎓 Best Practices + +1. **Feature Branches** + - Always create feature branches for new work + - Let CI validate before merging to main + +2. **Contracts Changes** + - Use `contracts_only:` for quick contract updates + - Other services can update references immediately + +3. **Release Process** + - Tag only from main branch + - Use semantic versioning (v1.2.3) + - Tags starting with `v` auto-deploy to k8s + +4. **Emergency Rollback** + ```bash + # Quick rollback via deploy-only + git commit --allow-empty -m "deploy_only: Rollback to v1.2.8" + git tag v1.2.8-rollback + git push origin v1.2.8-rollback + + # Or use rollback script directly on the cluster + kubectl exec -it deploy-pod -- bash + cd /flea/_deploy/scripts + ./rollback.sh all + ``` + +5. **Monitoring Deployments** + - Watch Drone UI for pipeline progress + - Check pod logs: `kubectl logs -f deployment/telegram-listener -n nocr` + - Run health checks: `./_deploy/scripts/health-check.sh` + +--- + +## 🐛 Troubleshooting + +### Pipeline stuck on "Waiting for contracts" +**Cause:** Contract publish failed +**Solution:** Check NuGet feed, verify API key + +### Docker build fails with "unauthorized" +**Cause:** Invalid registry credentials +**Solution:** Update `hub_username` and `hub_password` secrets + +### Tests fail with "Cannot connect to Docker daemon" +**Cause:** Testcontainers can't reach Docker-in-Docker service +**Solution:** Check `DOCKER_HOST` environment variable is set correctly + +### Deployment fails with "ImagePullBackOff" +**Cause:** Image not found in registry +**Solution:** Verify image was built and pushed successfully in previous step + +--- + +## 📚 Additional Resources + +- [Drone CI Documentation](https://docs.drone.io/) +- [Kaniko Documentation](https://github.com/GoogleContainerTools/kaniko) +- [Testcontainers for .NET](https://dotnet.testcontainers.org/) +- [Kubernetes Deployments](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) diff --git a/_deploy/scripts/deploy.sh b/_deploy/scripts/deploy.sh new file mode 100755 index 0000000..dcccd9d --- /dev/null +++ b/_deploy/scripts/deploy.sh @@ -0,0 +1,92 @@ +#!/bin/bash +set -e + +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🚀 Nocr Services Deployment Script +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# Usage: ./deploy.sh +# Example: ./deploy.sh v1.2.3 abc1234 +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +TAG=${1:-latest} +COMMIT_SHA=${2:-latest} +NAMESPACE="nocr" +DEPLOYMENT_FILE="../k8s/deployment.yaml" + +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "🚀 Starting deployment of Nocr services" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "📦 Tag: $TAG" +echo "📝 Commit SHA: $COMMIT_SHA" +echo "🎯 Namespace: $NAMESPACE" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# Check if kubectl is available +if ! command -v kubectl &> /dev/null; then + echo "❌ kubectl not found. Please install kubectl." + exit 1 +fi + +# Check cluster connection +echo "🔍 Checking connection to Kubernetes cluster..." +if ! kubectl cluster-info &> /dev/null; then + echo "❌ Cannot connect to Kubernetes cluster." + exit 1 +fi +echo "✅ Connected to cluster" + +# Create temporary deployment file with updated image tags +TEMP_DEPLOYMENT=$(mktemp) +cp "$DEPLOYMENT_FILE" "$TEMP_DEPLOYMENT" + +echo "🔧 Updating image tags in deployment manifests..." + +# Update image tags for all services +sed -i "s|hub.musk.fun/k8s/nocr/telegram_listener:.*|hub.musk.fun/k8s/nocr/telegram_listener:${COMMIT_SHA}|g" "$TEMP_DEPLOYMENT" +sed -i "s|hub.musk.fun/k8s/nocr/telegram_client:.*|hub.musk.fun/k8s/nocr/telegram_client:${COMMIT_SHA}|g" "$TEMP_DEPLOYMENT" +sed -i "s|hub.musk.fun/k8s/nocr/text_matcher:.*|hub.musk.fun/k8s/nocr/text_matcher:${COMMIT_SHA}|g" "$TEMP_DEPLOYMENT" +sed -i "s|hub.musk.fun/k8s/nocr/text_matcher_migrator:.*|hub.musk.fun/k8s/nocr/text_matcher_migrator:${COMMIT_SHA}|g" "$TEMP_DEPLOYMENT" +sed -i "s|hub.musk.fun/k8s/nocr/users:.*|hub.musk.fun/k8s/nocr/users:${COMMIT_SHA}|g" "$TEMP_DEPLOYMENT" +sed -i "s|hub.musk.fun/k8s/nocr/users_migrator:.*|hub.musk.fun/k8s/nocr/users_migrator:${COMMIT_SHA}|g" "$TEMP_DEPLOYMENT" + +echo "✅ Image tags updated" + +# Apply deployments +echo "📦 Applying deployment manifests to cluster..." +kubectl apply -f "$TEMP_DEPLOYMENT" -n "$NAMESPACE" + +# Clean up temp file +rm "$TEMP_DEPLOYMENT" + +echo "✅ Manifests applied" +echo "" +echo "⏳ Waiting for rollouts to complete..." +echo "" + +# Wait for each deployment to roll out +DEPLOYMENTS=("telegram-listener" "text-matcher" "users" "telegram-client") +TIMEOUT="300s" + +for deployment in "${DEPLOYMENTS[@]}"; do + echo "🔄 Rolling out $deployment..." + if kubectl rollout status deployment/"$deployment" -n "$NAMESPACE" --timeout="$TIMEOUT"; then + echo "✅ $deployment rolled out successfully" + else + echo "❌ $deployment rollout failed or timed out" + echo "🔍 Pod status:" + kubectl get pods -n "$NAMESPACE" -l app="$deployment" + echo "🔍 Recent events:" + kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' | grep "$deployment" | tail -10 + exit 1 + fi + echo "" +done + +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "✅ Deployment completed successfully!" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "📊 Pod status:" +kubectl get pods -n "$NAMESPACE" -o wide +echo "" +echo "🔍 Running health checks..." +bash "$(dirname "$0")/health-check.sh" diff --git a/_deploy/scripts/health-check.sh b/_deploy/scripts/health-check.sh new file mode 100755 index 0000000..4eb1872 --- /dev/null +++ b/_deploy/scripts/health-check.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# 🏥 Nocr Services Health Check Script +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# Checks health endpoints and pod status for all Nocr services +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +NAMESPACE="nocr" +FAILED_CHECKS=0 + +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "🏥 Running health checks for Nocr services" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# Check if kubectl is available +if ! command -v kubectl &> /dev/null; then + echo "❌ kubectl not found. Please install kubectl." + exit 1 +fi + +# Function to check pod health +check_pod_health() { + local deployment=$1 + local service_name=$2 + echo "" + echo "🔍 Checking $service_name..." + + # Get pod status + PODS=$(kubectl get pods -n "$NAMESPACE" -l app="$deployment" -o json) + + # Check if any pods exist + POD_COUNT=$(echo "$PODS" | jq -r '.items | length') + if [ "$POD_COUNT" -eq 0 ]; then + echo "❌ No pods found for $service_name" + FAILED_CHECKS=$((FAILED_CHECKS + 1)) + return 1 + fi + + # Check pod status + RUNNING_PODS=$(echo "$PODS" | jq -r '.items[] | select(.status.phase=="Running") | .metadata.name' | wc -l) + READY_PODS=$(echo "$PODS" | jq -r '.items[] | select(.status.conditions[] | select(.type=="Ready" and .status=="True")) | .metadata.name' | wc -l) + + echo " 📊 Pods: $RUNNING_PODS running, $READY_PODS ready (total: $POD_COUNT)" + + if [ "$RUNNING_PODS" -eq 0 ]; then + echo " ❌ No running pods for $service_name" + kubectl get pods -n "$NAMESPACE" -l app="$deployment" + FAILED_CHECKS=$((FAILED_CHECKS + 1)) + return 1 + fi + + if [ "$READY_PODS" -eq 0 ]; then + echo " ❌ No ready pods for $service_name" + kubectl get pods -n "$NAMESPACE" -l app="$deployment" + echo " 🔍 Pod details:" + kubectl describe pods -n "$NAMESPACE" -l app="$deployment" | grep -A 20 "Conditions:" + FAILED_CHECKS=$((FAILED_CHECKS + 1)) + return 1 + fi + + # Try to check health endpoint if service has one + case "$deployment" in + "telegram-listener"|"text-matcher"|"users"|"telegram-client") + POD_NAME=$(echo "$PODS" | jq -r '.items[0].metadata.name') + if [ -n "$POD_NAME" ]; then + echo " 🌐 Checking /health endpoint..." + if kubectl exec -n "$NAMESPACE" "$POD_NAME" -- curl -f -s http://localhost:8080/health > /dev/null 2>&1; then + echo " ✅ Health endpoint responding" + else + echo " ⚠️ Health endpoint not responding (might be warming up)" + fi + fi + ;; + esac + + echo " ✅ $service_name is healthy" + return 0 +} + +# Check each service +SERVICES=( + "telegram-listener:Telegram Listener" + "text-matcher:Text Matcher" + "users:Users Service" + "telegram-client:Telegram Client" +) + +for service_info in "${SERVICES[@]}"; do + IFS=':' read -r deployment service_name <<< "$service_info" + check_pod_health "$deployment" "$service_name" +done + +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "📊 Summary:" +kubectl get pods -n "$NAMESPACE" -o wide +echo "" + +if [ $FAILED_CHECKS -eq 0 ]; then + echo "✅ All health checks passed!" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 0 +else + echo "❌ $FAILED_CHECKS health check(s) failed" + echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + exit 1 +fi diff --git a/_deploy/scripts/rollback.sh b/_deploy/scripts/rollback.sh new file mode 100755 index 0000000..ee4b54d --- /dev/null +++ b/_deploy/scripts/rollback.sh @@ -0,0 +1,105 @@ +#!/bin/bash +set -e + +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# ⏮️ Nocr Services Rollback Script +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ +# Usage: ./rollback.sh [deployment-name] +# Example: ./rollback.sh telegram-listener +# ./rollback.sh all (rolls back all deployments) +# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + +DEPLOYMENT=${1:-all} +NAMESPACE="nocr" + +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "⏮️ Starting rollback of Nocr services" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "🎯 Deployment: $DEPLOYMENT" +echo "🎯 Namespace: $NAMESPACE" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" + +# Check if kubectl is available +if ! command -v kubectl &> /dev/null; then + echo "❌ kubectl not found. Please install kubectl." + exit 1 +fi + +# Check cluster connection +echo "🔍 Checking connection to Kubernetes cluster..." +if ! kubectl cluster-info &> /dev/null; then + echo "❌ Cannot connect to Kubernetes cluster." + exit 1 +fi +echo "✅ Connected to cluster" + +# Function to rollback a single deployment +rollback_deployment() { + local dep=$1 + echo "" + echo "⏮️ Rolling back deployment: $dep" + + # Check if deployment exists + if ! kubectl get deployment "$dep" -n "$NAMESPACE" &> /dev/null; then + echo "⚠️ Deployment $dep not found in namespace $NAMESPACE" + return 1 + fi + + # Show current revision + echo "📊 Current revision:" + kubectl rollout history deployment/"$dep" -n "$NAMESPACE" | tail -5 + + # Perform rollback + echo "🔄 Rolling back..." + if kubectl rollout undo deployment/"$dep" -n "$NAMESPACE"; then + echo "✅ Rollback command issued for $dep" + + # Wait for rollback to complete + echo "⏳ Waiting for rollback to complete..." + if kubectl rollout status deployment/"$dep" -n "$NAMESPACE" --timeout=300s; then + echo "✅ $dep rolled back successfully" + else + echo "❌ $dep rollback failed or timed out" + echo "🔍 Pod status:" + kubectl get pods -n "$NAMESPACE" -l app="$dep" + return 1 + fi + else + echo "❌ Failed to rollback $dep" + return 1 + fi +} + +# Rollback deployments +DEPLOYMENTS=("telegram-listener" "text-matcher" "users" "telegram-client") + +if [ "$DEPLOYMENT" = "all" ]; then + echo "🔄 Rolling back all deployments..." + FAILED=0 + for dep in "${DEPLOYMENTS[@]}"; do + if ! rollback_deployment "$dep"; then + FAILED=$((FAILED + 1)) + fi + done + + if [ $FAILED -gt 0 ]; then + echo "" + echo "❌ $FAILED deployment(s) failed to rollback" + exit 1 + fi +else + # Rollback single deployment + if ! rollback_deployment "$DEPLOYMENT"; then + exit 1 + fi +fi + +echo "" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "✅ Rollback completed successfully!" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "📊 Pod status:" +kubectl get pods -n "$NAMESPACE" -o wide +echo "" +echo "🔍 Running health checks..." +bash "$(dirname "$0")/health-check.sh"