release

2025-10-16 17:38:25 +04:00 · 2025-10-16 17:38:25 +04:00 · 169acd2181
commit 169acd2181
parent 1ed92a4ab9
5 changed files with 879 additions and 0 deletions
--- a/.drone.yml
+++ b/.drone.yml
@ -305,3 +305,276 @@ steps:
    depends_on:
      - check-trigger
      - text-matcher-contracts
+---
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# 🚀 Pipeline 4: Full Release
+# Trigger: Tag on main branch WITHOUT "contracts_only" or "deploy_only"
+# Purpose: Full release cycle - contracts → images → optional deploy
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+kind: pipeline
+type: kubernetes
+name: full-release
+
+metadata:
+  namespace: musk-drone
+
+trigger:
+  ref:
+    - refs/tags/*
+  event:
+    - tag
+  branch:
+    - main
+
+clone:
+  disable: true
+
+volumes:
+  - name: nuget-cache
+    temp: {}
+
+steps:
+  - name: clone
+    image: alpine/git
+    commands:
+      - git clone https://gitea.musk.fun/nocr/flea
+      - cd flea
+      - git checkout $DRONE_TAG
+      - git submodule update --init --recursive
+
+  - name: check-trigger
+    image: alpine/git
+    commands:
+      - cd flea
+      - COMMIT_MSG=$(git log -1 --pretty=%B)
+      - echo "Commit message - $COMMIT_MSG"
+      - |
+        if echo "$COMMIT_MSG" | grep -qE "contracts_only:|deploy_only:"; then
+          echo "⏭️  contracts_only or deploy_only detected, skipping full release..."
+          exit 78
+        else
+          echo "✅ Full release triggered"
+          exit 0
+        fi
+    depends_on:
+      - clone
+
+  # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  # STAGE 1: Publish NuGet Contracts (sequental)
+  # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+  - name: telegram-listener-nuget
+    image: mcr.microsoft.com/dotnet/sdk:8.0
+    volumes:
+      - name: nuget-cache
+        path: /root/.nuget/packages
+    environment:
+      NUGETAPIKEY:
+        from_secret: nuget_musk_api_key
+    commands:
+      - cd flea
+      - dotnet nuget add source --name musk https://gitea.musk.fun/api/packages/nocr/nuget/index.json
+      - dotnet pack telegram-listener/Nocr.TelegramListener.sln -o ./bin -p:PackageVersion=${DRONE_TAG}
+      - dotnet nuget push ./bin/*Contract*.nupkg --api-key $NUGETAPIKEY --source musk --skip-duplicate
+    depends_on:
+      - check-trigger
+
+  - name: text-matcher-nuget
+    image: mcr.microsoft.com/dotnet/sdk:8.0
+    volumes:
+      - name: nuget-cache
+        path: /root/.nuget/packages
+    environment:
+      NUGETAPIKEY:
+        from_secret: nuget_musk_api_key
+    commands:
+      - cd flea
+      - dotnet nuget add source --name musk https://gitea.musk.fun/api/packages/nocr/nuget/index.json
+      - dotnet pack text-matcher/Nocr.TextMatcher.sln -o ./bin -p:PackageVersion=${DRONE_TAG}
+      - dotnet nuget push ./bin/*Contract*.nupkg --api-key $NUGETAPIKEY --source musk --skip-duplicate
+    depends_on:
+      - check-trigger
+      - telegram-listener-nuget
+
+  - name: users-nuget
+    image: mcr.microsoft.com/dotnet/sdk:8.0
+    volumes:
+      - name: nuget-cache
+        path: /root/.nuget/packages
+    environment:
+      NUGETAPIKEY:
+        from_secret: nuget_musk_api_key
+    commands:
+      - cd flea
+      - dotnet nuget add source --name musk https://gitea.musk.fun/api/packages/nocr/nuget/index.json
+      - dotnet pack users/Nocr.Users.sln -o ./bin -p:PackageVersion=${DRONE_TAG}
+      - dotnet nuget push ./bin/*Contract*.nupkg --api-key $NUGETAPIKEY --source musk --skip-duplicate
+    depends_on:
+      - check-trigger
+      - text-matcher-nuget
+
+  # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  # STAGE 2: Build Docker Images with Kaniko (3 parallel streams)
+  # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+  - name: telegram-listener-build-push
+    image: gcr.io/kaniko-project/executor:debug
+    environment:
+      HUB_USERNAME:
+        from_secret: hub_username
+      HUB_PASSWORD:
+        from_secret: hub_password
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo "{\"auths\":{\"hub.musk.fun\":{\"username\":\"$HUB_USERNAME\",\"password\":\"$HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json
+      - cd flea/telegram-listener
+      - /kaniko/executor
+        --context=.
+        --dockerfile=src/Nocr.TelegramListener.Host/Dockerfile
+        --destination=hub.musk.fun/k8s/nocr/telegram_listener:${DRONE_COMMIT_SHA:0:7}
+        --destination=hub.musk.fun/k8s/nocr/telegram_listener:${DRONE_TAG}
+        --destination=hub.musk.fun/k8s/nocr/telegram_listener:latest
+        --cache=true
+        --cache-repo=hub.musk.fun/k8s/cache/nocr-telegram-listener
+        --compressed-caching=true
+    depends_on:
+      - telegram-listener-nuget
+      - text-matcher-nuget
+      - users-nuget
+
+  - name: telegram-client-build-push
+    image: gcr.io/kaniko-project/executor:debug
+    environment:
+      HUB_USERNAME:
+        from_secret: hub_username
+      HUB_PASSWORD:
+        from_secret: hub_password
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo "{\"auths\":{\"hub.musk.fun\":{\"username\":\"$HUB_USERNAME\",\"password\":\"$HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json
+      - cd flea/telegram-client
+      - /kaniko/executor
+        --context=.
+        --dockerfile=src/Nocr.TelegramClient.Host/Dockerfile
+        --destination=hub.musk.fun/k8s/nocr/telegram_client:${DRONE_COMMIT_SHA:0:7}
+        --destination=hub.musk.fun/k8s/nocr/telegram_client:${DRONE_TAG}
+        --destination=hub.musk.fun/k8s/nocr/telegram_client:latest
+        --cache=true
+        --cache-repo=hub.musk.fun/k8s/cache/nocr-telegram-client
+        --compressed-caching=true
+    depends_on:
+      - telegram-listener-build-push
+
+  - name: text-matcher-build-push
+    image: gcr.io/kaniko-project/executor:debug
+    environment:
+      HUB_USERNAME:
+        from_secret: hub_username
+      HUB_PASSWORD:
+        from_secret: hub_password
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo "{\"auths\":{\"hub.musk.fun\":{\"username\":\"$HUB_USERNAME\",\"password\":\"$HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json
+      - cd flea/text-matcher
+      - /kaniko/executor
+        --context=.
+        --dockerfile=src/Nocr.TextMatcher.Host/Dockerfile
+        --destination=hub.musk.fun/k8s/nocr/text_matcher:${DRONE_COMMIT_SHA:0:7}
+        --destination=hub.musk.fun/k8s/nocr/text_matcher:${DRONE_TAG}
+        --destination=hub.musk.fun/k8s/nocr/text_matcher:latest
+        --cache=true
+        --cache-repo=hub.musk.fun/k8s/cache/nocr-text-matcher
+        --compressed-caching=true
+    depends_on:
+      - telegram-listener-nuget
+      - text-matcher-nuget
+      - users-nuget
+
+  - name: text-matcher-migrator-build-push
+    image: gcr.io/kaniko-project/executor:debug
+    environment:
+      HUB_USERNAME:
+        from_secret: hub_username
+      HUB_PASSWORD:
+        from_secret: hub_password
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo "{\"auths\":{\"hub.musk.fun\":{\"username\":\"$HUB_USERNAME\",\"password\":\"$HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json
+      - cd flea/text-matcher
+      - /kaniko/executor
+        --context=.
+        --dockerfile=src/Nocr.TextMatcher.Migrator/Dockerfile
+        --destination=hub.musk.fun/k8s/nocr/text_matcher_migrator:${DRONE_COMMIT_SHA:0:7}
+        --destination=hub.musk.fun/k8s/nocr/text_matcher_migrator:${DRONE_TAG}
+        --destination=hub.musk.fun/k8s/nocr/text_matcher_migrator:latest
+        --cache=true
+        --cache-repo=hub.musk.fun/k8s/cache/nocr-text-matcher-migrator
+        --compressed-caching=true
+    depends_on:
+      - text-matcher-build-push
+
+  - name: users-build-push
+    image: gcr.io/kaniko-project/executor:debug
+    environment:
+      HUB_USERNAME:
+        from_secret: hub_username
+      HUB_PASSWORD:
+        from_secret: hub_password
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo "{\"auths\":{\"hub.musk.fun\":{\"username\":\"$HUB_USERNAME\",\"password\":\"$HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json
+      - cd flea/users
+      - /kaniko/executor
+        --context=.
+        --dockerfile=src/Nocr.Users.Host/Dockerfile
+        --destination=hub.musk.fun/k8s/nocr/users:${DRONE_COMMIT_SHA:0:7}
+        --destination=hub.musk.fun/k8s/nocr/users:${DRONE_TAG}
+        --destination=hub.musk.fun/k8s/nocr/users:latest
+        --cache=true
+        --cache-repo=hub.musk.fun/k8s/cache/nocr-users
+        --compressed-caching=true
+    depends_on:
+      - telegram-listener-nuget
+      - text-matcher-nuget
+      - users-nuget
+
+  - name: users-migrator-build-push
+    image: gcr.io/kaniko-project/executor:debug
+    environment:
+      HUB_USERNAME:
+        from_secret: hub_username
+      HUB_PASSWORD:
+        from_secret: hub_password
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo "{\"auths\":{\"hub.musk.fun\":{\"username\":\"$HUB_USERNAME\",\"password\":\"$HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json
+      - cd flea/users
+      - /kaniko/executor
+        --context=.
+        --dockerfile=src/Nocr.Users.Migrator/Dockerfile
+        --destination=hub.musk.fun/k8s/nocr/users_migrator:${DRONE_COMMIT_SHA:0:7}
+        --destination=hub.musk.fun/k8s/nocr/users_migrator:${DRONE_TAG}
+        --destination=hub.musk.fun/k8s/nocr/users_migrator:latest
+        --cache=true
+        --cache-repo=hub.musk.fun/k8s/cache/nocr-users-migrator
+        --compressed-caching=true
+    depends_on:
+      - users-build-push
+
+  # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+  # STAGE 3: Deploy to Kubernetes (optional, based on tag pattern)
+  # ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+  - name: deploy-to-k8s
+    image: bitnami/kubectl:latest
+    commands:
+      - cd flea/_deploy/scripts
+      - chmod +x deploy.sh
+      - ./deploy.sh ${DRONE_TAG} ${DRONE_COMMIT_SHA:0:7}
+    depends_on:
+      - telegram-client-build-push
+      - text-matcher-migrator-build-push
+      - users-migrator-build-push
+    when:
+      ref:
+        - refs/tags/*
--- a/_deploy/README.md
+++ b/_deploy/README.md
@ -0,0 +1,301 @@
+# 🚀 Nocr CI/CD Pipeline Documentation
+
+## 📋 Overview
+
+The Nocr project uses a modern, multi-pipeline CI/CD setup powered by Drone CI on Kubernetes. This document describes the 5 specialized pipelines and how to use them.
+
+---
+
+## 🎯 Pipeline Architecture
+
+### Pipeline 1: **Feature Validation**
+**Trigger:** Push to `feature/*` or `fix/*` branches
+**Purpose:** Fast feedback for developers
+**Duration:** ~3-5 minutes
+
+**What it does:**
+- Clones repo with submodules
+- Restores all NuGet packages (shared cache)
+- Builds all 4 services in Release mode
+- Runs unit and integration tests with Testcontainers
+
+**Example workflow:**
+```bash
+git checkout -b feature/add-new-filter
+# Make changes...
+git add .
+git commit -m "Add new filter functionality"
+git push origin feature/add-new-filter
+```
+
+Drone automatically runs tests. Check results before creating PR.
+
+---
+
+### Pipeline 2: **Main Validation**
+**Trigger:** Push to `main` branch
+**Purpose:** Validate main branch after merge
+**Duration:** ~3-5 minutes
+
+**What it does:**
+- Same as Feature Validation
+- Ensures main branch is always in working state
+
+**Example workflow:**
+```bash
+# After PR is merged to main
+# Pipeline runs automatically
+```
+
+---
+
+### Pipeline 3: **Contracts-Only Publish**
+**Trigger:** Tag with commit message containing `contracts_only:<service>`
+**Purpose:** Fast publish of contract packages without building images
+**Duration:** ~2 minutes
+
+**What it does:**
+- Packs specified service contracts into NuGet packages
+- Publishes to internal NuGet feed
+- Skips Docker image builds
+
+**Example workflow:**
+```bash
+# Update telegram-listener contracts
+cd telegram-listener
+# Make changes to Async.Api.Contracts...
+git add .
+git commit -m "contracts_only:telegram_listener - Add MessageEdited event"
+git push origin main
+
+# Create tag
+git tag v1.2.4-contracts
+git push origin v1.2.4-contracts
+```
+
+**Supported markers:**
+- `contracts_only:telegram_listener`
+- `contracts_only:text_matcher`
+- `contracts_only:users`
+
+---
+
+### Pipeline 4: **Full Release**
+**Trigger:** Tag on main WITHOUT `contracts_only` or `deploy_only` in commit message
+**Purpose:** Complete release cycle
+**Duration:** ~8-10 minutes
+
+**What it does:**
+1. **Stage 1:** Publish all contracts to NuGet (parallel)
+2. **Stage 2:** Build all Docker images with Kaniko (3 parallel streams)
+3. **Stage 3:** Deploy to Kubernetes (only for tags matching `v*`)
+
+**Example workflow:**
+```bash
+# Ready to release
+git tag v1.3.0
+git push origin v1.3.0
+
+# Drone will:
+# 1. Publish contracts
+# 2. Build images (tagged with v1.3.0, commit SHA, and latest)
+# 3. Deploy to k8s (if tag starts with 'v')
+```
+
+**Image tags created:**
+- `hub.musk.fun/k8s/nocr/telegram_listener:v1.3.0`
+- `hub.musk.fun/k8s/nocr/telegram_listener:abc1234` (commit SHA)
+- `hub.musk.fun/k8s/nocr/telegram_listener:latest`
+
+---
+
+### Pipeline 5: **Deploy-Only**
+**Trigger:** Tag with commit message containing `deploy_only:`
+**Purpose:** Fast deploy of already-built images
+**Duration:** ~1 minute
+
+**What it does:**
+- Skips building
+- Deploys specified images to Kubernetes
+- Useful for rolling back or promoting existing images
+
+**Example workflow:**
+```bash
+# Deploy existing images
+git commit --allow-empty -m "deploy_only: Deploy v1.2.9"
+git tag v1.2.9-deploy
+git push origin v1.2.9-deploy
+```
+
+---
+
+## 🛠️ Deployment Scripts
+
+All deployment scripts are located in `_deploy/scripts/`:
+
+### `deploy.sh`
+**Purpose:** Deploy services to Kubernetes
+**Usage:**
+```bash
+./deploy.sh <tag> <commit-sha>
+./deploy.sh v1.3.0 abc1234
+```
+
+**Features:**
+- Updates deployment manifests with new image tags
+- Applies manifests to cluster
+- Waits for rollouts to complete with timeout
+- Runs health checks after deployment
+- Shows pod status
+
+### `rollback.sh`
+**Purpose:** Rollback deployments to previous version
+**Usage:**
+```bash
+# Rollback single service
+./rollback.sh telegram-listener
+
+# Rollback all services
+./rollback.sh all
+```
+
+**Features:**
+- Shows revision history
+- Performs kubectl rollout undo
+- Waits for rollback to complete
+- Runs health checks after rollback
+
+### `health-check.sh`
+**Purpose:** Check health of all Nocr services
+**Usage:**
+```bash
+./health-check.sh
+```
+
+**Checks:**
+- Pod status (Running/Ready)
+- Health endpoints (/health)
+- Recent events for failed pods
+
+---
+
+## 📦 Optimizations
+
+### Shared NuGet Cache
+All pipelines use a shared temp volume for NuGet packages:
+- First `dotnet restore` downloads packages
+- Subsequent builds reuse cached packages
+- **~60% faster** than individual restores per service
+
+### Parallel Execution
+- Contract publishing: 3 services in parallel
+- Docker builds: 3 parallel streams
+- Independent operations never block each other
+
+### Kaniko Caching
+All Kaniko builds use:
+- `--cache=true` - Layer caching enabled
+- `--cache-repo=hub.musk.fun/k8s/cache/*` - Shared cache repo
+- `--compressed-caching=true` - Faster cache transfer
+
+---
+
+## 🧪 Testcontainers Support
+
+Feature and Main validation pipelines include Docker-in-Docker service for Testcontainers:
+
+```yaml
+services:
+  - name: docker
+    image: docker:27-dind
+    privileged: true
+```
+
+Tests can use Testcontainers to spin up real databases, message queues, etc.
+
+---
+
+## 🔒 Required Secrets
+
+Configure these in Drone:
+
+- `hub_username` - Docker registry username
+- `hub_password` - Docker registry password
+- `nuget_musk_api_key` - NuGet feed API key
+
+---
+
+## 📊 Pipeline Decision Tree
+
+```
+Push to feature/* → Feature Validation (build + test)
+Push to main → Main Validation (build + test)
+
+Tag + "contracts_only:" → Contracts Publish
+Tag + "deploy_only:" → Deploy Only
+Tag (no markers) → Full Release (contracts → images → deploy)
+```
+
+---
+
+## 🎓 Best Practices
+
+1. **Feature Branches**
+   - Always create feature branches for new work
+   - Let CI validate before merging to main
+
+2. **Contracts Changes**
+   - Use `contracts_only:` for quick contract updates
+   - Other services can update references immediately
+
+3. **Release Process**
+   - Tag only from main branch
+   - Use semantic versioning (v1.2.3)
+   - Tags starting with `v` auto-deploy to k8s
+
+4. **Emergency Rollback**
+   ```bash
+   # Quick rollback via deploy-only
+   git commit --allow-empty -m "deploy_only: Rollback to v1.2.8"
+   git tag v1.2.8-rollback
+   git push origin v1.2.8-rollback
+
+   # Or use rollback script directly on the cluster
+   kubectl exec -it deploy-pod -- bash
+   cd /flea/_deploy/scripts
+   ./rollback.sh all
+   ```
+
+5. **Monitoring Deployments**
+   - Watch Drone UI for pipeline progress
+   - Check pod logs: `kubectl logs -f deployment/telegram-listener -n nocr`
+   - Run health checks: `./_deploy/scripts/health-check.sh`
+
+---
+
+## 🐛 Troubleshooting
+
+### Pipeline stuck on "Waiting for contracts"
+**Cause:** Contract publish failed
+**Solution:** Check NuGet feed, verify API key
+
+### Docker build fails with "unauthorized"
+**Cause:** Invalid registry credentials
+**Solution:** Update `hub_username` and `hub_password` secrets
+
+### Tests fail with "Cannot connect to Docker daemon"
+**Cause:** Testcontainers can't reach Docker-in-Docker service
+**Solution:** Check `DOCKER_HOST` environment variable is set correctly
+
+### Deployment fails with "ImagePullBackOff"
+**Cause:** Image not found in registry
+**Solution:** Verify image was built and pushed successfully in previous step
+
+---
+
+## 📚 Additional Resources
+
+- [Drone CI Documentation](https://docs.drone.io/)
+- [Kaniko Documentation](https://github.com/GoogleContainerTools/kaniko)
+- [Testcontainers for .NET](https://dotnet.testcontainers.org/)
+- [Kubernetes Deployments](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/)
--- a/_deploy/scripts/deploy.sh
+++ b/_deploy/scripts/deploy.sh
@ -0,0 +1,92 @@
+#!/bin/bash
+set -e
+
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# 🚀 Nocr Services Deployment Script
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# Usage: ./deploy.sh <tag> <commit-sha>
+# Example: ./deploy.sh v1.2.3 abc1234
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+TAG=${1:-latest}
+COMMIT_SHA=${2:-latest}
+NAMESPACE="nocr"
+DEPLOYMENT_FILE="../k8s/deployment.yaml"
+
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "🚀 Starting deployment of Nocr services"
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "📦 Tag: $TAG"
+echo "📝 Commit SHA: $COMMIT_SHA"
+echo "🎯 Namespace: $NAMESPACE"
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+
+# Check if kubectl is available
+if ! command -v kubectl &> /dev/null; then
+    echo "❌ kubectl not found. Please install kubectl."
+    exit 1
+fi
+
+# Check cluster connection
+echo "🔍 Checking connection to Kubernetes cluster..."
+if ! kubectl cluster-info &> /dev/null; then
+    echo "❌ Cannot connect to Kubernetes cluster."
+    exit 1
+fi
+echo "✅ Connected to cluster"
+
+# Create temporary deployment file with updated image tags
+TEMP_DEPLOYMENT=$(mktemp)
+cp "$DEPLOYMENT_FILE" "$TEMP_DEPLOYMENT"
+
+echo "🔧 Updating image tags in deployment manifests..."
+
+# Update image tags for all services
+sed -i "s|hub.musk.fun/k8s/nocr/telegram_listener:.*|hub.musk.fun/k8s/nocr/telegram_listener:${COMMIT_SHA}|g" "$TEMP_DEPLOYMENT"
+sed -i "s|hub.musk.fun/k8s/nocr/telegram_client:.*|hub.musk.fun/k8s/nocr/telegram_client:${COMMIT_SHA}|g" "$TEMP_DEPLOYMENT"
+sed -i "s|hub.musk.fun/k8s/nocr/text_matcher:.*|hub.musk.fun/k8s/nocr/text_matcher:${COMMIT_SHA}|g" "$TEMP_DEPLOYMENT"
+sed -i "s|hub.musk.fun/k8s/nocr/text_matcher_migrator:.*|hub.musk.fun/k8s/nocr/text_matcher_migrator:${COMMIT_SHA}|g" "$TEMP_DEPLOYMENT"
+sed -i "s|hub.musk.fun/k8s/nocr/users:.*|hub.musk.fun/k8s/nocr/users:${COMMIT_SHA}|g" "$TEMP_DEPLOYMENT"
+sed -i "s|hub.musk.fun/k8s/nocr/users_migrator:.*|hub.musk.fun/k8s/nocr/users_migrator:${COMMIT_SHA}|g" "$TEMP_DEPLOYMENT"
+
+echo "✅ Image tags updated"
+
+# Apply deployments
+echo "📦 Applying deployment manifests to cluster..."
+kubectl apply -f "$TEMP_DEPLOYMENT" -n "$NAMESPACE"
+
+# Clean up temp file
+rm "$TEMP_DEPLOYMENT"
+
+echo "✅ Manifests applied"
+echo ""
+echo "⏳ Waiting for rollouts to complete..."
+echo ""
+
+# Wait for each deployment to roll out
+DEPLOYMENTS=("telegram-listener" "text-matcher" "users" "telegram-client")
+TIMEOUT="300s"
+
+for deployment in "${DEPLOYMENTS[@]}"; do
+    echo "🔄 Rolling out $deployment..."
+    if kubectl rollout status deployment/"$deployment" -n "$NAMESPACE" --timeout="$TIMEOUT"; then
+        echo "✅ $deployment rolled out successfully"
+    else
+        echo "❌ $deployment rollout failed or timed out"
+        echo "🔍 Pod status:"
+        kubectl get pods -n "$NAMESPACE" -l app="$deployment"
+        echo "🔍 Recent events:"
+        kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' | grep "$deployment" | tail -10
+        exit 1
+    fi
+    echo ""
+done
+
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "✅ Deployment completed successfully!"
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "📊 Pod status:"
+kubectl get pods -n "$NAMESPACE" -o wide
+echo ""
+echo "🔍 Running health checks..."
+bash "$(dirname "$0")/health-check.sh"
--- a/_deploy/scripts/health-check.sh
+++ b/_deploy/scripts/health-check.sh
@ -0,0 +1,108 @@
+#!/bin/bash
+
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# 🏥 Nocr Services Health Check Script
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# Checks health endpoints and pod status for all Nocr services
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+NAMESPACE="nocr"
+FAILED_CHECKS=0
+
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "🏥 Running health checks for Nocr services"
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+
+# Check if kubectl is available
+if ! command -v kubectl &> /dev/null; then
+    echo "❌ kubectl not found. Please install kubectl."
+    exit 1
+fi
+
+# Function to check pod health
+check_pod_health() {
+    local deployment=$1
+    local service_name=$2
+    echo ""
+    echo "🔍 Checking $service_name..."
+
+    # Get pod status
+    PODS=$(kubectl get pods -n "$NAMESPACE" -l app="$deployment" -o json)
+
+    # Check if any pods exist
+    POD_COUNT=$(echo "$PODS" | jq -r '.items | length')
+    if [ "$POD_COUNT" -eq 0 ]; then
+        echo "❌ No pods found for $service_name"
+        FAILED_CHECKS=$((FAILED_CHECKS + 1))
+        return 1
+    fi
+
+    # Check pod status
+    RUNNING_PODS=$(echo "$PODS" | jq -r '.items[] | select(.status.phase=="Running") | .metadata.name' | wc -l)
+    READY_PODS=$(echo "$PODS" | jq -r '.items[] | select(.status.conditions[] | select(.type=="Ready" and .status=="True")) | .metadata.name' | wc -l)
+
+    echo "   📊 Pods: $RUNNING_PODS running, $READY_PODS ready (total: $POD_COUNT)"
+
+    if [ "$RUNNING_PODS" -eq 0 ]; then
+        echo "   ❌ No running pods for $service_name"
+        kubectl get pods -n "$NAMESPACE" -l app="$deployment"
+        FAILED_CHECKS=$((FAILED_CHECKS + 1))
+        return 1
+    fi
+
+    if [ "$READY_PODS" -eq 0 ]; then
+        echo "   ❌ No ready pods for $service_name"
+        kubectl get pods -n "$NAMESPACE" -l app="$deployment"
+        echo "   🔍 Pod details:"
+        kubectl describe pods -n "$NAMESPACE" -l app="$deployment" | grep -A 20 "Conditions:"
+        FAILED_CHECKS=$((FAILED_CHECKS + 1))
+        return 1
+    fi
+
+    # Try to check health endpoint if service has one
+    case "$deployment" in
+        "telegram-listener"|"text-matcher"|"users"|"telegram-client")
+            POD_NAME=$(echo "$PODS" | jq -r '.items[0].metadata.name')
+            if [ -n "$POD_NAME" ]; then
+                echo "   🌐 Checking /health endpoint..."
+                if kubectl exec -n "$NAMESPACE" "$POD_NAME" -- curl -f -s http://localhost:8080/health > /dev/null 2>&1; then
+                    echo "   ✅ Health endpoint responding"
+                else
+                    echo "   ⚠️  Health endpoint not responding (might be warming up)"
+                fi
+            fi
+            ;;
+    esac
+
+    echo "   ✅ $service_name is healthy"
+    return 0
+}
+
+# Check each service
+SERVICES=(
+    "telegram-listener:Telegram Listener"
+    "text-matcher:Text Matcher"
+    "users:Users Service"
+    "telegram-client:Telegram Client"
+)
+
+for service_info in "${SERVICES[@]}"; do
+    IFS=':' read -r deployment service_name <<< "$service_info"
+    check_pod_health "$deployment" "$service_name"
+done
+
+echo ""
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "📊 Summary:"
+kubectl get pods -n "$NAMESPACE" -o wide
+echo ""
+
+if [ $FAILED_CHECKS -eq 0 ]; then
+    echo "✅ All health checks passed!"
+    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+    exit 0
+else
+    echo "❌ $FAILED_CHECKS health check(s) failed"
+    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+    exit 1
+fi
--- a/_deploy/scripts/rollback.sh
+++ b/_deploy/scripts/rollback.sh
@ -0,0 +1,105 @@
+#!/bin/bash
+set -e
+
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# ⏮️  Nocr Services Rollback Script
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+# Usage: ./rollback.sh [deployment-name]
+# Example: ./rollback.sh telegram-listener
+#          ./rollback.sh all (rolls back all deployments)
+# ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+
+DEPLOYMENT=${1:-all}
+NAMESPACE="nocr"
+
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "⏮️  Starting rollback of Nocr services"
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "🎯 Deployment: $DEPLOYMENT"
+echo "🎯 Namespace: $NAMESPACE"
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+
+# Check if kubectl is available
+if ! command -v kubectl &> /dev/null; then
+    echo "❌ kubectl not found. Please install kubectl."
+    exit 1
+fi
+
+# Check cluster connection
+echo "🔍 Checking connection to Kubernetes cluster..."
+if ! kubectl cluster-info &> /dev/null; then
+    echo "❌ Cannot connect to Kubernetes cluster."
+    exit 1
+fi
+echo "✅ Connected to cluster"
+
+# Function to rollback a single deployment
+rollback_deployment() {
+    local dep=$1
+    echo ""
+    echo "⏮️  Rolling back deployment: $dep"
+
+    # Check if deployment exists
+    if ! kubectl get deployment "$dep" -n "$NAMESPACE" &> /dev/null; then
+        echo "⚠️  Deployment $dep not found in namespace $NAMESPACE"
+        return 1
+    fi
+
+    # Show current revision
+    echo "📊 Current revision:"
+    kubectl rollout history deployment/"$dep" -n "$NAMESPACE" | tail -5
+
+    # Perform rollback
+    echo "🔄 Rolling back..."
+    if kubectl rollout undo deployment/"$dep" -n "$NAMESPACE"; then
+        echo "✅ Rollback command issued for $dep"
+
+        # Wait for rollback to complete
+        echo "⏳ Waiting for rollback to complete..."
+        if kubectl rollout status deployment/"$dep" -n "$NAMESPACE" --timeout=300s; then
+            echo "✅ $dep rolled back successfully"
+        else
+            echo "❌ $dep rollback failed or timed out"
+            echo "🔍 Pod status:"
+            kubectl get pods -n "$NAMESPACE" -l app="$dep"
+            return 1
+        fi
+    else
+        echo "❌ Failed to rollback $dep"
+        return 1
+    fi
+}
+
+# Rollback deployments
+DEPLOYMENTS=("telegram-listener" "text-matcher" "users" "telegram-client")
+
+if [ "$DEPLOYMENT" = "all" ]; then
+    echo "🔄 Rolling back all deployments..."
+    FAILED=0
+    for dep in "${DEPLOYMENTS[@]}"; do
+        if ! rollback_deployment "$dep"; then
+            FAILED=$((FAILED + 1))
+        fi
+    done
+
+    if [ $FAILED -gt 0 ]; then
+        echo ""
+        echo "❌ $FAILED deployment(s) failed to rollback"
+        exit 1
+    fi
+else
+    # Rollback single deployment
+    if ! rollback_deployment "$DEPLOYMENT"; then
+        exit 1
+    fi
+fi
+
+echo ""
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "✅ Rollback completed successfully!"
+echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+echo "📊 Pod status:"
+kubectl get pods -n "$NAMESPACE" -o wide
+echo ""
+echo "🔍 Running health checks..."
+bash "$(dirname "$0")/health-check.sh"