DevOps – Rollback Plan
# cloudbuild.yaml with rollback step
steps:
# ... build and deploy steps ...
# Deploy new version
- name: 'gcr.io/cloud-builders/gcloud'
args:
- 'run'
- 'deploy'
- '$_SERVICE_NAME'
- '--image'
- '$_ARTIFACT_REGISTRY/$_IMAGE_NAME:$COMMIT_SHA'
- '--region'
- '$_REGION'
- '--no-traffic' # Deploy but don't send traffic yet!
id: 'Deploy New Revision'
# Gradually shift traffic (canary)
- name: 'gcr.io/cloud-builders/gcloud'
args:
- 'run'
- 'services'
- 'update-traffic'
- '$_SERVICE_NAME'
- '--to-revisions'
- '$COMMIT_SHA=10' # 10% traffic
- '--region'
- '$_REGION'
id: 'Canary (10%)'
# Health check - wait and monitor
- name: 'bash'
args:
- '-c'
- |
echo "Waiting 60 seconds for canary to stabilize..."
sleep 60
# Check error rate for new revision
REVISION=$(gcloud run services describe $_SERVICE_NAME – region $_REGION – format 'value(status.latestCreatedRevisionName)')
# Count errors in last minute
ERRORS=$(gcloud logging read "resource.type=cloud_run_revision AND resource.labels.revision_name=$REVISION AND severity=ERROR AND timestamp > -60s" – format='value(textPayload)' – limit=100 | wc -l)
if [ $ERRORS -gt 0 ]; then
echo "🔥 Errors detected in new revision! Rolling back..."
gcloud run services update-traffic $_SERVICE_NAME – to-revisions=$_PREVIOUS_REVISION=100 – region $_REGION
exit 1
else
echo "✅ Canary healthy! Proceeding to full rollout..."
fi
id: 'Health Check'
# Full rollout
- name: 'gcr.io/cloud-builders/gcloud'
args:
- 'run'
- 'services'
- 'update-traffic'
- '$_SERVICE_NAME'
- '--to-revisions'
- '$COMMIT_SHA=100'
- '--region'
- '$_REGION'
id: 'Full Rollout'
# 1️⃣ Identify the problem
gcloud logging read "resource.type=cloud_run_revision AND severity=ERROR AND timestamp > -5m" – limit=20
# 2️⃣ Check current revision and traffic
gcloud run services describe churn-model-api – region=us-central1
# 3️⃣ List all available revisions
gcloud run revisions list – service=churn-model-api – region=us-central1
# 4️⃣ Get the last known good revision name
GOOD_REVISION="churn-model-api-00001-abc" # From the list above
# 5️⃣ Rollback to that revision (immediate)
gcloud run services update-traffic churn-model-api \
– to-revisions=$GOOD_REVISION=100 \
– region=us-central1
# 6️⃣ Verify rollback worked
curl -X POST https://churn-model-api-abc123-uc.a.run.app/predict \
-H "Content-Type: application/json" \
-d '{"features": [1.2, 3.4, 5.6]}'
# 7️⃣ Notify the team
# Send Slack message, email, or page on-call engineer
