How We Run Next.js at Scale on K3s with Zero Downtime

Developer pushes to main branch
    |
    v
CI/CD pipeline triggers (GitLab CI / GitHub Actions)
    |
    v
Docker build + push to Container Registry (GitLab Registry / ECR / GHCR)
    |
    v
kubectl set image → rolling update on K3s
    |
    v
Cloudflare cache purge (automatic)
    |
    v
Traffic: Cloudflare CDN → Tunnel → K3s Service → Next.js pods

stages:
  - build
  - deploy

variables:
  IMAGE: $CI_REGISTRY_IMAGE/app
  TAG: $CI_COMMIT_SHORT_SHA

build:
  stage: build
  image: docker:24
  services:
    - docker:24-dind
  script:
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker build
        --cache-from $IMAGE:latest
        -t $IMAGE:$TAG
        -t $IMAGE:latest
        -f Dockerfile .
    - docker push $IMAGE:$TAG
    - docker push $IMAGE:latest
  only:
    - main

deploy:
  stage: deploy
  image: bitnami/kubectl:latest
  script:
    - kubectl set image deployment/myapp app=$IMAGE:$TAG -n myapp
    - kubectl rollout status deployment/myapp -n myapp --timeout=300s
    - |
      curl -s -X POST \
        "https://api.cloudflare.com/client/v4/zones/$CF_ZONE_ID/purge_cache" \
        -H "Authorization: Bearer $CF_API_TOKEN" \
        -H "Content-Type: application/json" \
        --data '{"purge_everything":true}'
  only:
    - main

name: Deploy
on:
  push:
    branches: [main]

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - uses: docker/build-push-action@v5
        with:
          push: true
          tags: ghcr.io/${{ github.repository }}/app:${{ github.sha }}
      - uses: azure/k8s-set-context@v3
        with:
          kubeconfig: ${{ secrets.KUBECONFIG }}
      - run: |
          kubectl set image deployment/myapp \
            app=ghcr.io/${{ github.repository }}/app:${{ github.sha }} \
            -n myapp
          kubectl rollout status deployment/myapp -n myapp --timeout=300s

# Install K3s without the default ingress controller
curl -sfL https://get.k3s.io | sh -s - \
  --disable traefik \
  --disable servicelb \
  --write-kubeconfig-mode 644

apiVersion: apps/v1
kind: Deployment
metadata:
  name: myapp
  namespace: myapp
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  selector:
    matchLabels:
      app: myapp
  template:
    metadata:
      labels:
        app: myapp
    spec:
      terminationGracePeriodSeconds: 30
      containers:
        - name: myapp
          image: registry.example.com/myapp:latest
          ports:
            - containerPort: 3000
          env:
            - name: NODE_ENV
              value: "production"
            - name: NODE_OPTIONS
              value: "--max-old-space-size=384"
          resources:
            requests:
              cpu: 200m
              memory: 256Mi
            limits:
              cpu: "1"
              memory: 512Mi
          readinessProbe:
            httpGet:
              path: /api/health
              port: 3000
            initialDelaySeconds: 5
            periodSeconds: 10
          livenessProbe:
            httpGet:
              path: /api/health
              port: 3000
            initialDelaySeconds: 15
            periodSeconds: 30
          lifecycle:
            preStop:
              exec:
                command: ["/bin/sh", "-c", "sleep 5"]

apiVersion: apps/v1
kind: Deployment
metadata:
  name: cloudflared
  namespace: myapp
spec:
  replicas: 2
  selector:
    matchLabels:
      app: cloudflared
  template:
    metadata:
      labels:
        app: cloudflared
    spec:
      containers:
        - name: cloudflared
          image: cloudflare/cloudflared:latest
          args:
            - tunnel
            - --no-autoupdate
            - run
            - --token
            - $(TUNNEL_TOKEN)
          env:
            - name: TUNNEL_TOKEN
              valueFrom:
                secretKeyRef:
                  name: cf-tunnel-token
                  key: token
          resources:
            requests:
              cpu: 50m
              memory: 64Mi
            limits:
              cpu: 200m
              memory: 128Mi

curl -s -X POST \
  "https://api.cloudflare.com/client/v4/zones/$CF_ZONE_ID/purge_cache" \
  -H "Authorization: Bearer $CF_API_TOKEN" \
  -H "Content-Type: application/json" \
  --data '{"prefixes":["myapp.example.com/_next/"]}'

// app/api/metrics/route.ts
import { NextResponse } from "next/server";

let requestCount = 0;

export async function GET() {
  requestCount++;
  const memUsage = process.memoryUsage();
  const metrics = [
    `# HELP nodejs_heap_used_bytes Node.js heap used`,
    `# TYPE nodejs_heap_used_bytes gauge`,
    `nodejs_heap_used_bytes ${memUsage.heapUsed}`,
    `# HELP nodejs_heap_total_bytes Node.js heap total`,
    `# TYPE nodejs_heap_total_bytes gauge`,
    `nodejs_heap_total_bytes ${memUsage.heapTotal}`,
    `# HELP http_requests_total Total HTTP requests`,
    `# TYPE http_requests_total counter`,
    `http_requests_total ${requestCount}`,
  ].join("\n");

  return new NextResponse(metrics, {
    headers: { "Content-Type": "text/plain" },
  });
}

groups:
  - name: nextjs-alerts
    rules:
      - alert: HighMemoryUsage
        expr: container_memory_usage_bytes{container="myapp"} / container_spec_memory_limit_bytes{container="myapp"} > 0.85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Next.js pod memory above 85%"

      - alert: PodRestartLoop
        expr: increase(kube_pod_container_status_restarts_total{container="myapp"}[1h]) > 3
        for: 0m
        labels:
          severity: critical
        annotations:
          summary: "Next.js pod restarting frequently"

      - alert: HighErrorRate
        expr: rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m]) > 0.05
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Error rate above 5%"