Secrets Management for Cloud-Native Applications

# ❌ Hardcoded in source code — ends up in Git history forever
DATABASE_URL="postgres://admin:s3cret@db.example.com:5432/mydb"
 
# ❌ Environment variables in docker-compose — committed to repo
services:
  api:
    environment:
      - DB_PASSWORD=s3cret
      - STRIPE_KEY=sk_live_abc123
 
# ❌ .env file "gitignored" — but still on developer laptops
# One stolen laptop = all production credentials compromised
 
# ❌ CI/CD pipeline variables — accessible to anyone with repo access
# Screenshot-able, often logged in plain text during builds

# ✅ Application fetches secrets at runtime from a secure store
# No secrets in code, config files, or environment variable definitions
# Secrets are encrypted at rest, access is audited, rotation is automated

# Store a secret in Vault
vault kv put secret/myapp/database \
  url="postgres://user:pass@db.example.com:5432/mydb" \
  password="supersecret123"
 
# Read a secret from Vault
vault kv get secret/myapp/database
# Key         Value
# ---         -----
# url         postgres://user:pass@db.example.com:5432/mydb
# password    supersecret123

// Application fetches secrets from Vault at startup
import Vault from 'node-vault';
 
async function loadSecrets(): Promise<AppSecrets> {
  const vault = Vault({
    endpoint: process.env.VAULT_ADDR,
    token: process.env.VAULT_TOKEN,  // App token, not human token
  });
 
  const dbSecret = await vault.read('secret/data/myapp/database');
  const stripeSecret = await vault.read('secret/data/myapp/stripe');
 
  return {
    databaseUrl: dbSecret.data.data.url,
    stripeKey: stripeSecret.data.data.key,
  };
}
 
// Initialize secrets before starting the server
const secrets = await loadSecrets();
const pool = new Pool({ connectionString: secrets.databaseUrl });

# Vault policy — restrict access per application
path "secret/data/myapp/*" {
  capabilities = ["read"]
}
 
# Deny access to other applications' secrets
path "secret/data/other-app/*" {
  capabilities = ["deny"]
}

# Configure Vault to generate temporary database credentials
resource "vault_database_secret_backend_connection" "postgres" {
  backend       = "database"
  name          = "mydb"
  allowed_roles = ["api-role"]
 
  postgresql {
    connection_url = "postgres://vault_admin:admin_pass@db:5432/mydb"
  }
}
 
resource "vault_database_secret_backend_role" "api" {
  backend = "database"
  name    = "api-role"
  db_name = "mydb"
 
  creation_statements = [
    "CREATE ROLE \"{{name}}\" WITH LOGIN PASSWORD '{{password}}' VALID UNTIL '{{expiration}}';",
    "GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO \"{{name}}\";"
  ]
 
  default_ttl = "1h"
  max_ttl     = "24h"
}

// Application requests temporary database credentials
async function getDatabaseCredentials(): Promise<DatabaseCreds> {
  const vault = Vault({
    endpoint: process.env.VAULT_ADDR,
    token: process.env.VAULT_TOKEN,
  });
 
  const creds = await vault.read('database/creds/api-role');
 
  return {
    username: creds.data.username,  // e.g., "v-token-api-role-abc123"
    password: creds.data.password,  // randomly generated
    ttl: creds.lease_duration,      // 3600 seconds
    leaseId: creds.lease_id,        // for renewal
  };
 
  // Credentials auto-expire after 1 hour
  // Vault deletes the database role automatically
  // No credential to leak, rotate, or clean up
}

# ❌ Kubernetes Secret — base64, not encrypted
apiVersion: v1
kind: Secret
metadata:
  name: db-credentials
type: Opaque
data:
  password: c3VwZXJzZWNyZXQ=  # base64 of "supersecret" — trivially decoded

# ✅ External Secrets Operator — syncs from Vault/AWS/GCP
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: db-credentials
spec:
  refreshInterval: 1h
  secretStoreRef:
    name: vault-backend
    kind: ClusterSecretStore
  target:
    name: db-credentials
    creationPolicy: Owner
  data:
    - secretKey: password
      remoteRef:
        key: secret/data/myapp/database
        property: password
    - secretKey: url
      remoteRef:
        key: secret/data/myapp/database
        property: url

# SecretStore configuration — connects to Vault
apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore
metadata:
  name: vault-backend
spec:
  provider:
    vault:
      server: "https://vault.internal:8200"
      path: "secret"
      auth:
        kubernetes:
          mountPath: "kubernetes"
          role: "myapp"

// Pattern: dual-credential rotation (zero downtime)
// Step 1: Generate new credential alongside old one
// Step 2: Update application to accept both
// Step 3: Roll out new credential to all instances
// Step 4: Revoke old credential
 
async function connectWithFallback(
  primaryUrl: string,
  fallbackUrl: string
): Promise<Pool> {
  try {
    const pool = new Pool({ connectionString: primaryUrl });
    await pool.query('SELECT 1');  // Verify connection works
    return pool;
  } catch {
    // Primary credential might be mid-rotation
    const pool = new Pool({ connectionString: fallbackUrl });
    await pool.query('SELECT 1');
    return pool;
  }
}

# AWS Secrets Manager rotation with Lambda
aws secretsmanager rotate-secret \
  --secret-id myapp/database \
  --rotation-lambda-arn arn:aws:lambda:us-east-1:123:function:rotate-db \
  --rotation-rules '{"AutomaticallyAfterDays": 30}'
 
# The Lambda function:
# 1. Creates a new password
# 2. Sets it on the database
# 3. Stores both old and new in Secrets Manager
# 4. Marks new as AWSCURRENT, old as AWSPREVIOUS
# 5. Applications using AWSCURRENT get the new password

# Vault audit log configuration
vault audit enable file file_path=/var/log/vault/audit.log

// Vault audit log entry
{
  "time": "2021-05-26T14:30:00Z",
  "type": "response",
  "auth": {
    "token_type": "service",
    "policies": ["myapp-policy"],
    "metadata": {
      "role": "myapp",
      "service_account_name": "myapp-api"
    }
  },
  "request": {
    "path": "secret/data/myapp/database",
    "operation": "read",
    "remote_address": "10.0.1.15"
  }
}

# Alert on unusual access patterns
# Prometheus alert for secrets access anomalies
groups:
  - name: secrets-audit
    rules:
      - alert: UnusualSecretAccess
        expr: |
          rate(vault_audit_log_request_total{
            path=~"secret/data/.*"
          }[5m]) > 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High rate of secret reads — possible credential scraping"