Vault

Create a TLS secret used by Vault and Traefik to communicate with vault service

export CERT_DIR=cert
export SECRET_NAME=vault-server-tls

Generate key:

openssl genrsa -out ${CERT_DIR}/vault.key 4096

Generate server.csr:

openssl req -config ${CERT_DIR}/csr.conf -new -key ${CERT_DIR}/vault.key -subj "/CN=vault.vault.svc" -out ${CERT_DIR}/server.csr

Generate CertificateSigningRequest Create ${CERT_DIR}/csr.yaml

download

apiVersion: certificates.k8s.io/v1
kind: CertificateSigningRequest
metadata:
  name: vault-csr
spec:
  groups:
  - system:authenticated
  request: $(cat ${CERT_DIR}/server.csr | base64 | tr -d '\n')
  signerName: beta.eks.amazonaws.com/app-serving
  usages:
  - digital signature
  - key encipherment
  - server auth

Apply CertificateSigningRequest & approve:

kubectl create -f ${CERT_DIR}/csr.yaml
# Approve the Self-Signed Certificate by K8S CA
kubectl certificate approve vault-csr
# Write public certificate to file
kubectl get csr vault-csr -o jsonpath='{.status.certificate}' | openssl base64 -d -A -out ${CERT_DIR}/vault.crt

Retrieve Kubernetes CA certificate

kubectl config view \
--raw \
--minify \
--flatten \
-o jsonpath='{.clusters[].cluster.certificate-authority-data}' \
| base64 -d > ${CERT_DIR}/vault.ca

kubectl create secret generic ${SECRET_NAME} \
--namespace vault \
--from-file=vault.key=${CERT_DIR}/vault.key \
--from-file=vault.crt=${CERT_DIR}/vault.crt \
--from-file=ca.crt=${CERT_DIR}/vault.ca

vault:
  server:
    extraEnvironmentVars:
      VAULT_CACERT: /vault/userconfig/vault-server-tls/ca.crt
    extraVolumes:
      - type: secret
        name: vault-server-tls
        
    ha:
      raft:
        config: |
          ui = true

          listener "tcp" {
            address = "[::]:8200"
            cluster_address = "[::]:8201"
            tls_cert_file = "/vault/userconfig/vault-server-tls/vault.crt"
            tls_key_file = "/vault/userconfig/vault-server-tls/vault.key"
            tls_client_ca_file = "/vault/userconfig/vault-server-tls/ca.crt"
          }
          
          storage "raft" {
            path = "/vault/data"
              retry_join {
              leader_tls_servername = "*.vault-internal"
              leader_api_addr = "https://vault-0.vault-internal:8200"
              leader_client_cert_file = "/vault/userconfig/vault-server-tls/vault.crt"
              leader_client_key_file = "/vault/userconfig/vault-server-tls/vault.key"
              leader_ca_cert_file = "/vault/userconfig/vault-server-tls/ca.crt"
            }
          
            retry_join {
              leader_tls_servername = "*.vault-internal"
              leader_api_addr = "https://vault-1.vault-internal:8200"
              leader_client_cert_file = "/vault/userconfig/vault-server-tls/vault.crt"
              leader_client_key_file = "/vault/userconfig/vault-server-tls/vault.key"
              leader_ca_cert_file = "/vault/userconfig/vault-server-tls/ca.crt"
            }    
          }

• Enable kubernetes auth in UI
• Configure kubernetes auth

TOKEN_REVIEWER_JWT=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
vault write auth/kubernetes/config \
    token_reviewer_jwt="${TOKEN_REVIEWER_JWT}" \
    kubernetes_host="https://${KUBERNETES_PORT_443_TCP_ADDR}:443" \
    kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt

• Add backup policy in UI

path "sys/storage/raft/snapshot" {
  capabilities = ["read"]
}

• Create backup kubernetes role

vault write auth/kubernetes/role/backup \
bound_service_account_names=vault \
bound_service_account_namespaces=vault \
token_ttl=120m \
policies=backup

download

apiVersion: batch/v1
kind: CronJob
metadata:
  name: vault-backup
  namespace: vault
spec:
  schedule: "0 */2 * * *"
  successfulJobsHistoryLimit: 1
  failedJobsHistoryLimit: 3
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: vault
          volumes:
            - name: share
              emptyDir: {}
            - name: vault-server-tls
              secret:
                secretName: vault-server-tls
          initContainers:
            # Run an init container that creates the the snapshot of Vault
            - name: vault-snapshot
              image: hashicorp/vault:1.14.1
              command: ["/bin/sh", "-c"]
              args:
                # 1. Get the ServiceAccount token which we will use to authenticate against Vault
                # 2. Login to Vault using the SA token at the endpoint where the Kubernetes auth engine
                #    has been enabled
                # 3. Use the Vault CLI to store a snapshot in our empty volume
                - |
                  SA_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token);
                  export VAULT_TOKEN=$(vault write -field=token auth/kubernetes/login jwt=$SA_TOKEN role=backup);
                  vault operator raft snapshot save /share/vault.snap;
              env:
                - name: VAULT_ADDR
                  value: https://vault.vault.svc.cluster.local:8200
                - name: VAULT_CLIENT_CERT
                  value: /etc/vault-ssl/vault.crt
                - name: VAULT_CLIENT_KEY
                  value: /etc/vault-ssl/vault.key
                - name: VAULT_CACERT
                  value: /etc/vault-ssl/ca.crt
              volumeMounts:
                - mountPath: /share
                  name: share
                - mountPath: /etc/vault-ssl/
                  name: vault-server-tls
          containers:
            # Run a container with the AWS CLI and copy the snapshot to our S3 bucket
            - name: aws-s3-backup
              image: amazon/aws-cli:2.2.14
              command:
                - /bin/sh
              args:
                - -ec
                - aws s3 cp /share/vault.snap s3://my-backup-bucket/vault/vault_$(date +"%Y%m%d_%H%M%S").snap;
              volumeMounts:
                - mountPath: /share
                  name: share
          restartPolicy: OnFailure