GKEでVaultを高可用性構成に

$ kubectl port-forward vault-0 8200:8200

export PATH=$GOPATH/bin:$PATH

go get -u github.com/cloudflare/cfssl/cmd/cfssl
go get -u github.com/cloudflare/cfssl/cmd/cfssljson

$ mkdir vault-ca && cd vault-ca

$ cat <<EOF > ca-config.json
{
  "signing": {
    "default": {
      "expiry": "8760h"
    },
    "profiles": {
      "default": {
        "usages": ["signing", "key encipherment", "server auth", "client auth"],
        "expiry": "8760h"
      }
    }
  }
}
EOF

$ cat <<EOF > ca-csr.json
{
  "hosts": [\
    "cluster.local"\
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [\
    {\
      "C": "US",\
      "L": "NewYork",\
      "O": "Kubernetes",\
      "OU": "CA",\
      "ST": "NewYork"\
    }\
  ]
}
EOF

$ cat <<EOF > vault-csr.json
{
  "CN": "Vault-GKE",
  "hosts": [\
    "127.0.0.1",\
    "vault.default.svc.cluster.local"\
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [\
    {\
      "C": "US",\
      "L": "NewYork",\
      "O": "Innovia",\
      "OU": "Vault",\
      "ST": "NewYork"\
    }\
  ]
}
EOF

$ cfssl gencert -initca ca-csr.json | cfssljson -bare ca
2019/11/12 16:35:01 [INFO] generating a new CA key and certificate from CSR
2019/11/12 16:35:01 [INFO] generate received request
2019/11/12 16:35:01 [INFO] received CSR
2019/11/12 16:35:01 [INFO] generating key: rsa-2048
2019/11/12 16:35:01 [INFO] encoded CSR
2019/11/12 16:35:01 [INFO] signed certificate with serial number 425581644650417483788325060652779897454211028144

$ cfssl gencert \
  -ca=ca.pem \
  -ca-key=ca-key.pem \
  -config=ca-config.json \
  -profile=default \
  vault-csr.json | cfssljson -bare vault
2019/11/12 16:36:33 [INFO] generate received request
2019/11/12 16:36:33 [INFO] received CSR
2019/11/12 16:36:33 [INFO] generating key: rsa-2048
2019/11/12 16:36:34 [INFO] encoded CSR
2019/11/12 16:36:34 [INFO] signed certificate with serial number 311973563616303179057952194819087555625015840298

ca-key.pem
ca.pem
vault-key.pem
vault.pem

kubectl create secret generic vault-tls \
  --from-file=ca.pem \
  --from-file=vault.pem \
  --from-file=vault-key.pem

$ export GCP_PROJECT=<your_project_id>

$ gcloud services enable \
    cloudapis.googleapis.com \
    cloudkms.googleapis.com \
    container.googleapis.com \
    containerregistry.googleapis.com \
    iam.googleapis.com \
    --project ${GCP_PROJECT}
Operation "operations/acf.8e126724-bbde-4c0d-b516-5dca5b8443ee" finished successfully.

$ export GCS_BUCKET_NAME=mycompany-vault-data
$ gsutil mb gs://$GCS_BUCKET_NAME
$ gsutil versioning set on gs://$GCS_BUCKET_NAME

$ gcloud kms keyrings create vault-helm-unseal-kr \
    --location global \
    --project ${GCP_PROJECT}

$ gcloud kms keys create vault-helm-unseal-key \
    --location global \
    --keyring vault-helm-unseal-kr \
    --purpose encryption \
    --project ${GCP_PROJECT}

$ export VAULT_SA_NAME=vault-server;
 export VAULT_SA=$VAULT_SA_NAME@$GCP_PROJECT.iam.gserviceaccount.com

$ gcloud iam service-accounts create $VAULT_SA_NAME \
    --display-name "Vault server service account" \
    --project ${GCP_PROJECT}

$ gcloud iam service-accounts keys create \
      --iam-account $VAULT_SA /tmp/vault_gcs_key.json
created key [be22cfe6e30f3a3fcfc6ebaa23ca3ba905dd60ab] of type [json] as [/tmp/vault_gcs_key.json] for [[email protected]]

 VaultのGoogleサービスアカウントを格納するシークレットを作成

$ kubectl create secret generic vault-gcs \
  --from-file=/tmp/vault_gcs_key.json
secret/vault-gcs created

$ gsutil iam ch \
  serviceAccount:${VAULT_SA}:objectAdmin \
  gs://${GCS_BUCKET_NAME}

$ gcloud kms keys add-iam-policy-binding \
    vault-helm-unseal-key \
    --location global \
    --keyring vault-helm-unseal-kr \
    --member serviceAccount:${VAULT_SA} \
    --role roles/cloudkms.cryptoKeyEncrypterDecrypter \
    --project ${GCP_PROJECT}
Updated IAM policy for key [vault-helm-unseal-key].
bindings:
- members:
  - serviceAccount:[email protected]
  role: roles/cloudkms.cryptoKeyEncrypterDecrypter
etag: BwWZ6sIYovk=
version: 1

注意:
サービスアカウントを一度削除して再作成した場合は、鍵に紐づくIAMポリシーを削除する必要があります。該当しない場合は、HashiCorp Vault公式チャートの取得セクションへ進んでください。

$ gcloud kms keys get-iam-policy vault-helm-unseal-key --location global --keyring vault-helm-unseal-kr > kms-policy.yaml

 ポリシーファイルを編集し、binding配下のmembersを削除して保存

bindings:
etag: BwWXQz4HjuI=
version: 1

$ gcloud kms keys set-iam-policy vault-helm-unseal-key --location global --keyring vault-helm-unseal-kr  kms-policy.yaml

export CHART_VERSION=0.3.0

$ wget https://github.com/hashicorp/vault-helm/archive/v$CHART_VERSION.tar.gz && tar zxf v$CHART_VERSION.tar.gz && rm v$CHART_VERSION.tar.gz

global:
  tlsDisable: false

server:
  extraEnvironmentVars:
    GOOGLE_APPLICATION_CREDENTIALS: /vault/userconfig/vault-gcs/vault_gcs_key.json
  extraVolumes:
  - type: secret
    name: vault-gcs
    path: "/vault/userconfig"
  - type: secret
    name: vault-tls
    path: "/etc/tls"

  authDelegator:
    enabled: true

  ha:
    enabled: true
    config: |
      ui = true
      listener "tcp" {
        tls_disable = 0
        tls_cert_file = "/etc/tls/vault-tls/vault.pem"
        tls_key_file = "/etc/tls/vault-tls/vault-key.pem"
        tls_client_ca_file =  "/etc/tls/vault-tls/ca.pem"
        tls_min_version = "tls12"
        address = "[::]:8200"
        cluster_address = "[::]:8201"
      }
      storage "gcs" {
        bucket     = "GCS_BUCKET_NAME"
        ha_enabled = "true"
      }
      seal "gcpckms" {
        project     = "GCP_PROJECT"
        region      = "global"
        key_ring    = "vault-helm-unseal-kr"
        crypto_key  = "vault-helm-unseal-key"
      }

$ curl -s https://gist.githubusercontent.com/innovia/53c05bf69312706fc93ffe3bb685b223/raw/adc169605984da8ba82082191c8f631579b1b199/vault-gke-values.yaml | sed  "s/GCP_PROJECT/$GCP_PROJECT/g" | sed "s/GCS_BUCKET_NAME/$GCS_BUCKET_NAME/g" > vault-helm-$CHART_VERSION/vault-gke-values.yaml

$ cat vault-helm-$CHART_VERSION/vault-gke-values.yaml | grep -E 'bucket|project'
bucket     = "<COMPANY>-vault-data"
project    = "ami-playground"

$ brew install helm@2

helm init --client-only

helm-tillerlessプラグインをインストール

helm plugin install https://github.com/rimusz/helm-tiller

helm経由でtillerを起動

$ helm tiller start
Installed Helm version v2.16.1
Copied found /usr/local/bin/tiller to helm-tiller/bin
Helm and Tiller are the same version!
Starting Tiller...
Tiller namespace: kube-system

$ brew install helm

$ helm upgrade --install vault -f vault-helm-$CHART_VERSION/vault-gke-values.yaml vault-helm-$CHART_VERSION
release "vault" does not exist. Installing it now.
NAME:   vault
LAST DEPLOYED: Wed Nov 13 15:41:55 2019
NAMESPACE: default
STATUS: DEPLOYED
RESOURCES:
==> v1/ConfigMap
NAME          AGE
vault-config  0s
==> v1/Service
NAME   AGE
vault  0s
==> v1/ServiceAccount
NAME   AGE
vault  0s
==> v1/StatefulSet
NAME   AGE
vault  0s
==> v1beta1/ClusterRoleBinding
NAME                  AGE
vault-server-binding  0s
==> v1beta1/PodDisruptionBudget
NAME   AGE
vault  0s
NOTES:
Thank you for installing HashiCorp Vault!
Now that you have deployed Vault, you should look over the docs on using
Vault with Kubernetes available here:
https://www.vaultproject.io/docs/
Your release is named vault. To learn more about the release, try:
$ helm status vault
  $ helm get vault

Vaultが起動し、未初期化状態になっているはずです。

=> Vault server started! Log data will stream in below:
2019-12-17T19:07:37.937Z [INFO]  proxy environment: http_proxy= https_proxy= no_proxy=
2019-12-17T19:07:38.909Z [INFO]  core: stored unseal keys supported, attempting fetch
2019-12-17T19:07:39.037Z [WARN]  failed to unseal core: error="stored unseal keys are supported, but none were found"
2019-12-17T19:07:44.038Z [INFO]  core: stored unseal keys supported, attempting fetch
2019-12-17T19:07:44.080Z [INFO]  core: autoseal: seal configuration missing, but cannot check old path as core is sealed: seal_type=recovery
2019-12-17T19:07:44.174Z [WARN]  failed to unseal core: error="stored unseal keys are supported, but none were found"
---
kubectl describe pod vault-0
Events:
  Type     Reason     Age               From               Message
  ----     ------     ----              ----               -------
  ...
  Warning  Unhealthy  3s (x9 over 27s)  kubelet, minikube  Readiness probe failed: Key                      Value

$ kubectl port-forward vault-0 8200:8200 > /dev/null & export PID=$!; echo "vault port-forward pid: $PID"

$ export VAULT_ADDR=https://127.0.0.1:8200;
  export VAULT_CACERT=$PWD/ca.pem
my vault ca.pem for example is at:
VAULT_CACERT: /Users/ami/vault-gke-medium/ca.pem

$ brew install vault

$ vault status
Key                      Value
---                      -----
Recovery Seal Type       gcpckms
Initialized              false
Sealed                   true
Total Recovery Shares    0
Threshold                0
Unseal Progress          0/0
Unseal Nonce             n/a
Version                  n/a
HA Enabled               true

vault operator init
Recovery Key 1: 33nCanHWgYMR/VPj6bNQdHXJiayL6WeB8Ourx4kHYNaX
Recovery Key 2: IMf7RjptFxtGQUbEWUWehanCBiSY7VhElkM7rRVxczGc
Recovery Key 3: zGuzk/PhNet9OHL4cW2H7d3XypDxfwWXkmajclLPklK4
Recovery Key 4: nCFS0dt0cNGB2LWk0F+3Vmz9TbVNpeIsXbIXDbRarlnT
Recovery Key 5: 9GxXr/6T8OJWJrWqyHQxayR0BAK+WTdbT870AzKEFl2V
Initial Root Token: s.1ukhSgycySjZUJRD0bZjSEit
Success! Vault is initialized
Recovery key initialized with 5 key shares and a key threshold of 3. Please
securely distribute the key shares printed above.

$ sudo security add-trusted-cert -d -k /Library/Keychains/System.keychain $VAULT_CACERT

kubectl apply -f vault-reviewer.yaml

$ vault login
$ vault auth enable kubernetes
Success! Enabled kubernetes auth method at: kubernetes/

$ VAULT_SA_TOKEN_NAME=$(kubectl get sa vault-reviewer -o jsonpath="{.secrets[*]['name']}")
$ SA_JWT_TOKEN=$(kubectl get secret "$VAULT_SA_TOKEN_NAME" -o jsonpath="{.data.token}" | base64 --decode; echo)
$ SA_CA_CRT=$(kubectl get secret "$VAULT_SA_TOKEN_NAME" -o jsonpath="{.data['ca\.crt']}" | base64 --decode; echo)
$ vault write auth/kubernetes/config token_reviewer_jwt="$SA_JWT_TOKEN" kubernetes_host=https://kubernetes.default kubernetes_ca_cert="$SA_CA_CRT"
Success! Data written to: auth/kubernetes/config

path "secret/foo" {
 capabilities = ["read"]
}

vault write auth/kubernetes/role/<role_name> \
 bound_service_account_names=<service_account_name> \
 bound_service_account_namespaces=<service_account_namespace> \
 policies=<policy_name>

 $ gcloud compute addresses create vault-ui --global

$ kubectl apply -f managed-cert.yaml

$ kubectl describe ManagedCertificate vault-ui-certificate
Name:         vault-ui-certificate
Namespace:    default
Labels:       <none>
Annotations:  kubectl.kubernetes.io/last-applied-configuration:
                {"apiVersion":"networking.gke.io/v1beta1","kind":"ManagedCertificate","metadata":{"annotations":{},"name":"vault-ui-certificate","namespac...
API Version:  networking.gke.io/v1beta1
Kind:         ManagedCertificate
Metadata:
  Creation Timestamp:  2020-01-13T23:10:28Z
  Generation:          3
  Resource Version:    7120865
  Self Link:           /apis/networking.gke.io/v1beta1/namespaces/default/managedcertificates/vault-ui-certificate
  UID:                 e35e7a1b-3659-11ea-ae90-42010aa80174
Spec:
  Domains:
    vault.ami-playground.doit-intl.com
Status:
  Certificate Name:    mcrt-9462e1f4-6dd6-4cf2-8769-9693ba29789e
  Certificate Status:  Active
  Domain Status:
    Domain:     vault.ami-playground.doit-intl.com
    Status:     Active
  Expire Time:  2020-04-12T15:12:29.000-07:00
Events:         <none>

https://iap.googleapis.com/v1/oauth/clientIds/<CLIENT_ID>:handleRedirect

kubectl create secret generic my-secret --from-literal=client_id=client_id_key \
    --from-literal=client_secret=client_secret_key