From fe1bfe9ae16b03e6647f47ee62809ed8641bf51c Mon Sep 17 00:00:00 2001 From: PMExtra Date: Fri, 23 Dec 2022 16:34:18 +0800 Subject: [PATCH 1/4] improve vault and vault_api deployhooks --- deploy/vault.sh | 75 +++++++++++++++++++++++++++++++++++++-------- deploy/vault_cli.sh | 43 +++++++++++++++++++++++++- 2 files changed, 105 insertions(+), 13 deletions(-) diff --git a/deploy/vault.sh b/deploy/vault.sh index 399abaee..2e4d0702 100644 --- a/deploy/vault.sh +++ b/deploy/vault.sh @@ -7,13 +7,16 @@ # # VAULT_PREFIX - this contains the prefix path in vault # VAULT_ADDR - vault requires this to find your vault server +# VAULT_SAVE_TOKEN - set to anything if you want to save the token +# VAULT_RENEW_TOKEN - set to anything if you want to renew the token to default TTL before deploying +# VAULT_KV_V2 - set to anything if you are using v2 of the kv engine # # additionally, you need to ensure that VAULT_TOKEN is avialable # to access the vault server #returns 0 means success, otherwise error. -######## Public functions ##################### +######## Public functions ##################### #domain keyfile certfile cafile fullchain vault_deploy() { @@ -45,6 +48,24 @@ vault_deploy() { fi _savedeployconf VAULT_ADDR "$VAULT_ADDR" + _getdeployconf VAULT_SAVE_TOKEN + _savedeployconf VAULT_SAVE_TOKEN "$VAULT_SAVE_TOKEN" + + _getdeployconf VAULT_RENEW_TOKEN + _savedeployconf VAULT_RENEW_TOKEN "$VAULT_RENEW_TOKEN" + + _getdeployconf VAULT_KV_V2 + _savedeployconf VAULT_KV_V2 "$VAULT_KV_V2" + + _getdeployconf VAULT_TOKEN + if [ -z "$VAULT_TOKEN" ]; then + _err "VAULT_TOKEN needs to be defined" + return 1 + fi + if [ -n "$VAULT_SAVE_TOKEN" ]; then + _savedeployconf VAULT_TOKEN "$VAULT_TOKEN" + fi + # JSON does not allow multiline strings. # So replacing new-lines with "\n" here _ckey=$(sed -z 's/\n/\\n/g' <"$2") @@ -52,26 +73,56 @@ vault_deploy() { _cca=$(sed -z 's/\n/\\n/g' <"$4") _cfullchain=$(sed -z 's/\n/\\n/g' <"$5") - URL="$VAULT_ADDR/v1/$VAULT_PREFIX/$_cdomain" export _H1="X-Vault-Token: $VAULT_TOKEN" + if [ -n "$VAULT_RENEW_TOKEN" ]; then + URL="$VAULT_ADDR/v1/auth/token/renew-self" + _info "Renew the token to default TTL" + if ! _post "" "$URL" >/dev/null; then + _err "Failed to renew the token" + return 1 + fi + fi + + URL="$VAULT_ADDR/v1/$VAULT_PREFIX/$_cdomain" + if [ -n "$FABIO" ]; then + _info "Writing certificate and key to $URL in Fabio mode" if [ -n "$VAULT_KV_V2" ]; then - _post "{ \"data\": {\"cert\": \"$_cfullchain\", \"key\": \"$_ckey\"} }" "$URL" + _post "{ \"data\": {\"cert\": \"$_cfullchain\", \"key\": \"$_ckey\"} }" "$URL" >/dev/null || return 1 else - _post "{\"cert\": \"$_cfullchain\", \"key\": \"$_ckey\"}" "$URL" + _post "{\"cert\": \"$_cfullchain\", \"key\": \"$_ckey\"}" "$URL" >/dev/null || return 1 fi else if [ -n "$VAULT_KV_V2" ]; then - _post "{\"data\": {\"value\": \"$_ccert\"}}" "$URL/cert.pem" - _post "{\"data\": {\"value\": \"$_ckey\"}}" "$URL/cert.key" - _post "{\"data\": {\"value\": \"$_cca\"}}" "$URL/chain.pem" - _post "{\"data\": {\"value\": \"$_cfullchain\"}}" "$URL/fullchain.pem" + _info "Writing certificate to $URL/cert.pem" + _post "{\"data\": {\"value\": \"$_ccert\"}}" "$URL/cert.pem" >/dev/null || return 1 + _info "Writing key to $URL/cert.key" + _post "{\"data\": {\"value\": \"$_ckey\"}}" "$URL/cert.key" >/dev/null || return 1 + _info "Writing CA certificate to $URL/ca.pem" + _post "{\"data\": {\"value\": \"$_cca\"}}" "$URL/ca.pem" >/dev/null || return 1 + _info "Writing full-chain certificate to $URL/fullchain.pem" + _post "{\"data\": {\"value\": \"$_cfullchain\"}}" "$URL/fullchain.pem" >/dev/null || return 1 else - _post "{\"value\": \"$_ccert\"}" "$URL/cert.pem" - _post "{\"value\": \"$_ckey\"}" "$URL/cert.key" - _post "{\"value\": \"$_cca\"}" "$URL/chain.pem" - _post "{\"value\": \"$_cfullchain\"}" "$URL/fullchain.pem" + _info "Writing certificate to $URL/cert.pem" + _post "{\"value\": \"$_ccert\"}" "$URL/cert.pem" >/dev/null || return 1 + _info "Writing key to $URL/cert.key" + _post "{\"value\": \"$_ckey\"}" "$URL/cert.key" >/dev/null || return 1 + _info "Writing CA certificate to $URL/ca.pem" + _post "{\"value\": \"$_cca\"}" "$URL/ca.pem" >/dev/null || return 1 + _info "Writing full-chain certificate to $URL/fullchain.pem" + _post "{\"value\": \"$_cfullchain\"}" "$URL/fullchain.pem" >/dev/null || return 1 + fi + + # To make it compatible with the wrong ca path `chain.pem` which was used in former versions + if _get "$URL/chain.pem" >/dev/null; then + _err "The CA certificate has moved from chain.pem to ca.pem, if you don't depend on chain.pem anymore, you can delete it to avoid this warning" + _info "Updating CA certificate to $URL/chain.pem for backward compatibility" + if [ -n "$VAULT_KV_V2" ]; then + _post "{\"data\": {\"value\": \"$_cca\"}}" "$URL/chain.pem" >/dev/null || return 1 + else + _post "{\"value\": \"$_cca\"}" "$URL/chain.pem" >/dev/null || return 1 + fi fi fi diff --git a/deploy/vault_cli.sh b/deploy/vault_cli.sh index cbb8cc59..da6fe093 100644 --- a/deploy/vault_cli.sh +++ b/deploy/vault_cli.sh @@ -8,6 +8,8 @@ # # VAULT_PREFIX - this contains the prefix path in vault # VAULT_ADDR - vault requires this to find your vault server +# VAULT_SAVE_TOKEN - set to anything if you want to save the token +# VAULT_RENEW_TOKEN - set to anything if you want to renew the token to default TTL before deploying # # additionally, you need to ensure that VAULT_TOKEN is avialable or # `vault auth` has applied the appropriate authorization for the vault binary @@ -33,15 +35,34 @@ vault_cli_deploy() { _debug _cfullchain "$_cfullchain" # validate required env vars + _getdeployconf VAULT_PREFIX if [ -z "$VAULT_PREFIX" ]; then _err "VAULT_PREFIX needs to be defined (contains prefix path in vault)" return 1 fi + _savedeployconf VAULT_PREFIX "$VAULT_PREFIX" + _getdeployconf VAULT_ADDR if [ -z "$VAULT_ADDR" ]; then _err "VAULT_ADDR needs to be defined (contains vault connection address)" return 1 fi + _savedeployconf VAULT_ADDR "$VAULT_ADDR" + + _getdeployconf VAULT_SAVE_TOKEN + _savedeployconf VAULT_SAVE_TOKEN "$VAULT_SAVE_TOKEN" + + _getdeployconf VAULT_RENEW_TOKEN + _savedeployconf VAULT_RENEW_TOKEN "$VAULT_RENEW_TOKEN" + + _getdeployconf VAULT_TOKEN + if [ -z "$VAULT_TOKEN" ]; then + _err "VAULT_TOKEN needs to be defined" + return 1 + fi + if [ -n "$VAULT_SAVE_TOKEN" ]; then + _savedeployconf VAULT_TOKEN "$VAULT_TOKEN" + fi VAULT_CMD=$(command -v vault) if [ ! $? ]; then @@ -49,13 +70,33 @@ vault_cli_deploy() { return 1 fi + if [ -n "$VAULT_RENEW_TOKEN" ]; then + _info "Renew the token to default TTL" + if ! $VAULT_CMD token renew; then + _err "Failed to renew the token" + return 1 + fi + fi + if [ -n "$FABIO" ]; then + _info "Writing certificate and key to $URL in Fabio mode" $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}" cert=@"$_cfullchain" key=@"$_ckey" || return 1 else + _info "Writing certificate to $URL/cert.pem" $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}/cert.pem" value=@"$_ccert" || return 1 + _info "Writing key to $URL/cert.key" $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}/cert.key" value=@"$_ckey" || return 1 - $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}/chain.pem" value=@"$_cca" || return 1 + _info "Writing CA certificate to $URL/ca.pem" + $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}/ca.pem" value=@"$_cca" || return 1 + _info "Writing full-chain certificate to $URL/fullchain.pem" $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}/fullchain.pem" value=@"$_cfullchain" || return 1 + + # To make it compatible with the wrong ca path `chain.pem` which was used in former versions + if $VAULT_CMD kv get "${VAULT_PREFIX}/${_cdomain}/chain.pem" >/dev/null; then + _err "The CA certificate has moved from chain.pem to ca.pem, if you don't depend on chain.pem anymore, you can delete it to avoid this warning" + _info "Updating CA certificate to $URL/chain.pem for backward compatibility" + $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}/chain.pem" value=@"$_cca" || return 1 + fi fi } From b8d0d3c2429e2f33dceb9c94ad8e5273762be72a Mon Sep 17 00:00:00 2001 From: PMExtra Date: Fri, 23 Dec 2022 19:17:37 +0800 Subject: [PATCH 2/4] improve chain.pem exists evaluating --- deploy/vault.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/vault.sh b/deploy/vault.sh index 2e4d0702..6c953fc6 100644 --- a/deploy/vault.sh +++ b/deploy/vault.sh @@ -115,7 +115,7 @@ vault_deploy() { fi # To make it compatible with the wrong ca path `chain.pem` which was used in former versions - if _get "$URL/chain.pem" >/dev/null; then + if _contains "$(_get "$URL/chain.pem")" "-----BEGIN CERTIFICATE-----"; then _err "The CA certificate has moved from chain.pem to ca.pem, if you don't depend on chain.pem anymore, you can delete it to avoid this warning" _info "Updating CA certificate to $URL/chain.pem for backward compatibility" if [ -n "$VAULT_KV_V2" ]; then From ed63eb6833ab288b2a963255875bf9df99e39c79 Mon Sep 17 00:00:00 2001 From: PMExtra Date: Fri, 23 Dec 2022 19:32:06 +0800 Subject: [PATCH 3/4] migrate FABIO to VAULT_FABIO_MODE and persist it --- deploy/vault.sh | 4 +++- deploy/vault_cli.sh | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/deploy/vault.sh b/deploy/vault.sh index 6c953fc6..30e825eb 100644 --- a/deploy/vault.sh +++ b/deploy/vault.sh @@ -66,6 +66,8 @@ vault_deploy() { _savedeployconf VAULT_TOKEN "$VAULT_TOKEN" fi + _migratedeployconf FABIO VAULT_FABIO_MODE + # JSON does not allow multiline strings. # So replacing new-lines with "\n" here _ckey=$(sed -z 's/\n/\\n/g' <"$2") @@ -86,7 +88,7 @@ vault_deploy() { URL="$VAULT_ADDR/v1/$VAULT_PREFIX/$_cdomain" - if [ -n "$FABIO" ]; then + if [ -n "$VAULT_FABIO_MODE" ]; then _info "Writing certificate and key to $URL in Fabio mode" if [ -n "$VAULT_KV_V2" ]; then _post "{ \"data\": {\"cert\": \"$_cfullchain\", \"key\": \"$_ckey\"} }" "$URL" >/dev/null || return 1 diff --git a/deploy/vault_cli.sh b/deploy/vault_cli.sh index da6fe093..ecbffd23 100644 --- a/deploy/vault_cli.sh +++ b/deploy/vault_cli.sh @@ -64,6 +64,8 @@ vault_cli_deploy() { _savedeployconf VAULT_TOKEN "$VAULT_TOKEN" fi + _migratedeployconf FABIO VAULT_FABIO_MODE + VAULT_CMD=$(command -v vault) if [ ! $? ]; then _err "cannot find vault binary!" @@ -78,7 +80,7 @@ vault_cli_deploy() { fi fi - if [ -n "$FABIO" ]; then + if [ -n "$VAULT_FABIO_MODE" ]; then _info "Writing certificate and key to $URL in Fabio mode" $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}" cert=@"$_cfullchain" key=@"$_ckey" || return 1 else From 1ccfa96c2e2c78aaca1ba92549e62b9f52fa9461 Mon Sep 17 00:00:00 2001 From: PMExtra Date: Wed, 28 Dec 2022 02:47:49 +0800 Subject: [PATCH 4/4] improve logging --- deploy/vault.sh | 4 ++-- deploy/vault_cli.sh | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/deploy/vault.sh b/deploy/vault.sh index 30e825eb..569faba2 100644 --- a/deploy/vault.sh +++ b/deploy/vault.sh @@ -79,9 +79,9 @@ vault_deploy() { if [ -n "$VAULT_RENEW_TOKEN" ]; then URL="$VAULT_ADDR/v1/auth/token/renew-self" - _info "Renew the token to default TTL" + _info "Renew the Vault token to default TTL" if ! _post "" "$URL" >/dev/null; then - _err "Failed to renew the token" + _err "Failed to renew the Vault token" return 1 fi fi diff --git a/deploy/vault_cli.sh b/deploy/vault_cli.sh index ecbffd23..3ebb8074 100644 --- a/deploy/vault_cli.sh +++ b/deploy/vault_cli.sh @@ -73,30 +73,30 @@ vault_cli_deploy() { fi if [ -n "$VAULT_RENEW_TOKEN" ]; then - _info "Renew the token to default TTL" + _info "Renew the Vault token to default TTL" if ! $VAULT_CMD token renew; then - _err "Failed to renew the token" + _err "Failed to renew the Vault token" return 1 fi fi if [ -n "$VAULT_FABIO_MODE" ]; then - _info "Writing certificate and key to $URL in Fabio mode" + _info "Writing certificate and key to ${VAULT_PREFIX}/${_cdomain} in Fabio mode" $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}" cert=@"$_cfullchain" key=@"$_ckey" || return 1 else - _info "Writing certificate to $URL/cert.pem" + _info "Writing certificate to ${VAULT_PREFIX}/${_cdomain}/cert.pem" $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}/cert.pem" value=@"$_ccert" || return 1 - _info "Writing key to $URL/cert.key" + _info "Writing key to ${VAULT_PREFIX}/${_cdomain}/cert.key" $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}/cert.key" value=@"$_ckey" || return 1 - _info "Writing CA certificate to $URL/ca.pem" + _info "Writing CA certificate to ${VAULT_PREFIX}/${_cdomain}/ca.pem" $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}/ca.pem" value=@"$_cca" || return 1 - _info "Writing full-chain certificate to $URL/fullchain.pem" + _info "Writing full-chain certificate to ${VAULT_PREFIX}/${_cdomain}/fullchain.pem" $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}/fullchain.pem" value=@"$_cfullchain" || return 1 # To make it compatible with the wrong ca path `chain.pem` which was used in former versions if $VAULT_CMD kv get "${VAULT_PREFIX}/${_cdomain}/chain.pem" >/dev/null; then _err "The CA certificate has moved from chain.pem to ca.pem, if you don't depend on chain.pem anymore, you can delete it to avoid this warning" - _info "Updating CA certificate to $URL/chain.pem for backward compatibility" + _info "Updating CA certificate to ${VAULT_PREFIX}/${_cdomain}/chain.pem for backward compatibility" $VAULT_CMD kv put "${VAULT_PREFIX}/${_cdomain}/chain.pem" value=@"$_cca" || return 1 fi fi