blob: a0f62a06a9ff7fa041438f70e7b05046dc7cc19c [file] [log] [blame] [edit]
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: BUSL-1.1
scenario "seal_ha" {
matrix {
arch = ["amd64", "arm64"]
artifact_source = ["local", "crt", "artifactory"]
artifact_type = ["bundle", "package"]
backend = ["consul", "raft"]
consul_version = ["1.12.9", "1.13.9", "1.14.9", "1.15.5", "1.16.1"]
distro = ["ubuntu", "rhel"]
edition = ["ent", "ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
primary_seal = ["awskms"]
secondary_seal = ["awskms"]
# Our local builder always creates bundles
exclude {
artifact_source = ["local"]
artifact_type = ["package"]
}
# HSM and FIPS 140-2 are only supported on amd64
exclude {
arch = ["arm64"]
edition = ["ent.fips1402", "ent.hsm", "ent.hsm.fips1402"]
}
}
terraform_cli = terraform_cli.default
terraform = terraform.default
providers = [
provider.aws.default,
provider.enos.ubuntu,
provider.enos.rhel
]
locals {
artifact_path = matrix.artifact_source != "artifactory" ? abspath(var.vault_artifact_path) : null
enos_provider = {
rhel = provider.enos.rhel
ubuntu = provider.enos.ubuntu
}
manage_service = matrix.artifact_type == "bundle"
vault_install_dir = matrix.artifact_type == "bundle" ? var.vault_install_dir : global.vault_install_dir_packages[matrix.distro]
}
step "get_local_metadata" {
skip_step = matrix.artifact_source != "local"
module = module.get_local_metadata
}
step "build_vault" {
module = "build_${matrix.artifact_source}"
variables {
build_tags = var.vault_local_build_tags != null ? var.vault_local_build_tags : global.build_tags[matrix.edition]
artifact_path = local.artifact_path
goarch = matrix.arch
goos = "linux"
artifactory_host = matrix.artifact_source == "artifactory" ? var.artifactory_host : null
artifactory_repo = matrix.artifact_source == "artifactory" ? var.artifactory_repo : null
artifactory_username = matrix.artifact_source == "artifactory" ? var.artifactory_username : null
artifactory_token = matrix.artifact_source == "artifactory" ? var.artifactory_token : null
arch = matrix.artifact_source == "artifactory" ? matrix.arch : null
product_version = var.vault_product_version
artifact_type = matrix.artifact_type
distro = matrix.artifact_source == "artifactory" ? matrix.distro : null
edition = matrix.artifact_source == "artifactory" ? matrix.edition : null
revision = var.vault_revision
}
}
step "ec2_info" {
module = module.ec2_info
}
step "create_vpc" {
module = module.create_vpc
variables {
common_tags = global.tags
}
}
step "create_primary_seal_key" {
module = "seal_key_${matrix.primary_seal}"
variables {
cluster_id = step.create_vpc.cluster_id
cluster_meta = "primary"
common_tags = global.tags
}
}
step "create_secondary_seal_key" {
module = "seal_key_${matrix.secondary_seal}"
variables {
cluster_id = step.create_vpc.cluster_id
cluster_meta = "secondary"
common_tags = global.tags
other_resources = step.create_primary_seal_key.resource_names
}
}
// This step reads the contents of the backend license if we're using a Consul backend and
// the edition is "ent".
step "read_backend_license" {
skip_step = matrix.backend == "raft" || var.backend_edition == "ce"
module = module.read_license
variables {
file_name = global.backend_license_path
}
}
step "read_vault_license" {
skip_step = matrix.edition == "ce"
module = module.read_license
variables {
file_name = global.vault_license_path
}
}
step "create_vault_cluster_targets" {
module = module.target_ec2_instances
depends_on = [step.create_vpc]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
ami_id = step.ec2_info.ami_ids[matrix.arch][matrix.distro][global.distro_version[matrix.distro]]
cluster_tag_key = global.vault_tag_key
common_tags = global.tags
seal_key_names = step.create_secondary_seal_key.resource_names
vpc_id = step.create_vpc.id
}
}
step "create_vault_cluster_backend_targets" {
module = matrix.backend == "consul" ? module.target_ec2_instances : module.target_ec2_shim
depends_on = [step.create_vpc]
providers = {
enos = provider.enos.ubuntu
}
variables {
ami_id = step.ec2_info.ami_ids["arm64"]["ubuntu"]["22.04"]
seal_key_names = step.create_secondary_seal_key.resource_names
cluster_tag_key = global.backend_tag_key
common_tags = global.tags
vpc_id = step.create_vpc.id
}
}
step "create_backend_cluster" {
module = "backend_${matrix.backend}"
depends_on = [
step.create_vault_cluster_backend_targets
]
providers = {
enos = provider.enos.ubuntu
}
variables {
cluster_name = step.create_vault_cluster_backend_targets.cluster_name
cluster_tag_key = global.backend_tag_key
license = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
release = {
edition = var.backend_edition
version = matrix.consul_version
}
target_hosts = step.create_vault_cluster_backend_targets.hosts
}
}
step "create_vault_cluster" {
module = module.vault_cluster
depends_on = [
step.create_backend_cluster,
step.build_vault,
step.create_vault_cluster_targets
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
artifactory_release = matrix.artifact_source == "artifactory" ? step.build_vault.vault_artifactory_release : null
backend_cluster_name = step.create_vault_cluster_backend_targets.cluster_name
backend_cluster_tag_key = global.backend_tag_key
cluster_name = step.create_vault_cluster_targets.cluster_name
consul_license = (matrix.backend == "consul" && var.backend_edition == "ent") ? step.read_backend_license.license : null
consul_release = matrix.backend == "consul" ? {
edition = var.backend_edition
version = matrix.consul_version
} : null
enable_audit_devices = var.vault_enable_audit_devices
install_dir = local.vault_install_dir
license = matrix.edition != "ce" ? step.read_vault_license.license : null
local_artifact_path = local.artifact_path
manage_service = local.manage_service
packages = concat(global.packages, global.distro_packages[matrix.distro])
// Only configure our primary seal during our initial cluster setup
seal_type = matrix.primary_seal
seal_key_name = step.create_primary_seal_key.resource_name
storage_backend = matrix.backend
target_hosts = step.create_vault_cluster_targets.hosts
}
}
// Wait for our cluster to elect a leader
step "wait_for_leader" {
module = module.vault_wait_for_leader
depends_on = [step.create_vault_cluster]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
timeout = 120 # seconds
vault_hosts = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.root_token
}
}
step "get_vault_cluster_ips" {
module = module.vault_get_cluster_ips
depends_on = [step.wait_for_leader]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_hosts = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_vault_unsealed" {
module = module.vault_verify_unsealed
depends_on = [step.wait_for_leader]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster_targets.hosts
}
}
// Write some test data before we create the new seal
step "verify_write_test_data" {
module = module.vault_verify_write_data
depends_on = [
step.create_vault_cluster,
step.get_vault_cluster_ips,
step.verify_vault_unsealed,
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
leader_public_ip = step.get_vault_cluster_ips.leader_public_ip
leader_private_ip = step.get_vault_cluster_ips.leader_private_ip
vault_instances = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.root_token
}
}
// Wait for the initial seal rewrap to complete before we add our HA seal.
step "wait_for_initial_seal_rewrap" {
module = module.vault_wait_for_seal_rewrap
depends_on = [
step.verify_write_test_data,
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_hosts = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.root_token
}
}
// Stop the vault service on all nodes before we restart with new seal config
step "stop_vault" {
module = module.stop_vault
depends_on = [
step.create_vault_cluster,
step.verify_write_test_data,
step.wait_for_initial_seal_rewrap,
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
target_hosts = step.create_vault_cluster_targets.hosts
}
}
// Add the secondary seal to the cluster
step "add_ha_seal_to_cluster" {
module = module.start_vault
depends_on = [step.stop_vault]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
cluster_name = step.create_vault_cluster_targets.cluster_name
install_dir = local.vault_install_dir
license = matrix.edition != "ce" ? step.read_vault_license.license : null
manage_service = local.manage_service
seal_type = matrix.primary_seal
seal_key_name = step.create_primary_seal_key.resource_name
seal_type_secondary = matrix.secondary_seal
seal_key_name_secondary = step.create_secondary_seal_key.resource_name
storage_backend = matrix.backend
target_hosts = step.create_vault_cluster_targets.hosts
}
}
// Wait for our cluster to elect a leader
step "wait_for_new_leader" {
module = module.vault_wait_for_leader
depends_on = [step.add_ha_seal_to_cluster]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
timeout = 120 # seconds
vault_hosts = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.root_token
}
}
step "get_updated_cluster_ips" {
module = module.vault_get_cluster_ips
depends_on = [step.wait_for_new_leader]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_hosts = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_vault_unsealed_with_new_seal" {
module = module.vault_verify_unsealed
depends_on = [step.wait_for_new_leader]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster_targets.hosts
}
}
// Wait for the seal rewrap to complete and verify that no entries failed
step "wait_for_seal_rewrap" {
module = module.vault_wait_for_seal_rewrap
depends_on = [
step.add_ha_seal_to_cluster,
step.verify_vault_unsealed_with_new_seal,
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_hosts = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.root_token
}
}
// Perform all of our standard verifications after we've enabled multiseal
step "verify_vault_version" {
module = module.vault_verify_version
depends_on = [step.wait_for_seal_rewrap]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_instances = step.create_vault_cluster_targets.hosts
vault_edition = matrix.edition
vault_install_dir = local.vault_install_dir
vault_product_version = matrix.artifact_source == "local" ? step.get_local_metadata.version : var.vault_product_version
vault_revision = matrix.artifact_source == "local" ? step.get_local_metadata.revision : var.vault_revision
vault_build_date = matrix.artifact_source == "local" ? step.get_local_metadata.build_date : var.vault_build_date
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_raft_auto_join_voter" {
skip_step = matrix.backend != "raft"
module = module.vault_verify_raft_auto_join_voter
depends_on = [step.wait_for_seal_rewrap]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster_targets.hosts
vault_root_token = step.create_vault_cluster.root_token
}
}
step "verify_replication" {
module = module.vault_verify_replication
depends_on = [step.wait_for_seal_rewrap]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_edition = matrix.edition
vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster_targets.hosts
}
}
// Make sure our data is still available
step "verify_read_test_data" {
module = module.vault_verify_read_data
depends_on = [step.wait_for_seal_rewrap]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
node_public_ips = step.get_updated_cluster_ips.follower_public_ips
vault_install_dir = local.vault_install_dir
}
}
step "verify_ui" {
module = module.vault_verify_ui
depends_on = [step.wait_for_seal_rewrap]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_instances = step.create_vault_cluster_targets.hosts
}
}
// Make sure we have a "multiseal" seal type
step "verify_seal_type" {
// Don't run this on versions less than 1.16.0-beta1 until VAULT-21053 is fixed on prior branches.
skip_step = semverconstraint(var.vault_product_version, "< 1.16.0-beta1")
module = module.verify_seal_type
depends_on = [step.wait_for_seal_rewrap]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_install_dir = local.vault_install_dir
vault_hosts = step.create_vault_cluster_targets.hosts
seal_type = "multiseal"
}
}
// Now we'll migrate away from our initial seal to our secondary seal
// Stop the vault service on all nodes before we restart with new seal config
step "stop_vault_for_migration" {
module = module.stop_vault
depends_on = [
step.wait_for_seal_rewrap,
step.verify_read_test_data,
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
target_hosts = step.create_vault_cluster_targets.hosts
}
}
// Remove the "primary" seal from the cluster. Set our "secondary" seal to priority 1. We do this
// by restarting vault with the correct config.
step "remove_primary_seal" {
module = module.start_vault
depends_on = [step.stop_vault_for_migration]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
cluster_name = step.create_vault_cluster_targets.cluster_name
install_dir = local.vault_install_dir
license = matrix.edition != "ce" ? step.read_vault_license.license : null
manage_service = local.manage_service
seal_alias = "secondary"
seal_type = matrix.secondary_seal
seal_key_name = step.create_secondary_seal_key.resource_name
storage_backend = matrix.backend
target_hosts = step.create_vault_cluster_targets.hosts
}
}
// Wait for our cluster to elect a leader after restarting vault with a new primary seal
step "wait_for_leader_after_migration" {
module = module.vault_wait_for_leader
depends_on = [step.remove_primary_seal]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
timeout = 120 # seconds
vault_hosts = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.root_token
}
}
// Since we've restarted our cluster we might have a new leader and followers. Get the new IPs.
step "get_cluster_ips_after_migration" {
module = module.vault_get_cluster_ips
depends_on = [step.wait_for_leader_after_migration]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_hosts = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.root_token
}
}
// Make sure we unsealed
step "verify_vault_unsealed_after_migration" {
module = module.vault_verify_unsealed
depends_on = [step.wait_for_leader_after_migration]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_install_dir = local.vault_install_dir
vault_instances = step.create_vault_cluster_targets.hosts
}
}
// Wait for the seal rewrap to complete and verify that no entries failed
step "wait_for_seal_rewrap_after_migration" {
module = module.vault_wait_for_seal_rewrap
depends_on = [
step.wait_for_leader_after_migration,
step.verify_vault_unsealed_after_migration,
]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_hosts = step.create_vault_cluster_targets.hosts
vault_install_dir = local.vault_install_dir
vault_root_token = step.create_vault_cluster.root_token
}
}
// Make sure our data is still available after migration
step "verify_read_test_data_after_migration" {
module = module.vault_verify_read_data
depends_on = [step.wait_for_seal_rewrap_after_migration]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
node_public_ips = step.get_cluster_ips_after_migration.follower_public_ips
vault_install_dir = local.vault_install_dir
}
}
// Make sure we have our secondary seal type after migration
step "verify_seal_type_after_migration" {
// Don't run this on versions less than 1.16.0-beta1 until VAULT-21053 is fixed on prior branches.
skip_step = semverconstraint(var.vault_product_version, "<= 1.16.0-beta1")
module = module.verify_seal_type
depends_on = [step.wait_for_seal_rewrap_after_migration]
providers = {
enos = local.enos_provider[matrix.distro]
}
variables {
vault_install_dir = local.vault_install_dir
vault_hosts = step.create_vault_cluster_targets.hosts
seal_type = matrix.secondary_seal
}
}
output "audit_device_file_path" {
description = "The file path for the file audit device, if enabled"
value = step.create_vault_cluster.audit_device_file_path
}
output "cluster_name" {
description = "The Vault cluster name"
value = step.create_vault_cluster.cluster_name
}
output "hosts" {
description = "The Vault cluster target hosts"
value = step.create_vault_cluster.target_hosts
}
output "primary_seal_key_name" {
description = "The Vault cluster primary seal key name"
value = step.create_primary_seal_key.resource_name
}
output "private_ips" {
description = "The Vault cluster private IPs"
value = step.create_vault_cluster.private_ips
}
output "public_ips" {
description = "The Vault cluster public IPs"
value = step.create_vault_cluster.public_ips
}
output "root_token" {
description = "The Vault cluster root token"
value = step.create_vault_cluster.root_token
}
output "recovery_key_shares" {
description = "The Vault cluster recovery key shares"
value = step.create_vault_cluster.recovery_key_shares
}
output "recovery_keys_b64" {
description = "The Vault cluster recovery keys b64"
value = step.create_vault_cluster.recovery_keys_b64
}
output "recovery_keys_hex" {
description = "The Vault cluster recovery keys hex"
value = step.create_vault_cluster.recovery_keys_hex
}
output "secondary_seal_key_name" {
description = "The Vault cluster secondary seal key name"
value = step.create_secondary_seal_key.resource_name
}
output "unseal_keys_b64" {
description = "The Vault cluster unseal keys"
value = step.create_vault_cluster.unseal_keys_b64
}
output "unseal_keys_hex" {
description = "The Vault cluster unseal keys hex"
value = step.create_vault_cluster.unseal_keys_hex
}
}