v1.4.7/internal/backend/remote-state/oss/client_test.go

package oss

import (
	"fmt"
	"strings"
	"testing"
	"time"

	"bytes"
	"crypto/md5"

	"github.com/hashicorp/terraform/internal/backend"
	"github.com/hashicorp/terraform/internal/states/remote"
	"github.com/hashicorp/terraform/internal/states/statefile"
	"github.com/hashicorp/terraform/internal/states/statemgr"
)

// NOTE: Before running this testcase, please create a OTS instance called 'tf-oss-remote'
var RemoteTestUsedOTSEndpoint = "https://tf-oss-remote.cn-hangzhou.ots.aliyuncs.com"

func TestRemoteClient_impl(t *testing.T) {
	var _ remote.Client = new(RemoteClient)
	var _ remote.ClientLocker = new(RemoteClient)
}

func TestRemoteClient(t *testing.T) {
	testACC(t)
	bucketName := fmt.Sprintf("tf-remote-oss-test-%x", time.Now().Unix())
	path := "testState"

	b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
		"bucket":  bucketName,
		"prefix":  path,
		"encrypt": true,
	})).(*Backend)

	createOSSBucket(t, b.ossClient, bucketName)
	defer deleteOSSBucket(t, b.ossClient, bucketName)

	state, err := b.StateMgr(backend.DefaultStateName)
	if err != nil {
		t.Fatal(err)
	}

	remote.TestClient(t, state.(*remote.State).Client)
}

func TestRemoteClientLocks(t *testing.T) {
	testACC(t)
	bucketName := fmt.Sprintf("tf-remote-oss-test-%x", time.Now().Unix())
	tableName := fmt.Sprintf("tfRemoteTestForce%x", time.Now().Unix())
	path := "testState"

	b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
		"bucket":              bucketName,
		"prefix":              path,
		"encrypt":             true,
		"tablestore_table":    tableName,
		"tablestore_endpoint": RemoteTestUsedOTSEndpoint,
	})).(*Backend)

	b2 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
		"bucket":              bucketName,
		"prefix":              path,
		"encrypt":             true,
		"tablestore_table":    tableName,
		"tablestore_endpoint": RemoteTestUsedOTSEndpoint,
	})).(*Backend)

	createOSSBucket(t, b1.ossClient, bucketName)
	defer deleteOSSBucket(t, b1.ossClient, bucketName)
	createTablestoreTable(t, b1.otsClient, tableName)
	defer deleteTablestoreTable(t, b1.otsClient, tableName)

	s1, err := b1.StateMgr(backend.DefaultStateName)
	if err != nil {
		t.Fatal(err)
	}

	s2, err := b2.StateMgr(backend.DefaultStateName)
	if err != nil {
		t.Fatal(err)
	}

	remote.TestRemoteLocks(t, s1.(*remote.State).Client, s2.(*remote.State).Client)
}

// verify that the backend can handle more than one state in the same table
func TestRemoteClientLocks_multipleStates(t *testing.T) {
	testACC(t)
	bucketName := fmt.Sprintf("tf-remote-oss-test-force-%x", time.Now().Unix())
	tableName := fmt.Sprintf("tfRemoteTestForce%x", time.Now().Unix())
	path := "testState"

	b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
		"bucket":              bucketName,
		"prefix":              path,
		"encrypt":             true,
		"tablestore_table":    tableName,
		"tablestore_endpoint": RemoteTestUsedOTSEndpoint,
	})).(*Backend)

	b2 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
		"bucket":              bucketName,
		"prefix":              path,
		"encrypt":             true,
		"tablestore_table":    tableName,
		"tablestore_endpoint": RemoteTestUsedOTSEndpoint,
	})).(*Backend)

	createOSSBucket(t, b1.ossClient, bucketName)
	defer deleteOSSBucket(t, b1.ossClient, bucketName)
	createTablestoreTable(t, b1.otsClient, tableName)
	defer deleteTablestoreTable(t, b1.otsClient, tableName)

	s1, err := b1.StateMgr("s1")
	if err != nil {
		t.Fatal(err)
	}
	if _, err := s1.Lock(statemgr.NewLockInfo()); err != nil {
		t.Fatal("failed to get lock for s1:", err)
	}

	// s1 is now locked, s2 should not be locked as it's a different state file
	s2, err := b2.StateMgr("s2")
	if err != nil {
		t.Fatal(err)
	}
	if _, err := s2.Lock(statemgr.NewLockInfo()); err != nil {
		t.Fatal("failed to get lock for s2:", err)
	}
}

// verify that we can unlock a state with an existing lock
func TestRemoteForceUnlock(t *testing.T) {
	testACC(t)
	bucketName := fmt.Sprintf("tf-remote-oss-test-force-%x", time.Now().Unix())
	tableName := fmt.Sprintf("tfRemoteTestForce%x", time.Now().Unix())
	path := "testState"

	b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
		"bucket":              bucketName,
		"prefix":              path,
		"encrypt":             true,
		"tablestore_table":    tableName,
		"tablestore_endpoint": RemoteTestUsedOTSEndpoint,
	})).(*Backend)

	b2 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
		"bucket":              bucketName,
		"prefix":              path,
		"encrypt":             true,
		"tablestore_table":    tableName,
		"tablestore_endpoint": RemoteTestUsedOTSEndpoint,
	})).(*Backend)

	createOSSBucket(t, b1.ossClient, bucketName)
	defer deleteOSSBucket(t, b1.ossClient, bucketName)
	createTablestoreTable(t, b1.otsClient, tableName)
	defer deleteTablestoreTable(t, b1.otsClient, tableName)

	// first test with default
	s1, err := b1.StateMgr(backend.DefaultStateName)
	if err != nil {
		t.Fatal(err)
	}

	info := statemgr.NewLockInfo()
	info.Operation = "test"
	info.Who = "clientA"

	lockID, err := s1.Lock(info)
	if err != nil {
		t.Fatal("unable to get initial lock:", err)
	}

	// s1 is now locked, get the same state through s2 and unlock it
	s2, err := b2.StateMgr(backend.DefaultStateName)
	if err != nil {
		t.Fatal("failed to get default state to force unlock:", err)
	}

	if err := s2.Unlock(lockID); err != nil {
		t.Fatal("failed to force-unlock default state")
	}

	// now try the same thing with a named state
	// first test with default
	s1, err = b1.StateMgr("test")
	if err != nil {
		t.Fatal(err)
	}

	info = statemgr.NewLockInfo()
	info.Operation = "test"
	info.Who = "clientA"

	lockID, err = s1.Lock(info)
	if err != nil {
		t.Fatal("unable to get initial lock:", err)
	}

	// s1 is now locked, get the same state through s2 and unlock it
	s2, err = b2.StateMgr("test")
	if err != nil {
		t.Fatal("failed to get named state to force unlock:", err)
	}

	if err = s2.Unlock(lockID); err != nil {
		t.Fatal("failed to force-unlock named state")
	}
}

func TestRemoteClient_clientMD5(t *testing.T) {
	testACC(t)

	bucketName := fmt.Sprintf("tf-remote-oss-test-%x", time.Now().Unix())
	tableName := fmt.Sprintf("tfRemoteTestForce%x", time.Now().Unix())
	path := "testState"

	b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
		"bucket":              bucketName,
		"prefix":              path,
		"tablestore_table":    tableName,
		"tablestore_endpoint": RemoteTestUsedOTSEndpoint,
	})).(*Backend)

	createOSSBucket(t, b.ossClient, bucketName)
	defer deleteOSSBucket(t, b.ossClient, bucketName)
	createTablestoreTable(t, b.otsClient, tableName)
	defer deleteTablestoreTable(t, b.otsClient, tableName)

	s, err := b.StateMgr(backend.DefaultStateName)
	if err != nil {
		t.Fatal(err)
	}
	client := s.(*remote.State).Client.(*RemoteClient)

	sum := md5.Sum([]byte("test"))

	if err := client.putMD5(sum[:]); err != nil {
		t.Fatal(err)
	}

	getSum, err := client.getMD5()
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(getSum, sum[:]) {
		t.Fatalf("getMD5 returned the wrong checksum: expected %x, got %x", sum[:], getSum)
	}

	if err := client.deleteMD5(); err != nil {
		t.Fatal(err)
	}

	if getSum, err := client.getMD5(); err == nil {
		t.Fatalf("expected getMD5 error, got none. checksum: %x", getSum)
	}
}

// verify that a client won't return a state with an incorrect checksum.
func TestRemoteClient_stateChecksum(t *testing.T) {
	testACC(t)

	bucketName := fmt.Sprintf("tf-remote-oss-test-%x", time.Now().Unix())
	tableName := fmt.Sprintf("tfRemoteTestForce%x", time.Now().Unix())
	path := "testState"

	b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
		"bucket":              bucketName,
		"prefix":              path,
		"tablestore_table":    tableName,
		"tablestore_endpoint": RemoteTestUsedOTSEndpoint,
	})).(*Backend)

	createOSSBucket(t, b1.ossClient, bucketName)
	defer deleteOSSBucket(t, b1.ossClient, bucketName)
	createTablestoreTable(t, b1.otsClient, tableName)
	defer deleteTablestoreTable(t, b1.otsClient, tableName)

	s1, err := b1.StateMgr(backend.DefaultStateName)
	if err != nil {
		t.Fatal(err)
	}
	client1 := s1.(*remote.State).Client

	// create an old and new state version to persist
	s := statemgr.TestFullInitialState()
	sf := &statefile.File{State: s}
	var oldState bytes.Buffer
	if err := statefile.Write(sf, &oldState); err != nil {
		t.Fatal(err)
	}
	sf.Serial++
	var newState bytes.Buffer
	if err := statefile.Write(sf, &newState); err != nil {
		t.Fatal(err)
	}

	// Use b2 without a tablestore_table to bypass the lock table to write the state directly.
	// client2 will write the "incorrect" state, simulating oss eventually consistency delays
	b2 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
		"bucket": bucketName,
		"prefix": path,
	})).(*Backend)
	s2, err := b2.StateMgr(backend.DefaultStateName)
	if err != nil {
		t.Fatal(err)
	}
	client2 := s2.(*remote.State).Client

	// write the new state through client2 so that there is no checksum yet
	if err := client2.Put(newState.Bytes()); err != nil {
		t.Fatal(err)
	}

	// verify that we can pull a state without a checksum
	if _, err := client1.Get(); err != nil {
		t.Fatal(err)
	}

	// write the new state back with its checksum
	if err := client1.Put(newState.Bytes()); err != nil {
		t.Fatal(err)
	}

	// put an empty state in place to check for panics during get
	if err := client2.Put([]byte{}); err != nil {
		t.Fatal(err)
	}

	// remove the timeouts so we can fail immediately
	origTimeout := consistencyRetryTimeout
	origInterval := consistencyRetryPollInterval
	defer func() {
		consistencyRetryTimeout = origTimeout
		consistencyRetryPollInterval = origInterval
	}()
	consistencyRetryTimeout = 0
	consistencyRetryPollInterval = 0

	// fetching an empty state through client1 should now error out due to a
	// mismatched checksum.
	if _, err := client1.Get(); !strings.HasPrefix(err.Error(), errBadChecksumFmt[:80]) {
		t.Fatalf("expected state checksum error: got %s", err)
	}

	// put the old state in place of the new, without updating the checksum
	if err := client2.Put(oldState.Bytes()); err != nil {
		t.Fatal(err)
	}

	// fetching the wrong state through client1 should now error out due to a
	// mismatched checksum.
	if _, err := client1.Get(); !strings.HasPrefix(err.Error(), errBadChecksumFmt[:80]) {
		t.Fatalf("expected state checksum error: got %s", err)
	}

	// update the state with the correct one after we Get again
	testChecksumHook = func() {
		if err := client2.Put(newState.Bytes()); err != nil {
			t.Fatal(err)
		}
		testChecksumHook = nil
	}

	consistencyRetryTimeout = origTimeout

	// this final Get will fail to fail the checksum verification, the above
	// callback will update the state with the correct version, and Get should
	// retry automatically.
	if _, err := client1.Get(); err != nil {
		t.Fatal(err)
	}
}