v1.16.5/internal/command/junit/junit.go

// Copyright IBM Corp. 2014, 2026
// SPDX-License-Identifier: BUSL-1.1
package junit

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"maps"
	"os"
	"slices"
	"strconv"
	"strings"

	"github.com/hashicorp/hcl/v2"
	"github.com/hashicorp/terraform/internal/command/format"
	"github.com/hashicorp/terraform/internal/configs/configload"
	"github.com/hashicorp/terraform/internal/moduletest"
	"github.com/hashicorp/terraform/internal/tfdiags"
)

// TestJUnitXMLFile produces a JUnit XML file at the conclusion of a test
// run, summarizing the outcome of the test in a form that can then be
// interpreted by tools which render JUnit XML result reports.
//
// The de-facto convention for JUnit XML is for it to be emitted as a separate
// file as a complement to human-oriented output, rather than _instead of_
// human-oriented output. To meet that expectation the method [TestJUnitXMLFile.Save]
// should be called at the same time as the test's view reaches its "Conclusion" event.
// If that event isn't reached for any reason then no file should be created at
// all, which JUnit XML-consuming tools tend to expect as an outcome of a
// catastrophically-errored test suite.
//
// TestJUnitXMLFile implements the JUnit interface, which allows creation of a local
// file that contains a description of a completed test suite. It is intended only
// for use in conjunction with a View that provides the streaming output of ongoing
// testing events.

type TestJUnitXMLFile struct {
	filename string

	// A config loader is required to access sources, which are used with diagnostics to create XML content
	configLoader *configload.Loader

	// A pointer to the containing test suite runner is needed to monitor details like the command being stopped
	testSuiteRunner moduletest.TestSuiteRunner
}

type JUnit interface {
	Save(*moduletest.Suite) tfdiags.Diagnostics
}

var _ JUnit = (*TestJUnitXMLFile)(nil)

// NewTestJUnitXML returns a [Test] implementation that will, when asked to
// report "conclusion", write a JUnit XML report to the given filename.
//
// If the file already exists then this view will silently overwrite it at the
// point of being asked to write a conclusion. Otherwise it will create the
// file at that time. If creating or overwriting the file fails, a subsequent
// call to method Err will return information about the problem.
func NewTestJUnitXMLFile(filename string, configLoader *configload.Loader, testSuiteRunner moduletest.TestSuiteRunner) *TestJUnitXMLFile {
	return &TestJUnitXMLFile{
		filename:        filename,
		configLoader:    configLoader,
		testSuiteRunner: testSuiteRunner,
	}
}

// Save takes in a test suite, generates JUnit XML summarising the test results,
// and saves the content to the filename specified by user
func (v *TestJUnitXMLFile) Save(suite *moduletest.Suite) tfdiags.Diagnostics {
	var diags tfdiags.Diagnostics

	// Prepare XML content
	sources := v.configLoader.Parser().Sources()
	xmlSrc, err := junitXMLTestReport(suite, v.testSuiteRunner.IsStopped(), sources)
	if err != nil {
		diags = diags.Append(&hcl.Diagnostic{
			Severity: hcl.DiagError,
			Summary:  "error generating JUnit XML test output",
			Detail:   err.Error(),
		})
		return diags
	}

	// Save XML to the specified path
	saveDiags := v.save(xmlSrc)
	diags = append(diags, saveDiags...)

	return diags

}

func (v *TestJUnitXMLFile) save(xmlSrc []byte) tfdiags.Diagnostics {
	var diags tfdiags.Diagnostics
	err := os.WriteFile(v.filename, xmlSrc, 0660)
	if err != nil {
		diags = diags.Append(&hcl.Diagnostic{
			Severity: hcl.DiagError,
			Summary:  fmt.Sprintf("error saving JUnit XML to file %q", v.filename),
			Detail:   err.Error(),
		})
		return diags
	}

	return nil
}

type withMessage struct {
	Message string `xml:"message,attr,omitempty"`
	Body    string `xml:",cdata"`
}

type testCase struct {
	Name      string       `xml:"name,attr"`
	Classname string       `xml:"classname,attr"`
	Skipped   *withMessage `xml:"skipped,omitempty"`
	Failure   *withMessage `xml:"failure,omitempty"`
	Error     *withMessage `xml:"error,omitempty"`
	Stderr    *withMessage `xml:"system-err,omitempty"`

	// RunTime is the time spent executing the run associated
	// with this test case, in seconds with the fractional component
	// representing partial seconds.
	//
	// We assume here that it's not practically possible for an
	// execution to take literally zero fractional seconds at
	// the accuracy we're using here (nanoseconds converted into
	// floating point seconds) and so use zero to represent
	// "not known", and thus omit that case. (In practice many
	// JUnit XML consumers treat the absense of this attribute
	// as zero anyway.)
	RunTime   float64 `xml:"time,attr,omitempty"`
	Timestamp string  `xml:"timestamp,attr,omitempty"`
}

func junitXMLTestReport(suite *moduletest.Suite, suiteRunnerStopped bool, sources map[string][]byte) ([]byte, error) {
	var buf bytes.Buffer
	enc := xml.NewEncoder(&buf)
	enc.EncodeToken(xml.ProcInst{
		Target: "xml",
		Inst:   []byte(`version="1.0" encoding="UTF-8"`),
	})
	enc.Indent("", "  ")

	// Some common element/attribute names we'll use repeatedly below.
	suitesName := xml.Name{Local: "testsuites"}
	suiteName := xml.Name{Local: "testsuite"}
	caseName := xml.Name{Local: "testcase"}
	nameName := xml.Name{Local: "name"}
	testsName := xml.Name{Local: "tests"}
	skippedName := xml.Name{Local: "skipped"}
	failuresName := xml.Name{Local: "failures"}
	errorsName := xml.Name{Local: "errors"}

	enc.EncodeToken(xml.StartElement{Name: suitesName})

	// Sort the file names to ensure consistent ordering in XML
	for _, name := range slices.Sorted(maps.Keys(suite.Files)) {
		file := suite.Files[name]
		// Each test file is modelled as a "test suite".

		// First we'll count the number of tests and number of failures/errors
		// for the suite-level summary.
		totalTests := len(file.Runs)
		totalFails := 0
		totalErrs := 0
		totalSkipped := 0
		for _, run := range file.Runs {
			switch run.Status {
			case moduletest.Skip:
				totalSkipped++
			case moduletest.Fail:
				totalFails++
			case moduletest.Error:
				totalErrs++
			}
		}
		enc.EncodeToken(xml.StartElement{
			Name: suiteName,
			Attr: []xml.Attr{
				{Name: nameName, Value: file.Name},
				{Name: testsName, Value: strconv.Itoa(totalTests)},
				{Name: skippedName, Value: strconv.Itoa(totalSkipped)},
				{Name: failuresName, Value: strconv.Itoa(totalFails)},
				{Name: errorsName, Value: strconv.Itoa(totalErrs)},
			},
		})

		// Check if there are file-level errors that will be reported at suite level
		hasFileLevelErrors := file.Status == moduletest.Error && file.Diagnostics.HasErrors()

		for i, run := range file.Runs {
			// Each run is a "test case".

			testCase := testCase{
				Name: run.Name,

				// We treat the test scenario filename as the "class name",
				// implying that the run name is the "method name", just
				// because that seems to inspire more useful rendering in
				// some consumers of JUnit XML that were designed for
				// Java-shaped languages.
				Classname: file.Name,
			}
			if execMeta := run.ExecutionMeta; execMeta != nil {
				testCase.RunTime = execMeta.Duration.Seconds()
				testCase.Timestamp = execMeta.StartTimestamp()
			}

			// Depending on run status, add either of: "skipped", "failure", or "error" elements
			switch run.Status {
			case moduletest.Skip:
				testCase.Skipped = skipDetails(i, file, suiteRunnerStopped, hasFileLevelErrors)

			case moduletest.Fail:
				// When the test fails we only use error diags that originate from failing assertions
				var failedAssertions tfdiags.Diagnostics
				for _, d := range run.Diagnostics {
					if tfdiags.DiagnosticCausedByTestFailure(d) {
						failedAssertions = failedAssertions.Append(d)
					}
				}

				testCase.Failure = &withMessage{
					Message: failureMessage(failedAssertions, len(run.Config.CheckRules)),
					Body:    getDiagString(failedAssertions, sources),
				}

			case moduletest.Error:
				// When the test errors we use all diags with Error severity
				var errDiags tfdiags.Diagnostics
				for _, d := range run.Diagnostics {
					if d.Severity() == tfdiags.Error {
						errDiags = errDiags.Append(d)
					}
				}

				testCase.Error = &withMessage{
					Message: "Encountered an error",
					Body:    getDiagString(errDiags, sources),
				}
			}

			// Determine if there are diagnostics left unused by the switch block above
			// that should be included in the "system-err" element
			if len(run.Diagnostics) > 0 {
				var systemErrDiags tfdiags.Diagnostics

				if run.Status == moduletest.Error && run.Diagnostics.HasWarnings() {
					// If the test case errored, then all Error diags are in the "error" element
					// Therefore we'd only need to include warnings in "system-err"
					systemErrDiags = run.Diagnostics.Warnings()
				}

				if run.Status != moduletest.Error {
					// If a test hasn't errored then we need to find all diagnostics that aren't due
					// to a failing assertion in a test (these are already displayed in the "failure" element)

					// Collect diags not due to failed assertions, both errors and warnings
					for _, d := range run.Diagnostics {
						if !tfdiags.DiagnosticCausedByTestFailure(d) {
							systemErrDiags = systemErrDiags.Append(d)
						}
					}
				}

				if len(systemErrDiags) > 0 {
					testCase.Stderr = &withMessage{
						Body: getDiagString(systemErrDiags, sources),
					}
				}
			}

			enc.EncodeElement(&testCase, xml.StartElement{
				Name: caseName,
			})
		}

		// Add suite-level system-err if there are file-level errors
		if hasFileLevelErrors {
			systemErr := &withMessage{
				Body: getDiagString(file.Diagnostics, sources),
			}
			enc.EncodeElement(systemErr, xml.StartElement{
				Name: xml.Name{Local: "system-err"},
			})
		}

		enc.EncodeToken(xml.EndElement{Name: suiteName})
	}
	enc.EncodeToken(xml.EndElement{Name: suitesName})
	enc.Close()
	return buf.Bytes(), nil
}

func failureMessage(failedAssertions tfdiags.Diagnostics, checkCount int) string {
	if len(failedAssertions) == 0 {
		return ""
	}

	if len(failedAssertions) == 1 {
		// Slightly different format if only single assertion failure
		return fmt.Sprintf("%d of %d assertions failed: %s", len(failedAssertions), checkCount, failedAssertions[0].Description().Detail)
	}

	// Handle multiple assertion failures
	return fmt.Sprintf("%d of %d assertions failed, including: %s", len(failedAssertions), checkCount, failedAssertions[0].Description().Detail)
}

// skipDetails checks data about the test suite, file and runs to determine why a given run was skipped
// Test can be skipped due to:
// 1. terraform test recieving an interrupt from users; all unstarted tests will be skipped
// 2. A previous run in a file has failed, causing subsequent run blocks to be skipped
// 3. File-level errors (e.g., invalid variable references) causing all tests to be skipped
// The returned value is used to set content in the "skipped" element
func skipDetails(runIndex int, file *moduletest.File, suiteStopped bool, hasFileLevelErrors bool) *withMessage {
	if suiteStopped {
		// Test suite experienced an interrupt
		// This block only handles graceful Stop interrupts, as Cancel interrupts will prevent a JUnit file being produced at all
		return &withMessage{
			Message: "Testcase skipped due to an interrupt",
			Body:    "Terraform received an interrupt and stopped gracefully. This caused all remaining testcases to be skipped",
		}
	}

	if file.Status == moduletest.Error {
		// Overall test file marked as errored in the context of a skipped test means tests have been skipped after
		// a previous test/run blocks has errored out
		for i := runIndex; i >= 0; i-- {
			if file.Runs[i].Status == moduletest.Error {
				// Skipped due to error in previous run within the file
				return &withMessage{
					Message: "Testcase skipped due to a previous testcase error",
					Body:    fmt.Sprintf("Previous testcase %q ended in error, which caused the remaining tests in the file to be skipped", file.Runs[i].Name),
				}
			}
		}

		// Check for file-level error diagnostics that caused tests to be skipped
		// Note: Full diagnostic details are included in suite-level <system-err> element
		if hasFileLevelErrors {
			return &withMessage{
				Message: "Testcase skipped due to file-level errors",
			}
		}
	}

	// Unhandled case: This results in <skipped></skipped> with no attributes or body
	return &withMessage{}
}

func getDiagString(diags tfdiags.Diagnostics, sources map[string][]byte) string {
	var diagsStr strings.Builder
	for _, d := range diags {
		diagsStr.WriteString(format.DiagnosticPlain(d, sources, 80))
	}
	return diagsStr.String()
}