| // Copyright (c) HashiCorp, Inc. |
| // SPDX-License-Identifier: MPL-2.0 |
| |
| package views |
| |
| import ( |
| "encoding/xml" |
| "fmt" |
| "io/ioutil" |
| "sort" |
| "strings" |
| |
| "github.com/hashicorp/terraform/internal/command/arguments" |
| "github.com/hashicorp/terraform/internal/command/format" |
| "github.com/hashicorp/terraform/internal/moduletest" |
| "github.com/hashicorp/terraform/internal/terminal" |
| "github.com/hashicorp/terraform/internal/tfdiags" |
| "github.com/mitchellh/colorstring" |
| ) |
| |
| // Test is the view interface for the "terraform test" command. |
| type Test interface { |
| // Results presents the given test results. |
| Results(map[string]*moduletest.Suite) tfdiags.Diagnostics |
| |
| // Diagnostics is for reporting warnings or errors that occurred with the |
| // mechanics of running tests. For this command in particular, some |
| // errors are considered to be test failures rather than mechanism failures, |
| // and so those will be reported via Results rather than via Diagnostics. |
| Diagnostics(tfdiags.Diagnostics) |
| } |
| |
| // NewTest returns an implementation of Test configured to respect the |
| // settings described in the given arguments. |
| func NewTest(base *View, args arguments.TestOutput) Test { |
| return &testHuman{ |
| streams: base.streams, |
| showDiagnostics: base.Diagnostics, |
| colorize: base.colorize, |
| junitXMLFile: args.JUnitXMLFile, |
| } |
| } |
| |
| type testHuman struct { |
| // This is the subset of functionality we need from the base view. |
| streams *terminal.Streams |
| showDiagnostics func(diags tfdiags.Diagnostics) |
| colorize *colorstring.Colorize |
| |
| // If junitXMLFile is not empty then results will be written to |
| // the given file path in addition to the usual output. |
| junitXMLFile string |
| } |
| |
| func (v *testHuman) Results(results map[string]*moduletest.Suite) tfdiags.Diagnostics { |
| var diags tfdiags.Diagnostics |
| |
| // FIXME: Due to how this prototype command evolved concurrently with |
| // establishing the idea of command views, the handling of JUnit output |
| // as part of the "human" view rather than as a separate view in its |
| // own right is a little odd and awkward. We should refactor this |
| // prior to making "terraform test" a real supported command to make |
| // it be structured more like the other commands that use the views |
| // package. |
| |
| v.humanResults(results) |
| |
| if v.junitXMLFile != "" { |
| moreDiags := v.junitXMLResults(results, v.junitXMLFile) |
| diags = diags.Append(moreDiags) |
| } |
| |
| return diags |
| } |
| |
| func (v *testHuman) Diagnostics(diags tfdiags.Diagnostics) { |
| if len(diags) == 0 { |
| return |
| } |
| v.showDiagnostics(diags) |
| } |
| |
| func (v *testHuman) humanResults(results map[string]*moduletest.Suite) { |
| failCount := 0 |
| width := v.streams.Stderr.Columns() |
| |
| suiteNames := make([]string, 0, len(results)) |
| for suiteName := range results { |
| suiteNames = append(suiteNames, suiteName) |
| } |
| sort.Strings(suiteNames) |
| for _, suiteName := range suiteNames { |
| suite := results[suiteName] |
| |
| componentNames := make([]string, 0, len(suite.Components)) |
| for componentName := range suite.Components { |
| componentNames = append(componentNames, componentName) |
| } |
| for _, componentName := range componentNames { |
| component := suite.Components[componentName] |
| |
| assertionNames := make([]string, 0, len(component.Assertions)) |
| for assertionName := range component.Assertions { |
| assertionNames = append(assertionNames, assertionName) |
| } |
| sort.Strings(assertionNames) |
| |
| for _, assertionName := range assertionNames { |
| assertion := component.Assertions[assertionName] |
| |
| fullName := fmt.Sprintf("%s.%s.%s", suiteName, componentName, assertionName) |
| if strings.HasPrefix(componentName, "(") { |
| // parenthesis-prefixed components are placeholders that |
| // the test harness generates to represent problems that |
| // prevented checking any assertions at all, so we'll |
| // just hide them and show the suite name. |
| fullName = suiteName |
| } |
| headingExtra := fmt.Sprintf("%s (%s)", fullName, assertion.Description) |
| |
| switch assertion.Outcome { |
| case moduletest.Failed: |
| // Failed means that the assertion was successfully |
| // excecuted but that the assertion condition didn't hold. |
| v.eprintRuleHeading("yellow", "Failed", headingExtra) |
| |
| case moduletest.Error: |
| // Error means that the system encountered an unexpected |
| // error when trying to evaluate the assertion. |
| v.eprintRuleHeading("red", "Error", headingExtra) |
| |
| default: |
| // We don't do anything for moduletest.Passed or |
| // moduletest.Skipped. Perhaps in future we'll offer a |
| // -verbose option to include information about those. |
| continue |
| } |
| failCount++ |
| |
| if len(assertion.Message) > 0 { |
| dispMsg := format.WordWrap(assertion.Message, width) |
| v.streams.Eprintln(dispMsg) |
| } |
| if len(assertion.Diagnostics) > 0 { |
| // We'll do our own writing of the diagnostics in this |
| // case, rather than using v.Diagnostics, because we |
| // specifically want all of these diagnostics to go to |
| // Stderr along with all of the other output we've |
| // generated. |
| for _, diag := range assertion.Diagnostics { |
| diagStr := format.Diagnostic(diag, nil, v.colorize, width) |
| v.streams.Eprint(diagStr) |
| } |
| } |
| } |
| } |
| } |
| |
| if failCount > 0 { |
| // If we've printed at least one failure then we'll have printed at |
| // least one horizontal rule across the terminal, and so we'll balance |
| // that with another horizontal rule. |
| if width > 1 { |
| rule := strings.Repeat("─", width-1) |
| v.streams.Eprintln(v.colorize.Color("[dark_gray]" + rule)) |
| } |
| } |
| |
| if failCount == 0 { |
| if len(results) > 0 { |
| // This is not actually an error, but it's convenient if all of our |
| // result output goes to the same stream for when this is running in |
| // automation that might be gathering this output via a pipe. |
| v.streams.Eprint(v.colorize.Color("[bold][green]Success![reset] All of the test assertions passed.\n\n")) |
| } else { |
| v.streams.Eprint(v.colorize.Color("[bold][yellow]No tests defined.[reset] This module doesn't have any test suites to run.\n\n")) |
| } |
| } |
| |
| // Try to flush any buffering that might be happening. (This isn't always |
| // successful, depending on what sort of fd Stderr is connected to.) |
| v.streams.Stderr.File.Sync() |
| } |
| |
| func (v *testHuman) junitXMLResults(results map[string]*moduletest.Suite, filename string) tfdiags.Diagnostics { |
| var diags tfdiags.Diagnostics |
| |
| // "JUnit XML" is a file format that has become a de-facto standard for |
| // test reporting tools but that is not formally specified anywhere, and |
| // so each producer and consumer implementation unfortunately tends to |
| // differ in certain ways from others. |
| // With that in mind, this is a best effort sort of thing aimed at being |
| // broadly compatible with various consumers, but it's likely that |
| // some consumers will present these results better than others. |
| // This implementation is based mainly on the pseudo-specification of the |
| // format curated here, based on the Jenkins parser implementation: |
| // https://llg.cubic.org/docs/junit/ |
| |
| // An "Outcome" represents one of the various XML elements allowed inside |
| // a testcase element to indicate the test outcome. |
| type Outcome struct { |
| Message string `xml:"message,omitempty"` |
| } |
| |
| // TestCase represents an individual test case as part of a suite. Note |
| // that a JUnit XML incorporates both the "component" and "assertion" |
| // levels of our model: we pretend that component is a class name and |
| // assertion is a method name in order to match with the Java-flavored |
| // expectations of JUnit XML, which are hopefully close enough to get |
| // a test result rendering that's useful to humans. |
| type TestCase struct { |
| AssertionName string `xml:"name"` |
| ComponentName string `xml:"classname"` |
| |
| // These fields represent the different outcomes of a TestCase. Only one |
| // of these should be populated in each TestCase; this awkward |
| // structure is just to make this play nicely with encoding/xml's |
| // expecatations. |
| Skipped *Outcome `xml:"skipped,omitempty"` |
| Error *Outcome `xml:"error,omitempty"` |
| Failure *Outcome `xml:"failure,omitempty"` |
| |
| Stderr string `xml:"system-out,omitempty"` |
| } |
| |
| // TestSuite represents an individual test suite, of potentially many |
| // in a JUnit XML document. |
| type TestSuite struct { |
| Name string `xml:"name"` |
| TotalCount int `xml:"tests"` |
| SkippedCount int `xml:"skipped"` |
| ErrorCount int `xml:"errors"` |
| FailureCount int `xml:"failures"` |
| Cases []*TestCase `xml:"testcase"` |
| } |
| |
| // TestSuites represents the root element of the XML document. |
| type TestSuites struct { |
| XMLName struct{} `xml:"testsuites"` |
| ErrorCount int `xml:"errors"` |
| FailureCount int `xml:"failures"` |
| TotalCount int `xml:"tests"` |
| Suites []*TestSuite `xml:"testsuite"` |
| } |
| |
| xmlSuites := TestSuites{} |
| suiteNames := make([]string, 0, len(results)) |
| for suiteName := range results { |
| suiteNames = append(suiteNames, suiteName) |
| } |
| sort.Strings(suiteNames) |
| for _, suiteName := range suiteNames { |
| suite := results[suiteName] |
| |
| xmlSuite := &TestSuite{ |
| Name: suiteName, |
| } |
| xmlSuites.Suites = append(xmlSuites.Suites, xmlSuite) |
| |
| componentNames := make([]string, 0, len(suite.Components)) |
| for componentName := range suite.Components { |
| componentNames = append(componentNames, componentName) |
| } |
| for _, componentName := range componentNames { |
| component := suite.Components[componentName] |
| |
| assertionNames := make([]string, 0, len(component.Assertions)) |
| for assertionName := range component.Assertions { |
| assertionNames = append(assertionNames, assertionName) |
| } |
| sort.Strings(assertionNames) |
| |
| for _, assertionName := range assertionNames { |
| assertion := component.Assertions[assertionName] |
| xmlSuites.TotalCount++ |
| xmlSuite.TotalCount++ |
| |
| xmlCase := &TestCase{ |
| ComponentName: componentName, |
| AssertionName: assertionName, |
| } |
| xmlSuite.Cases = append(xmlSuite.Cases, xmlCase) |
| |
| switch assertion.Outcome { |
| case moduletest.Pending: |
| // We represent "pending" cases -- cases blocked by |
| // upstream errors -- as if they were "skipped" in JUnit |
| // terms, because we didn't actually check them and so |
| // can't say whether they succeeded or not. |
| xmlSuite.SkippedCount++ |
| xmlCase.Skipped = &Outcome{ |
| Message: assertion.Message, |
| } |
| case moduletest.Failed: |
| xmlSuites.FailureCount++ |
| xmlSuite.FailureCount++ |
| xmlCase.Failure = &Outcome{ |
| Message: assertion.Message, |
| } |
| case moduletest.Error: |
| xmlSuites.ErrorCount++ |
| xmlSuite.ErrorCount++ |
| xmlCase.Error = &Outcome{ |
| Message: assertion.Message, |
| } |
| |
| // We'll also include the diagnostics in the "stderr" |
| // portion of the output, so they'll hopefully be visible |
| // in a test log viewer in JUnit-XML-Consuming CI systems. |
| var buf strings.Builder |
| for _, diag := range assertion.Diagnostics { |
| diagStr := format.DiagnosticPlain(diag, nil, 68) |
| buf.WriteString(diagStr) |
| } |
| xmlCase.Stderr = buf.String() |
| } |
| |
| } |
| } |
| } |
| |
| xmlOut, err := xml.MarshalIndent(&xmlSuites, "", " ") |
| if err != nil { |
| // If marshalling fails then that's a bug in the code above, |
| // because we should always be producing a value that is |
| // accepted by encoding/xml. |
| panic(fmt.Sprintf("invalid values to marshal as JUnit XML: %s", err)) |
| } |
| |
| err = ioutil.WriteFile(filename, xmlOut, 0644) |
| if err != nil { |
| diags = diags.Append(tfdiags.Sourceless( |
| tfdiags.Error, |
| "Failed to write JUnit XML file", |
| fmt.Sprintf( |
| "Could not create %s to record the test results in JUnit XML format: %s.", |
| filename, |
| err, |
| ), |
| )) |
| } |
| |
| return diags |
| } |
| |
| func (v *testHuman) eprintRuleHeading(color, prefix, extra string) { |
| const lineCell string = "─" |
| textLen := len(prefix) + len(": ") + len(extra) |
| spacingLen := 2 |
| leftLineLen := 3 |
| |
| rightLineLen := 0 |
| width := v.streams.Stderr.Columns() |
| if (textLen + spacingLen + leftLineLen) < (width - 1) { |
| // (we allow an extra column at the end because some terminals can't |
| // print in the final column without wrapping to the next line) |
| rightLineLen = width - (textLen + spacingLen + leftLineLen) - 1 |
| } |
| |
| colorCode := "[" + color + "]" |
| |
| // We'll prepare what we're going to print in memory first, so that we can |
| // send it all to stderr in one write in case other programs are also |
| // concurrently trying to write to the terminal for some reason. |
| var buf strings.Builder |
| buf.WriteString(v.colorize.Color(colorCode + strings.Repeat(lineCell, leftLineLen))) |
| buf.WriteByte(' ') |
| buf.WriteString(v.colorize.Color("[bold]" + colorCode + prefix + ":")) |
| buf.WriteByte(' ') |
| buf.WriteString(extra) |
| if rightLineLen > 0 { |
| buf.WriteByte(' ') |
| buf.WriteString(v.colorize.Color(colorCode + strings.Repeat(lineCell, rightLineLen))) |
| } |
| v.streams.Eprintln(buf.String()) |
| } |
