| package registry |
| |
| import ( |
| "context" |
| "encoding/json" |
| "fmt" |
| "io/ioutil" |
| "log" |
| "net/http" |
| "net/url" |
| "os" |
| "path" |
| "strconv" |
| "strings" |
| "time" |
| |
| "github.com/hashicorp/go-retryablehttp" |
| svchost "github.com/hashicorp/terraform-svchost" |
| "github.com/hashicorp/terraform-svchost/disco" |
| "github.com/hashicorp/terraform/internal/httpclient" |
| "github.com/hashicorp/terraform/internal/logging" |
| "github.com/hashicorp/terraform/internal/registry/regsrc" |
| "github.com/hashicorp/terraform/internal/registry/response" |
| "github.com/hashicorp/terraform/version" |
| ) |
| |
| const ( |
| xTerraformGet = "X-Terraform-Get" |
| xTerraformVersion = "X-Terraform-Version" |
| modulesServiceID = "modules.v1" |
| providersServiceID = "providers.v1" |
| |
| // registryDiscoveryRetryEnvName is the name of the environment variable that |
| // can be configured to customize number of retries for module and provider |
| // discovery requests with the remote registry. |
| registryDiscoveryRetryEnvName = "TF_REGISTRY_DISCOVERY_RETRY" |
| defaultRetry = 1 |
| |
| // registryClientTimeoutEnvName is the name of the environment variable that |
| // can be configured to customize the timeout duration (seconds) for module |
| // and provider discovery with the remote registry. |
| registryClientTimeoutEnvName = "TF_REGISTRY_CLIENT_TIMEOUT" |
| |
| // defaultRequestTimeout is the default timeout duration for requests to the |
| // remote registry. |
| defaultRequestTimeout = 10 * time.Second |
| ) |
| |
| var ( |
| tfVersion = version.String() |
| |
| discoveryRetry int |
| requestTimeout time.Duration |
| ) |
| |
| func init() { |
| configureDiscoveryRetry() |
| configureRequestTimeout() |
| } |
| |
| // Client provides methods to query Terraform Registries. |
| type Client struct { |
| // this is the client to be used for all requests. |
| client *retryablehttp.Client |
| |
| // services is a required *disco.Disco, which may have services and |
| // credentials pre-loaded. |
| services *disco.Disco |
| } |
| |
| // NewClient returns a new initialized registry client. |
| func NewClient(services *disco.Disco, client *http.Client) *Client { |
| if services == nil { |
| services = disco.New() |
| } |
| |
| if client == nil { |
| client = httpclient.New() |
| client.Timeout = requestTimeout |
| } |
| retryableClient := retryablehttp.NewClient() |
| retryableClient.HTTPClient = client |
| retryableClient.RetryMax = discoveryRetry |
| retryableClient.RequestLogHook = requestLogHook |
| retryableClient.ErrorHandler = maxRetryErrorHandler |
| |
| logOutput := logging.LogOutput() |
| retryableClient.Logger = log.New(logOutput, "", log.Flags()) |
| |
| services.Transport = retryableClient.HTTPClient.Transport |
| |
| services.SetUserAgent(httpclient.TerraformUserAgent(version.String())) |
| |
| return &Client{ |
| client: retryableClient, |
| services: services, |
| } |
| } |
| |
| // Discover queries the host, and returns the url for the registry. |
| func (c *Client) Discover(host svchost.Hostname, serviceID string) (*url.URL, error) { |
| service, err := c.services.DiscoverServiceURL(host, serviceID) |
| if err != nil { |
| return nil, &ServiceUnreachableError{err} |
| } |
| if !strings.HasSuffix(service.Path, "/") { |
| service.Path += "/" |
| } |
| return service, nil |
| } |
| |
| // ModuleVersions queries the registry for a module, and returns the available versions. |
| func (c *Client) ModuleVersions(ctx context.Context, module *regsrc.Module) (*response.ModuleVersions, error) { |
| host, err := module.SvcHost() |
| if err != nil { |
| return nil, err |
| } |
| |
| service, err := c.Discover(host, modulesServiceID) |
| if err != nil { |
| return nil, err |
| } |
| |
| p, err := url.Parse(path.Join(module.Module(), "versions")) |
| if err != nil { |
| return nil, err |
| } |
| |
| service = service.ResolveReference(p) |
| |
| log.Printf("[DEBUG] fetching module versions from %q", service) |
| |
| req, err := retryablehttp.NewRequest("GET", service.String(), nil) |
| if err != nil { |
| return nil, err |
| } |
| req = req.WithContext(ctx) |
| |
| c.addRequestCreds(host, req.Request) |
| req.Header.Set(xTerraformVersion, tfVersion) |
| |
| resp, err := c.client.Do(req) |
| if err != nil { |
| return nil, err |
| } |
| defer resp.Body.Close() |
| |
| switch resp.StatusCode { |
| case http.StatusOK: |
| // OK |
| case http.StatusNotFound: |
| return nil, &errModuleNotFound{addr: module} |
| default: |
| return nil, fmt.Errorf("error looking up module versions: %s", resp.Status) |
| } |
| |
| var versions response.ModuleVersions |
| |
| dec := json.NewDecoder(resp.Body) |
| if err := dec.Decode(&versions); err != nil { |
| return nil, err |
| } |
| |
| for _, mod := range versions.Modules { |
| for _, v := range mod.Versions { |
| log.Printf("[DEBUG] found available version %q for %s", v.Version, mod.Source) |
| } |
| } |
| |
| return &versions, nil |
| } |
| |
| func (c *Client) addRequestCreds(host svchost.Hostname, req *http.Request) { |
| creds, err := c.services.CredentialsForHost(host) |
| if err != nil { |
| log.Printf("[WARN] Failed to get credentials for %s: %s (ignoring)", host, err) |
| return |
| } |
| |
| if creds != nil { |
| creds.PrepareRequest(req) |
| } |
| } |
| |
| // ModuleLocation find the download location for a specific version module. |
| // This returns a string, because the final location may contain special go-getter syntax. |
| func (c *Client) ModuleLocation(ctx context.Context, module *regsrc.Module, version string) (string, error) { |
| host, err := module.SvcHost() |
| if err != nil { |
| return "", err |
| } |
| |
| service, err := c.Discover(host, modulesServiceID) |
| if err != nil { |
| return "", err |
| } |
| |
| var p *url.URL |
| if version == "" { |
| p, err = url.Parse(path.Join(module.Module(), "download")) |
| } else { |
| p, err = url.Parse(path.Join(module.Module(), version, "download")) |
| } |
| if err != nil { |
| return "", err |
| } |
| download := service.ResolveReference(p) |
| |
| log.Printf("[DEBUG] looking up module location from %q", download) |
| |
| req, err := retryablehttp.NewRequest("GET", download.String(), nil) |
| if err != nil { |
| return "", err |
| } |
| |
| req = req.WithContext(ctx) |
| |
| c.addRequestCreds(host, req.Request) |
| req.Header.Set(xTerraformVersion, tfVersion) |
| |
| resp, err := c.client.Do(req) |
| if err != nil { |
| return "", err |
| } |
| defer resp.Body.Close() |
| |
| // there should be no body, but save it for logging |
| body, err := ioutil.ReadAll(resp.Body) |
| if err != nil { |
| return "", fmt.Errorf("error reading response body from registry: %s", err) |
| } |
| |
| switch resp.StatusCode { |
| case http.StatusOK, http.StatusNoContent: |
| // OK |
| case http.StatusNotFound: |
| return "", fmt.Errorf("module %q version %q not found", module, version) |
| default: |
| // anything else is an error: |
| return "", fmt.Errorf("error getting download location for %q: %s resp:%s", module, resp.Status, body) |
| } |
| |
| // the download location is in the X-Terraform-Get header |
| location := resp.Header.Get(xTerraformGet) |
| if location == "" { |
| return "", fmt.Errorf("failed to get download URL for %q: %s resp:%s", module, resp.Status, body) |
| } |
| |
| // If location looks like it's trying to be a relative URL, treat it as |
| // one. |
| // |
| // We don't do this for just _any_ location, since the X-Terraform-Get |
| // header is a go-getter location rather than a URL, and so not all |
| // possible values will parse reasonably as URLs.) |
| // |
| // When used in conjunction with go-getter we normally require this header |
| // to be an absolute URL, but we are more liberal here because third-party |
| // registry implementations may not "know" their own absolute URLs if |
| // e.g. they are running behind a reverse proxy frontend, or such. |
| if strings.HasPrefix(location, "/") || strings.HasPrefix(location, "./") || strings.HasPrefix(location, "../") { |
| locationURL, err := url.Parse(location) |
| if err != nil { |
| return "", fmt.Errorf("invalid relative URL for %q: %s", module, err) |
| } |
| locationURL = download.ResolveReference(locationURL) |
| location = locationURL.String() |
| } |
| |
| return location, nil |
| } |
| |
| // configureDiscoveryRetry configures the number of retries the registry client |
| // will attempt for requests with retryable errors, like 502 status codes |
| func configureDiscoveryRetry() { |
| discoveryRetry = defaultRetry |
| |
| if v := os.Getenv(registryDiscoveryRetryEnvName); v != "" { |
| retry, err := strconv.Atoi(v) |
| if err == nil && retry > 0 { |
| discoveryRetry = retry |
| } |
| } |
| } |
| |
| func requestLogHook(logger retryablehttp.Logger, req *http.Request, i int) { |
| if i > 0 { |
| logger.Printf("[INFO] Previous request to the remote registry failed, attempting retry.") |
| } |
| } |
| |
| func maxRetryErrorHandler(resp *http.Response, err error, numTries int) (*http.Response, error) { |
| // Close the body per library instructions |
| if resp != nil { |
| resp.Body.Close() |
| } |
| |
| // Additional error detail: if we have a response, use the status code; |
| // if we have an error, use that; otherwise nothing. We will never have |
| // both response and error. |
| var errMsg string |
| if resp != nil { |
| errMsg = fmt.Sprintf(": %s returned from %s", resp.Status, resp.Request.URL) |
| } else if err != nil { |
| errMsg = fmt.Sprintf(": %s", err) |
| } |
| |
| // This function is always called with numTries=RetryMax+1. If we made any |
| // retry attempts, include that in the error message. |
| if numTries > 1 { |
| return resp, fmt.Errorf("the request failed after %d attempts, please try again later%s", |
| numTries, errMsg) |
| } |
| return resp, fmt.Errorf("the request failed, please try again later%s", errMsg) |
| } |
| |
| // configureRequestTimeout configures the registry client request timeout from |
| // environment variables |
| func configureRequestTimeout() { |
| requestTimeout = defaultRequestTimeout |
| |
| if v := os.Getenv(registryClientTimeoutEnvName); v != "" { |
| timeout, err := strconv.Atoi(v) |
| if err == nil && timeout > 0 { |
| requestTimeout = time.Duration(timeout) * time.Second |
| } |
| } |
| } |