Add optional new symbol upload API to sym_upload.
Change-Id: I6a49e9f4a699fa6f5f8e9f0fc86afb4cb342a442
Reviewed-on: https://chromium-review.googlesource.com/c/breakpad/breakpad/+/1422400
Reviewed-by: Mark Mentovai <mark@chromium.org>
Reviewed-by: Ivan Penkov <ivanpe@chromium.org>
Reviewed-by: Mike Frysinger <vapier@chromium.org>
diff --git a/Makefile.am b/Makefile.am
index 3072a47..d5f60a3 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -615,6 +615,10 @@
src_tools_linux_symupload_sym_upload_SOURCES = \
src/common/linux/http_upload.cc \
src/common/linux/http_upload.h \
+ src/common/linux/libcurl_wrapper.cc \
+ src/common/linux/libcurl_wrapper.h \
+ src/common/linux/symbol_collector_client.cc \
+ src/common/linux/symbol_collector_client.h \
src/common/linux/symbol_upload.cc \
src/common/linux/symbol_upload.h \
src/tools/linux/symupload/sym_upload.cc
diff --git a/Makefile.in b/Makefile.in
index 4b23a87..96ad72e 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -1495,10 +1495,16 @@
src_tools_linux_symupload_minidump_upload_DEPENDENCIES =
am__src_tools_linux_symupload_sym_upload_SOURCES_DIST = \
src/common/linux/http_upload.cc src/common/linux/http_upload.h \
+ src/common/linux/libcurl_wrapper.cc \
+ src/common/linux/libcurl_wrapper.h \
+ src/common/linux/symbol_collector_client.cc \
+ src/common/linux/symbol_collector_client.h \
src/common/linux/symbol_upload.cc \
src/common/linux/symbol_upload.h \
src/tools/linux/symupload/sym_upload.cc
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@am_src_tools_linux_symupload_sym_upload_OBJECTS = src/common/linux/http_upload.$(OBJEXT) \
+@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/libcurl_wrapper.$(OBJEXT) \
+@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_collector_client.$(OBJEXT) \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.$(OBJEXT) \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/tools/linux/symupload/sym_upload.$(OBJEXT)
src_tools_linux_symupload_sym_upload_OBJECTS = \
@@ -2477,6 +2483,10 @@
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@src_tools_linux_symupload_sym_upload_SOURCES = \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/http_upload.cc \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/http_upload.h \
+@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/libcurl_wrapper.cc \
+@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/libcurl_wrapper.h \
+@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_collector_client.cc \
+@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_collector_client.h \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.cc \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/common/linux/symbol_upload.h \
@DISABLE_TOOLS_FALSE@@LINUX_HOST_TRUE@ src/tools/linux/symupload/sym_upload.cc
@@ -4726,6 +4736,9 @@
src/tools/linux/symupload/minidump_upload$(EXEEXT): $(src_tools_linux_symupload_minidump_upload_OBJECTS) $(src_tools_linux_symupload_minidump_upload_DEPENDENCIES) $(EXTRA_src_tools_linux_symupload_minidump_upload_DEPENDENCIES) src/tools/linux/symupload/$(am__dirstamp)
@rm -f src/tools/linux/symupload/minidump_upload$(EXEEXT)
$(AM_V_CXXLD)$(CXXLINK) $(src_tools_linux_symupload_minidump_upload_OBJECTS) $(src_tools_linux_symupload_minidump_upload_LDADD) $(LIBS)
+src/common/linux/symbol_collector_client.$(OBJEXT): \
+ src/common/linux/$(am__dirstamp) \
+ src/common/linux/$(DEPDIR)/$(am__dirstamp)
src/common/linux/symbol_upload.$(OBJEXT): \
src/common/linux/$(am__dirstamp) \
src/common/linux/$(DEPDIR)/$(am__dirstamp)
@@ -4996,6 +5009,7 @@
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-linux_libc_support.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-memory_mapped_file.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/src_tools_linux_dump_syms_dump_syms-safe_readlink.Po@am__quote@
+@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/symbol_collector_client.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/$(DEPDIR)/symbol_upload.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/tests/$(DEPDIR)/src_client_linux_linux_client_unittest_shlib-crash_generator.Po@am__quote@
@AMDEP_TRUE@@am__include@ @am__quote@src/common/linux/tests/$(DEPDIR)/src_common_dumper_unittest-crash_generator.Po@am__quote@
diff --git a/docs/sym_upload_v2_protocol.md b/docs/sym_upload_v2_protocol.md
new file mode 100644
index 0000000..d56a4c8
--- /dev/null
+++ b/docs/sym_upload_v2_protocol.md
@@ -0,0 +1,214 @@
+# Introduction
+
+The `sym_upload` tool is able to operate in `sym-upload-v2` protocol mode, in
+addition to the legacy protocol (which will be referred to as `sym-upload-v1`
+for the rest of this document). For now `sym-upload-v2` is HTTP/REST-based but
+it could be extended to operate over gRPC instead, in the future.
+
+# Table of Contents
+* [Why](#why)
+* [How](#how)
+ * [Uploading](#uploading)
+ * [Uploading with `sym_upload`](#uploading-with-sym_upload)
+ * [Uploading with curl](#uploading-with-curl)
+ * [Serving the `sym-upload-v2` protocol](#serving-the-sym-upload-v2-protocol)
+ * [Authenticate using `key`](#authenticate-using-key)
+ * [Symbol `checkStatus`](#symbol-checkstatus)
+ * [Upload `create`](#upload-create)
+ * [Uploading the symbol file](#uploading-the-symbol-file)
+ * [Upload complete](#upload-complete)
+
+
+# Why
+
+Using `sym_upload` in `sym-upload-v2` protocol mode has the following features
+beyond `sym-upload-v1`:
+ * Authentication via `key` (arbitrary secret).
+ * Symbol identifier (product of `debug_file` and `debug_id`, as recorded in
+output from `dump_syms`) can be checked against existing symbol information on
+server. If it's present, then the upload is skipped entirely.
+
+# How
+
+## Uploading
+
+### Uploading with `sym_upload`
+
+Uploading in `sym-upload-v2` protocol mode is easy. Invoke `sym_upload` like
+```
+$ ./sym_upload -p sym-upload-v2 [-k <API-key>] <symbol-file> <API-URL>
+```
+
+Where `symbol-file` is a symbol file created by `dump_syms`, `API-URL` is the
+URL of your `sym-upload-v2` API service (see next section for details), and
+`API-key` is a secret known to your uploader and server.
+
+For more options see `sym_upload --help`.
+
+### Uploading with curl
+
+As an example, if:
+ * Your API's URL was "https://sym-upload-api".
+ * Your service has assigned you `key` "myfancysecret123".
+ * You wanted to upload the symbol file at "path/to/file_name", with
+`debug_file` being "file_name" and `debug_id` being
+"123123123123123123123123123". Normally you would read these values from
+"path/to/file_name", which in turn was generated by `dump_syms`.
+
+Then you might run:
+```
+$ curl https://sym-upload-api/symbols/file_name/123123123123123123123123123:checkStatus?key=myfancysecret123
+```
+
+And, upon seeing that this `debug_file`/`debug_id` combo is missing from symbol
+storage then you could run:
+```
+$ curl --request POST https://sym-upload-api/uploads:create?key=myfancysecret123
+```
+
+Which returns `upload_url` "https://upload-server/42?creds=shhhhh" and
+`upload_key` "42". Next you upload the file directly like:
+```
+$ curl -T path/to/file_name "https://upload-server/42?creds=shhhhh"
+```
+
+Once the HTTP PUT is complete, run:
+```
+$ curl --header "Content-Type: application/json" \
+ --request POST \
+ --data '{symbol_id:{"debugFile":"file_name",'\
+ '"debugId":"123123123123123123123123123"}}' \
+ https://sym-upload-api/uploads/42:complete?key=myfancysecret123
+```
+
+### Serving the `sym-upload-v2` Protocol
+
+The protocol is currently defined only in HTTP/REST. There are three necessary
+REST operations to implement in your service:
+* `/symbols/<debug_file>/<debug_id>:checkStatus?key=<key>`
+* `/uploads:create?key=<key>`
+* `/uploads/<upload_key>:complete?key=<key>`
+
+#### Authenticate Using `key`
+
+The query string arg `key` contains some secret that both the uploader and
+server understand. It is up to the service implementer to decide on what
+constitutes a valid `key`, how the uploader acquires one, and how to handle
+requests made with invalid ones.
+
+#### Symbol `checkStatus`
+
+```
+/symbols/<debug_file>/<debug_id>:checkStatus?key=<key>
+```
+
+This operation expects an empty (or no) JSON payload in the request.
+
+This operation should return the status of the symbol file uniquely identified
+by the given `debug_file` and `debug_id`. JSON schema:
+```
+{
+ "type": object",
+ "properties": {
+ "status": {
+ "type": "string",
+ "enum": ["STATUS_UNSPECIFIED", "MISING", "FOUND"],
+ "required": true
+ }
+ }
+}
+```
+
+Where `MISSING` denotes that the symbol file does not exist on the server and
+`FOUND` denotes that the symbol file exists on the server.
+
+#### Upload `create`
+
+```
+/uploads:create?key=<key>
+```
+
+This operation expects an empty (or no) JSON payload in the request.
+
+This operation should return a URL that uploader can HTTP PUT their symbol file
+to, along with an "upload key" that can be used to notify the service once the
+file upload is completed. JSON schema:
+```
+{
+ "type": "object",
+ "properties": {
+ "upload_url": {
+ "type: "string",
+ "required": true
+ },
+ "upload_key": {
+ "type": "string",
+ "required": true
+ }
+ }
+}
+```
+
+Since this REST API operation can be authenticated via the `key` query string
+arg, the service can return a URL that encodes permission delegation to the
+upload endpoint resource and thereby constrain the ability to upload to those
+with valid `key`s.
+
+#### Uploading the Symbol File
+
+Note that the actual symbol upload step is _not_ part of the REST API. The
+upload URL obtained in the above operation is meant to be used as the endpoint
+for a normal HTTP PUT request for the contents of the symbol file. Once that
+HTTP PUT request is completed use the upload `complete` operation.
+
+#### Upload `complete`
+
+```
+/uploads/<upload_key>:complete?key=<key>
+```
+
+This operation expects a JSON payload in the HTTP request body with the
+following schema:
+```
+{
+ "type": "object",
+ "properties": {
+ "symbol_id": {
+ "type": "object",
+ "properties": {
+ "debug_file": {
+ "type": "string",
+ "required": true
+ },
+ "debug_id": {
+ "type": "string",
+ "required": true
+ }
+ }
+ }
+ }
+}
+```
+
+This operation should cause the symbol storage back-end (however implemented)
+to consume the symbol file identified by `upload_key`. It is up to the service
+implementation to decide how uploads are assigned `upload_key`s and how to
+retrieve a completed upload by its `upload_key`. If the symbol file cannot be
+found, is malformed, or the operation cannot be completed for any other reason
+then an HTTP error will be returned. JSON schema of non-error responses:
+```
+{
+ "type": "object",
+ "properties": {
+ "result": {
+ "type": string,
+ "enum": ["RESULT_UNSPECIFIED", "OK", "DUPLICATE_DATA"],
+ "required": true
+ }
+ }
+}
+```
+
+Where `OK` denotes that the symbol storage was updated with the new symbol file
+and `DUPLICATE_DATA` denotes that the symbol file data was identical to data
+already in symbol storage and therefore nothing changed.
diff --git a/src/common/common.gyp b/src/common/common.gyp
index 7d5e5c7..c0d71a3 100644
--- a/src/common/common.gyp
+++ b/src/common/common.gyp
@@ -74,8 +74,8 @@
'dwarf/dwarf2reader.cc',
'dwarf/dwarf2reader.h',
'dwarf/dwarf2reader_test_common.h',
- 'dwarf/elf_reader.cc',
- 'dwarf/elf_reader.h',
+ 'dwarf/elf_reader.cc',
+ 'dwarf/elf_reader.h',
'dwarf/functioninfo.cc',
'dwarf/functioninfo.h',
'dwarf/line_state_machine.h',
@@ -118,6 +118,8 @@
'linux/memory_mapped_file.h',
'linux/safe_readlink.cc',
'linux/safe_readlink.h',
+ 'linux/symbol_collector_client.cc',
+ 'linux/symbol_collector_client.h',
'linux/synth_elf.cc',
'linux/synth_elf.h',
'long_string_dictionary.cc',
diff --git a/src/common/linux/libcurl_wrapper.cc b/src/common/linux/libcurl_wrapper.cc
index fd4e34c..d935174 100644
--- a/src/common/linux/libcurl_wrapper.cc
+++ b/src/common/linux/libcurl_wrapper.cc
@@ -38,32 +38,24 @@
namespace google_breakpad {
LibcurlWrapper::LibcurlWrapper()
: init_ok_(false),
- formpost_(NULL),
- lastptr_(NULL),
- headerlist_(NULL) {
- curl_lib_ = dlopen("libcurl.so", RTLD_NOW);
- if (!curl_lib_) {
- curl_lib_ = dlopen("libcurl.so.4", RTLD_NOW);
- }
- if (!curl_lib_) {
- curl_lib_ = dlopen("libcurl.so.3", RTLD_NOW);
- }
- if (!curl_lib_) {
- std::cout << "Could not find libcurl via dlopen";
- return;
- }
- std::cout << "LibcurlWrapper init succeeded";
- init_ok_ = true;
- return;
-}
+ curl_lib_(nullptr),
+ last_curl_error_(""),
+ curl_(nullptr),
+ formpost_(nullptr),
+ lastptr_(nullptr),
+ headerlist_(nullptr) {}
-LibcurlWrapper::~LibcurlWrapper() {}
+LibcurlWrapper::~LibcurlWrapper() {
+ if (init_ok_) {
+ (*easy_cleanup_)(curl_);
+ dlclose(curl_lib_);
+ }
+}
bool LibcurlWrapper::SetProxy(const string& proxy_host,
const string& proxy_userpwd) {
- if (!init_ok_) {
- return false;
- }
+ if (!CheckInit()) return false;
+
// Set proxy information if necessary.
if (!proxy_host.empty()) {
(*easy_setopt_)(curl_, CURLOPT_PROXY, proxy_host.c_str());
@@ -83,9 +75,8 @@
bool LibcurlWrapper::AddFile(const string& upload_file_path,
const string& basename) {
- if (!init_ok_) {
- return false;
- }
+ if (!CheckInit()) return false;
+
std::cout << "Adding " << upload_file_path << " to form upload.";
// Add form file.
(*formadd_)(&formpost_, &lastptr_,
@@ -110,10 +101,11 @@
bool LibcurlWrapper::SendRequest(const string& url,
const std::map<string, string>& parameters,
- int* http_status_code,
+ long* http_status_code,
string* http_header_data,
string* http_response_data) {
- (*easy_setopt_)(curl_, CURLOPT_URL, url.c_str());
+ if (!CheckInit()) return false;
+
std::map<string, string>::const_iterator iter = parameters.begin();
for (; iter != parameters.end(); ++iter)
(*formadd_)(&formpost_, &lastptr_,
@@ -122,55 +114,79 @@
CURLFORM_END);
(*easy_setopt_)(curl_, CURLOPT_HTTPPOST, formpost_);
- if (http_response_data != NULL) {
- http_response_data->clear();
- (*easy_setopt_)(curl_, CURLOPT_WRITEFUNCTION, WriteCallback);
- (*easy_setopt_)(curl_, CURLOPT_WRITEDATA,
- reinterpret_cast<void *>(http_response_data));
- }
- if (http_header_data != NULL) {
- http_header_data->clear();
- (*easy_setopt_)(curl_, CURLOPT_HEADERFUNCTION, WriteCallback);
- (*easy_setopt_)(curl_, CURLOPT_HEADERDATA,
- reinterpret_cast<void *>(http_header_data));
+
+ return SendRequestInner(url, http_status_code, http_header_data,
+ http_response_data);
+}
+
+bool LibcurlWrapper::SendGetRequest(const string& url,
+ long* http_status_code,
+ string* http_header_data,
+ string* http_response_data) {
+ if (!CheckInit()) return false;
+
+ (*easy_setopt_)(curl_, CURLOPT_HTTPGET, 1L);
+
+ return SendRequestInner(url, http_status_code, http_header_data,
+ http_response_data);
+}
+
+bool LibcurlWrapper::SendPutRequest(const string& url,
+ const string& path,
+ long* http_status_code,
+ string* http_header_data,
+ string* http_response_data) {
+ if (!CheckInit()) return false;
+
+ FILE* file = fopen(path.c_str(), "rb");
+ (*easy_setopt_)(curl_, CURLOPT_UPLOAD, 1L);
+ (*easy_setopt_)(curl_, CURLOPT_PUT, 1L);
+ (*easy_setopt_)(curl_, CURLOPT_READDATA, file);
+
+ bool success = SendRequestInner(url, http_status_code, http_header_data,
+ http_response_data);
+
+ fclose(file);
+ return success;
+}
+
+bool LibcurlWrapper::SendSimplePostRequest(const string& url,
+ const string& body,
+ const string& content_type,
+ long* http_status_code,
+ string* http_header_data,
+ string* http_response_data) {
+ if (!CheckInit()) return false;
+
+ (*easy_setopt_)(curl_, CURLOPT_POSTFIELDSIZE, body.size());
+ (*easy_setopt_)(curl_, CURLOPT_COPYPOSTFIELDS, body.c_str());
+
+ if (!content_type.empty()) {
+ string content_type_header = "Content-Type: " + content_type;
+ headerlist_ = (*slist_append_)(
+ headerlist_,
+ content_type_header.c_str());
}
- CURLcode err_code = CURLE_OK;
- err_code = (*easy_perform_)(curl_);
- easy_strerror_ = reinterpret_cast<const char* (*)(CURLcode)>
- (dlsym(curl_lib_, "curl_easy_strerror"));
-
- if (http_status_code != NULL) {
- (*easy_getinfo_)(curl_, CURLINFO_RESPONSE_CODE, http_status_code);
- }
-
-#ifndef NDEBUG
- if (err_code != CURLE_OK)
- fprintf(stderr, "Failed to send http request to %s, error: %s\n",
- url.c_str(),
- (*easy_strerror_)(err_code));
-#endif
- if (headerlist_ != NULL) {
- (*slist_free_all_)(headerlist_);
- }
-
- (*easy_cleanup_)(curl_);
- if (formpost_ != NULL) {
- (*formfree_)(formpost_);
- }
-
- return err_code == CURLE_OK;
+ return SendRequestInner(url, http_status_code, http_header_data,
+ http_response_data);
}
bool LibcurlWrapper::Init() {
- if (!init_ok_) {
- std::cout << "Init_OK was not true in LibcurlWrapper::Init(), check earlier log messages";
+ curl_lib_ = dlopen("libcurl.so", RTLD_NOW);
+ if (!curl_lib_) {
+ curl_lib_ = dlopen("libcurl.so.4", RTLD_NOW);
+ }
+ if (!curl_lib_) {
+ curl_lib_ = dlopen("libcurl.so.3", RTLD_NOW);
+ }
+ if (!curl_lib_) {
+ std::cout << "Could not find libcurl via dlopen";
return false;
}
if (!SetFunctionPointers()) {
std::cout << "Could not find function pointers";
- init_ok_ = false;
return false;
}
@@ -184,11 +200,7 @@
return false;
}
- // Disable 100-continue header.
- char buf[] = "Expect:";
-
- headerlist_ = (*slist_append_)(headerlist_, buf);
- (*easy_setopt_)(curl_, CURLOPT_HTTPHEADER, headerlist_);
+ init_ok_ = true;
return true;
}
@@ -228,6 +240,10 @@
"curl_easy_getinfo",
CURLcode(*)(CURL *, CURLINFO info, ...));
+ SET_AND_CHECK_FUNCTION_POINTER(easy_reset_,
+ "curl_easy_reset",
+ void(*)(CURL*));
+
SET_AND_CHECK_FUNCTION_POINTER(slist_free_all_,
"curl_slist_free_all",
void(*)(curl_slist*));
@@ -238,4 +254,73 @@
return true;
}
+bool LibcurlWrapper::SendRequestInner(const string& url,
+ long* http_status_code,
+ string* http_header_data,
+ string* http_response_data) {
+ string url_copy(url);
+ (*easy_setopt_)(curl_, CURLOPT_URL, url_copy.c_str());
+
+ // Disable 100-continue header.
+ char buf[] = "Expect:";
+ headerlist_ = (*slist_append_)(headerlist_, buf);
+ (*easy_setopt_)(curl_, CURLOPT_HTTPHEADER, headerlist_);
+
+ if (http_response_data != nullptr) {
+ http_response_data->clear();
+ (*easy_setopt_)(curl_, CURLOPT_WRITEFUNCTION, WriteCallback);
+ (*easy_setopt_)(curl_, CURLOPT_WRITEDATA,
+ reinterpret_cast<void*>(http_response_data));
+ }
+ if (http_header_data != nullptr) {
+ http_header_data->clear();
+ (*easy_setopt_)(curl_, CURLOPT_HEADERFUNCTION, WriteCallback);
+ (*easy_setopt_)(curl_, CURLOPT_HEADERDATA,
+ reinterpret_cast<void*>(http_header_data));
+ }
+ CURLcode err_code = CURLE_OK;
+ err_code = (*easy_perform_)(curl_);
+ easy_strerror_ = reinterpret_cast<const char* (*)(CURLcode)>
+ (dlsym(curl_lib_, "curl_easy_strerror"));
+
+ if (http_status_code != nullptr) {
+ (*easy_getinfo_)(curl_, CURLINFO_RESPONSE_CODE, http_status_code);
+ }
+
+#ifndef NDEBUG
+ if (err_code != CURLE_OK)
+ fprintf(stderr, "Failed to send http request to %s, error: %s\n",
+ url.c_str(),
+ (*easy_strerror_)(err_code));
+#endif
+
+ Reset();
+
+ return err_code == CURLE_OK;
}
+
+void LibcurlWrapper::Reset() {
+ if (headerlist_ != nullptr) {
+ (*slist_free_all_)(headerlist_);
+ headerlist_ = nullptr;
+ }
+
+ if (formpost_ != nullptr) {
+ (*formfree_)(formpost_);
+ formpost_ = nullptr;
+ }
+
+ (*easy_reset_)(curl_);
+}
+
+bool LibcurlWrapper::CheckInit() {
+ if (!init_ok_) {
+ std::cout << "LibcurlWrapper: You must call Init(), and have it return "
+ "'true' before invoking any other methods.\n";
+ return false;
+ }
+
+ return true;
+}
+
+} // namespace google_breakpad
diff --git a/src/common/linux/libcurl_wrapper.h b/src/common/linux/libcurl_wrapper.h
index 25905ad..77aa6cb 100644
--- a/src/common/linux/libcurl_wrapper.h
+++ b/src/common/linux/libcurl_wrapper.h
@@ -51,14 +51,39 @@
const string& basename);
virtual bool SendRequest(const string& url,
const std::map<string, string>& parameters,
- int* http_status_code,
+ long* http_status_code,
string* http_header_data,
string* http_response_data);
+ bool SendGetRequest(const string& url,
+ long* http_status_code,
+ string* http_header_data,
+ string* http_response_data);
+ bool SendPutRequest(const string& url,
+ const string& path,
+ long* http_status_code,
+ string* http_header_data,
+ string* http_response_data);
+ bool SendSimplePostRequest(const string& url,
+ const string& body,
+ const string& content_type,
+ long* http_status_code,
+ string* http_header_data,
+ string* http_response_data);
+
private:
// This function initializes class state corresponding to function
// pointers into the CURL library.
bool SetFunctionPointers();
+ bool SendRequestInner(const string& url,
+ long* http_status_code,
+ string* http_header_data,
+ string* http_response_data);
+
+ void Reset();
+
+ bool CheckInit();
+
bool init_ok_; // Whether init succeeded
void* curl_lib_; // Pointer to result of dlopen() on
// curl library
@@ -85,6 +110,7 @@
const char* (*easy_strerror_)(CURLcode);
void (*easy_cleanup_)(CURL *);
CURLcode (*easy_getinfo_)(CURL *, CURLINFO info, ...);
+ void (*easy_reset_)(CURL*);
void (*formfree_)(struct curl_httppost *);
};
diff --git a/src/common/linux/symbol_collector_client.cc b/src/common/linux/symbol_collector_client.cc
new file mode 100644
index 0000000..ea995c4
--- /dev/null
+++ b/src/common/linux/symbol_collector_client.cc
@@ -0,0 +1,193 @@
+// Copyright (c) 2019 Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "common/linux/symbol_collector_client.h"
+
+#include <stdio.h>
+
+#include <iostream>
+#include <regex>
+
+#include "common/linux/libcurl_wrapper.h"
+
+namespace google_breakpad {
+namespace sym_upload {
+
+// static
+bool SymbolCollectorClient::CreateUploadUrl(
+ LibcurlWrapper* libcurl_wrapper,
+ const string& api_url,
+ const string& api_key,
+ UploadUrlResponse* uploadUrlResponse) {
+ string header, response;
+ long response_code;
+
+ string url = api_url + "/v1/uploads:create";
+ if (!api_key.empty()) {
+ url += "?key=" + api_key;
+ }
+
+ if (!libcurl_wrapper->SendSimplePostRequest(url,
+ /*body=*/"",
+ /*content_type=*/"",
+ &response_code,
+ &header,
+ &response)) {
+ printf("Failed to create upload url.\n");
+ printf("Response code: %ld\n", response_code);
+ printf("Response:\n");
+ printf("%s\n", response.c_str());
+ return false;
+ }
+
+ // Note camel-case rather than underscores.
+ std::regex upload_url_regex("\"uploadUrl\": \"([^\"]+)\"");
+ std::regex upload_key_regex("\"uploadKey\": \"([^\"]+)\"");
+
+ std::smatch upload_url_match;
+ if (!std::regex_search(response, upload_url_match, upload_url_regex) ||
+ upload_url_match.size() != 2) {
+ printf("Failed to parse create url response.");
+ printf("Response:\n");
+ printf("%s\n", response.c_str());
+ return false;
+ }
+ string upload_url = upload_url_match[1].str();
+
+ std::smatch upload_key_match;
+ if (!std::regex_search(response, upload_key_match, upload_key_regex) ||
+ upload_key_match.size() != 2) {
+ printf("Failed to parse create url response.");
+ printf("Response:\n");
+ printf("%s\n", response.c_str());
+ return false;
+ }
+ string upload_key = upload_key_match[1].str();
+
+ uploadUrlResponse->upload_url = upload_url;
+ uploadUrlResponse->upload_key = upload_key;
+ return true;
+}
+
+// static
+CompleteUploadResult SymbolCollectorClient::CompleteUpload(
+ LibcurlWrapper* libcurl_wrapper,
+ const string& api_url,
+ const string& api_key,
+ const string& upload_key,
+ const string& debug_file,
+ const string& debug_id) {
+ string header, response;
+ long response_code;
+
+ string url = api_url + "/v1/uploads/" + upload_key + ":complete";
+ if (!api_key.empty()) {
+ url += "?key=" + api_key;
+ }
+ string body =
+ "{ symbol_id: {"
+ "debug_file: \"" + debug_file + "\", "
+ "debug_id: \"" + debug_id + "\" } }";
+
+ if (!libcurl_wrapper->SendSimplePostRequest(url,
+ body,
+ "application/son",
+ &response_code,
+ &header,
+ &response)) {
+ printf("Failed to complete upload.\n");
+ printf("Response code: %ld\n", response_code);
+ printf("Response:\n");
+ printf("%s\n", response.c_str());
+ return CompleteUploadResult::Error;
+ }
+
+ std::regex result_regex("\"result\": \"([^\"]+)\"");
+ std::smatch result_match;
+ if (!std::regex_search(response, result_match, result_regex) ||
+ result_match.size() != 2) {
+ printf("Failed to parse complete upload response.");
+ printf("Response:\n");
+ printf("%s\n", response.c_str());
+ return CompleteUploadResult::Error;
+ }
+ string result = result_match[1].str();
+
+ if (result.compare("DUPLICATE_DATA") == 0) {
+ return CompleteUploadResult::DuplicateData;
+ }
+
+ return CompleteUploadResult::Ok;
+}
+
+// static
+SymbolStatus SymbolCollectorClient::CheckSymbolStatus(
+ LibcurlWrapper* libcurl_wrapper,
+ const string& api_url,
+ const string& api_key,
+ const string& debug_file,
+ const string& debug_id) {
+ string header, response;
+ long response_code;
+ string url = api_url +
+ "/v1/symbols/" + debug_file + "/" + debug_id + ":checkStatus";
+ if (!api_key.empty()) {
+ url += "?key=" + api_key;
+ }
+
+ if (!libcurl_wrapper->SendGetRequest(
+ url,
+ &response_code,
+ &header,
+ &response)) {
+ printf("Failed to check symbol status, error message.\n");
+ printf("Response code: %ld\n", response_code);
+ printf("Response:\n");
+ printf("%s\n", response.c_str());
+ return SymbolStatus::Unknown;
+ }
+
+ std::regex status_regex("\"status\": \"([^\"]+)\"");
+ std::smatch status_match;
+ if (!std::regex_search(response, status_match, status_regex) ||
+ status_match.size() != 2) {
+ printf("Failed to parse check symbol status response.");
+ printf("Response:\n");
+ printf("%s\n", response.c_str());
+ return SymbolStatus::Unknown;
+ }
+ string status = status_match[1].str();
+
+ return (status.compare("FOUND") == 0) ?
+ SymbolStatus::Found :
+ SymbolStatus::Missing;
+}
+
+} // namespace sym_upload
+} // namespace google_breakpad
diff --git a/src/common/linux/symbol_collector_client.h b/src/common/linux/symbol_collector_client.h
new file mode 100644
index 0000000..5f811de
--- /dev/null
+++ b/src/common/linux/symbol_collector_client.h
@@ -0,0 +1,87 @@
+// Copyright (c) 2019, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_
+#define COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_
+
+#include <string>
+
+#include "common/linux/libcurl_wrapper.h"
+#include "common/using_std_string.h"
+
+namespace google_breakpad {
+namespace sym_upload {
+
+struct UploadUrlResponse {
+ string upload_url;
+ string upload_key;
+};
+
+enum SymbolStatus {
+ Found,
+ Missing,
+ Unknown
+};
+
+enum CompleteUploadResult {
+ Ok,
+ DuplicateData,
+ Error
+};
+
+// Helper class to communicate with a sym-upload-v2 service over HTTP/REST,
+// via libcurl.
+class SymbolCollectorClient {
+ public:
+ static bool CreateUploadUrl(
+ LibcurlWrapper* libcurl_wrapper,
+ const string& api_url,
+ const string& api_key,
+ UploadUrlResponse* uploadUrlResponse);
+
+ static CompleteUploadResult CompleteUpload(
+ LibcurlWrapper* libcurl_wrapper,
+ const string& api_url,
+ const string& api_key,
+ const string& upload_key,
+ const string& debug_file,
+ const string& debug_id);
+
+ static SymbolStatus CheckSymbolStatus(
+ LibcurlWrapper* libcurl_wrapper,
+ const string& api_url,
+ const string& api_key,
+ const string& debug_file,
+ const string& debug_id);
+};
+
+} // namespace sym_upload
+} // namespace google_breakpad
+
+#endif // COMMON_LINUX_SYMBOL_COLLECTOR_CLIENT_H_
diff --git a/src/common/linux/symbol_upload.cc b/src/common/linux/symbol_upload.cc
index bbd3181..99750fd 100644
--- a/src/common/linux/symbol_upload.cc
+++ b/src/common/linux/symbol_upload.cc
@@ -30,15 +30,19 @@
// symbol_upload.cc: implemented google_breakpad::sym_upload::Start, a helper
// function for linux symbol upload tool.
-#include "common/linux/http_upload.h"
#include "common/linux/symbol_upload.h"
#include <assert.h>
#include <stdio.h>
#include <functional>
+#include <iostream>
#include <vector>
+#include "common/linux/http_upload.h"
+#include "common/linux/libcurl_wrapper.h"
+#include "common/linux/symbol_collector_client.h"
+
namespace google_breakpad {
namespace sym_upload {
@@ -95,21 +99,19 @@
return result;
}
-//=============================================================================
-void Start(Options *options) {
+// |options| describes the current sym_upload options.
+// |module_parts| contains the strings parsed from the MODULE entry of the
+// Breakpad symbol file being uploaded.
+// |compacted_id| is the debug_id from the MODULE entry of the Breakpad symbol
+// file being uploaded, with all hyphens removed.
+bool SymUploadV1Start(
+ const Options& options,
+ std::vector<string> module_parts,
+ const string& compacted_id) {
std::map<string, string> parameters;
- options->success = false;
- std::vector<string> module_parts;
- if (!ModuleDataForSymbolFile(options->symbolsPath, &module_parts)) {
- fprintf(stderr, "Failed to parse symbol file!\n");
- return;
- }
-
- string compacted_id = CompactIdentifier(module_parts[3]);
-
// Add parameters
- if (!options->version.empty())
- parameters["version"] = options->version;
+ if (!options.version.empty())
+ parameters["version"] = options.version;
// MODULE <os> <cpu> <uuid> <module-name>
// 0 1 2 3 4
@@ -120,16 +122,16 @@
parameters["debug_identifier"] = compacted_id;
std::map<string, string> files;
- files["symbol_file"] = options->symbolsPath;
+ files["symbol_file"] = options.symbolsPath;
string response, error;
long response_code;
- bool success = HTTPUpload::SendRequest(options->uploadURLStr,
+ bool success = HTTPUpload::SendRequest(options.uploadURLStr,
parameters,
files,
- options->proxy,
- options->proxy_user_pwd,
- "",
+ options.proxy,
+ options.proxy_user_pwd,
+ /*ca_certificate_file=*/"",
&response,
&response_code,
&error);
@@ -148,7 +150,116 @@
} else {
printf("Successfully sent the symbol file.\n");
}
- options->success = success;
+
+ return success;
+}
+
+// |options| describes the current sym_upload options.
+// |module_parts| contains the strings parsed from the MODULE entry of the
+// Breakpad symbol file being uploaded.
+// |compacted_id| is the debug_id from the MODULE entry of the Breakpad symbol
+// file being uploaded, with all hyphens removed.
+bool SymUploadV2Start(
+ const Options& options,
+ std::vector<string> module_parts,
+ const string& compacted_id) {
+ string debug_file = module_parts[4];
+ string debug_id = compacted_id;
+
+ google_breakpad::LibcurlWrapper libcurl_wrapper;
+ if (!libcurl_wrapper.Init()) {
+ printf("Failed to init google_breakpad::LibcurlWrapper.\n");
+ return false;
+ }
+
+ if (!options.force) {
+ SymbolStatus symbolStatus = SymbolCollectorClient::CheckSymbolStatus(
+ &libcurl_wrapper,
+ options.uploadURLStr,
+ options.api_key,
+ debug_file,
+ debug_id);
+ if (symbolStatus == SymbolStatus::Found) {
+ printf("Symbol file already exists, upload aborted."
+ " Use \"-f\" to overwrite.\n");
+ return true;
+ } else if (symbolStatus == SymbolStatus::Unknown) {
+ printf("Failed to check for existing symbol.\n");
+ return false;
+ }
+ }
+
+ UploadUrlResponse uploadUrlResponse;
+ if (!SymbolCollectorClient::CreateUploadUrl(
+ &libcurl_wrapper,
+ options.uploadURLStr,
+ options.api_key,
+ &uploadUrlResponse)) {
+ printf("Failed to create upload URL.\n");
+ return false;
+ }
+
+ string signed_url = uploadUrlResponse.upload_url;
+ string upload_key = uploadUrlResponse.upload_key;
+ string header;
+ string response;
+ long response_code;
+
+ if (!libcurl_wrapper.SendPutRequest(signed_url,
+ options.symbolsPath,
+ &response_code,
+ &header,
+ &response)) {
+ printf("Failed to send symbol file.\n");
+ printf("Response code: %ld\n", response_code);
+ printf("Response:\n");
+ printf("%s\n", response.c_str());
+ return false;
+ } else if (response_code == 0) {
+ printf("Failed to send symbol file: No response code\n");
+ return false;
+ } else if (response_code != 200) {
+ printf("Failed to send symbol file: Response code %ld\n", response_code);
+ printf("Response:\n");
+ printf("%s\n", response.c_str());
+ return false;
+ }
+
+ CompleteUploadResult completeUploadResult =
+ SymbolCollectorClient::CompleteUpload(&libcurl_wrapper,
+ options.uploadURLStr,
+ options.api_key,
+ upload_key,
+ debug_file,
+ debug_id);
+ if (completeUploadResult == CompleteUploadResult::Error) {
+ printf("Failed to complete upload.\n");
+ return false;
+ } else if (completeUploadResult == CompleteUploadResult::DuplicateData) {
+ printf("Uploaded file checksum matched existing file checksum,"
+ " no change necessary.\n");
+ } else {
+ printf("Successfully sent the symbol file.\n");
+ }
+
+ return true;
+}
+
+//=============================================================================
+void Start(Options* options) {
+ std::vector<string> module_parts;
+ if (!ModuleDataForSymbolFile(options->symbolsPath, &module_parts)) {
+ fprintf(stderr, "Failed to parse symbol file!\n");
+ return;
+ }
+
+ const string compacted_id = CompactIdentifier(module_parts[3]);
+
+ if (options->upload_protocol == UploadProtocol::SYM_UPLOAD_V2) {
+ options->success = SymUploadV2Start(*options, module_parts, compacted_id);
+ } else {
+ options->success = SymUploadV1Start(*options, module_parts, compacted_id);
+ }
}
} // namespace sym_upload
diff --git a/src/common/linux/symbol_upload.h b/src/common/linux/symbol_upload.h
index 0a46969..040e980 100644
--- a/src/common/linux/symbol_upload.h
+++ b/src/common/linux/symbol_upload.h
@@ -41,14 +41,24 @@
namespace google_breakpad {
namespace sym_upload {
-typedef struct {
+enum class UploadProtocol {
+ SYM_UPLOAD_V1,
+ SYM_UPLOAD_V2,
+};
+
+struct Options {
+ Options() : upload_protocol(UploadProtocol::SYM_UPLOAD_V1), force(false) {}
+
string symbolsPath;
string uploadURLStr;
string proxy;
string proxy_user_pwd;
string version;
bool success;
-} Options;
+ UploadProtocol upload_protocol;
+ bool force;
+ string api_key;
+};
// Starts upload to symbol server with options.
void Start(Options* options);
diff --git a/src/tools/linux/symupload/sym_upload.cc b/src/tools/linux/symupload/sym_upload.cc
index 9eeb2d4..cb5321a 100644
--- a/src/tools/linux/symupload/sym_upload.cc
+++ b/src/tools/linux/symupload/sym_upload.cc
@@ -41,25 +41,43 @@
#include <stdio.h>
#include <stdlib.h>
+#include <string.h>
#include <unistd.h>
#include "common/linux/symbol_upload.h"
+using google_breakpad::sym_upload::UploadProtocol;
using google_breakpad::sym_upload::Options;
//=============================================================================
static void
Usage(int argc, const char *argv[]) {
fprintf(stderr, "Submit symbol information.\n");
- fprintf(stderr, "Usage: %s [options...] <symbols> <upload-URL>\n", argv[0]);
+ fprintf(stderr, "Usage: %s [options...] <symbol-file> <upload-URL>\n",
+ argv[0]);
fprintf(stderr, "Options:\n");
- fprintf(stderr, "<symbols> should be created by using the dump_syms tool.\n");
+ fprintf(stderr, "<symbol-file> should be created by using the dump_syms"
+ "tool.\n");
fprintf(stderr, "<upload-URL> is the destination for the upload\n");
+ fprintf(stderr, "-p:\t <protocol> One of ['sym-upload-v1',"
+ " 'sym-upload-v2'], defaults to 'sym-upload-v1'.\n");
+ fprintf(stderr, "-k:\t <API-key> A secret used to authenticate with the"
+ " API [Only supported when using 'sym-upload-v2' protocol].\n");
+ fprintf(stderr, "-f:\t Force symbol upload if already exists [Only"
+ " supported when using 'sym-upload-v2' protocol].\n");
fprintf(stderr, "-v:\t Version information (e.g., 1.2.3.4)\n");
fprintf(stderr, "-x:\t <host[:port]> Use HTTP proxy on given port\n");
fprintf(stderr, "-u:\t <user[:password]> Set proxy user and password\n");
fprintf(stderr, "-h:\t Usage\n");
fprintf(stderr, "-?:\t Usage\n");
+ fprintf(stderr, "\n");
+ fprintf(stderr, "Examples:\n");
+ fprintf(stderr, " With 'sym-upload-v1':\n");
+ fprintf(stderr, " %s path/to/symbol_file http://myuploadserver\n",
+ argv[0]);
+ fprintf(stderr, " With 'sym-upload-v2':\n");
+ fprintf(stderr, " %s -p sym-upload-v2 -k mysecret123! "
+ "path/to/symbol_file http://myuploadserver\n", argv[0]);
}
//=============================================================================
@@ -68,7 +86,7 @@
extern int optind;
int ch;
- while ((ch = getopt(argc, (char * const *)argv, "u:v:x:h?")) != -1) {
+ while ((ch = getopt(argc, (char * const *)argv, "u:v:x:p:k:hf?")) != -1) {
switch (ch) {
case 'h':
case '?':
@@ -84,6 +102,23 @@
case 'x':
options->proxy = optarg;
break;
+ case 'p':
+ if (strcmp(optarg, "sym-upload-v2") == 0) {
+ options->upload_protocol = UploadProtocol::SYM_UPLOAD_V2;
+ } else if (strcmp(optarg, "sym-upload-v1") == 0) {
+ options->upload_protocol = UploadProtocol::SYM_UPLOAD_V1;
+ } else {
+ fprintf(stderr, "Invalid protocol '%c'\n", optarg);
+ Usage(argc, argv);
+ exit(1);
+ }
+ break;
+ case 'k':
+ options->api_key = optarg;
+ break;
+ case 'f':
+ options->force = true;
+ break;
default:
fprintf(stderr, "Invalid option '%c'\n", ch);
