Merge pull request #11768 from JakubAndrysek/http-client/collecting-headers

feat(HTTPClient): add support for collecting all HTTP headers

Author: me-no-dev

libraries/HTTPClient/examples/CustomHeaders/CustomHeaders.ino

@@ -0,0 +1,83 @@
+#include <Arduino.h>
+
+#include <WiFi.h>
+#include <HTTPClient.h>
+
+// Enable or disable collecting all headers
+#define COLLECT_ALL_HEADERS true
+
+void setup() {
+
+  Serial.begin(115200);
+
+  Serial.println();
+  Serial.println();
+  Serial.println();
+
+  for (uint8_t t = 4; t > 0; t--) {
+    Serial.printf("[SETUP] WAIT %d...\n", t);
+    Serial.flush();
+    delay(1000);
+  }
+
+  WiFi.begin("SSID", "PASSWORD");
+
+  while (WiFi.status() != WL_CONNECTED) {
+    delay(500);
+    Serial.print(".");
+  }
+  Serial.println();
+  Serial.println("Connected to WiFi: " + WiFi.SSID());
+}
+
+void loop() {
+
+  HTTPClient http;
+
+  Serial.print("[HTTP] Preparing HTTP request...\n");
+  // This page will return the headers we want to test + some others
+  http.begin("https://httpbingo.org/response-headers?x-custom-header=value:42");
+
+#if COLLECT_ALL_HEADERS
+  // Collect all headers
+  http.collectAllHeaders();
+#else
+  // Collect specific headers, only that one will be stored
+  const char *headerKeys[] = {"x-custom-header"};
+  const size_t headerKeysCount = sizeof(headerKeys) / sizeof(headerKeys[0]);
+  http.collectHeaders(headerKeys, headerKeysCount);
+#endif
+
+  Serial.print("[HTTP] Sending HTTP GET request...\n");
+  // start connection and send HTTP header
+  int httpCode = http.GET();
+
+  // httpCode will be negative on error
+  if (httpCode > 0) {
+    // HTTP header has been send and Server response header has been handled
+    Serial.printf("[HTTP] GET response code: %d\n", httpCode);
+
+    Serial.println("[HTTP] Headers collected:");
+    for (size_t i = 0; i < http.headers(); i++) {
+      Serial.printf("[HTTP] - '%s': '%s'\n", http.headerName(i).c_str(), http.header(i).c_str());
+    }
+
+    Serial.println("[HTTP] Has header 'x-custom-header'? " + String(http.hasHeader("x-custom-header")) + " (expected true)");
+    Serial.printf("[HTTP] x-custom-header: '%s' (expected 'value:42')\n", http.header("x-custom-header").c_str());
+    Serial.printf("[HTTP] non-existing-header: '%s' (expected empty string)\n", http.header("non-existing-header").c_str());
+
+#if COLLECT_ALL_HEADERS
+    // Server response with multiple headers, one of them is 'server'
+    Serial.println("[HTTP] Has header 'server'? " + String(http.hasHeader("server")) + " (expected true)");
+    Serial.printf("[HTTP] server: '%s'\n", http.header("server").c_str());
+#endif
+
+  } else {
+    Serial.printf("[HTTP] GET... failed, error: %s\n", http.errorToString(httpCode).c_str());
+  }
+
+  http.end();
+
+  Serial.println("[HTTP] end connection\n\n");
+  delay(5000);
+}

libraries/HTTPClient/examples/CustomHeaders/ci.json

@@ -0,0 +1,6 @@
+{
+  "requires_any": [
+    "CONFIG_SOC_WIFI_SUPPORTED=y",
+    "CONFIG_ESP_WIFI_REMOTE_ENABLED=y"
+  ]
+}

libraries/HTTPClient/src/HTTPClient.cpp

@@ -90,9 +90,6 @@ HTTPClient::~HTTPClient() {
   if (_client) {
     _client->stop();
   }
-  if (_currentHeaders) {
-    delete[] _currentHeaders;
-  }
   if (_tcpDeprecated) {
     _tcpDeprecated.reset(nullptr);
   }
@@ -564,9 +561,12 @@ int HTTPClient::sendRequest(const char *type, uint8_t *payload, size_t size) {
   bool redirect = false;
   uint16_t redirectCount = 0;
   do {
-    // wipe out any existing headers from previous request
-    for (size_t i = 0; i < _headerKeysCount; i++) {
-      if (_currentHeaders[i].value.length() > 0) {
+    // wipe out any existing headers from previous request, but preserve the keys if collecting specific headers
+    if (_collectAllHeaders) {
+      _currentHeaders.clear();
+    } else {
+      // Only clear values, keep the keys for specific header collection
+      for (size_t i = 0; i < _currentHeaders.size(); ++i) {
         _currentHeaders[i].value.clear();
       }
     }
@@ -1015,19 +1015,24 @@ void HTTPClient::addHeader(const String &name, const String &value, bool first,
   }
 }
 
+void HTTPClient::collectAllHeaders(bool collectAll) {
+  _collectAllHeaders = collectAll;
+}
+
 void HTTPClient::collectHeaders(const char *headerKeys[], const size_t headerKeysCount) {
-  _headerKeysCount = headerKeysCount;
-  if (_currentHeaders) {
-    delete[] _currentHeaders;
+  if (_collectAllHeaders) {
+    log_w("collectHeaders is ignored when collectAllHeaders is set");
+    return;
   }
-  _currentHeaders = new RequestArgument[_headerKeysCount];
-  for (size_t i = 0; i < _headerKeysCount; i++) {
+  _currentHeaders.clear();
+  _currentHeaders.resize(headerKeysCount);
+  for (size_t i = 0; i < headerKeysCount; i++) {
     _currentHeaders[i].key = headerKeys[i];
   }
 }
 
 String HTTPClient::header(const char *name) {
-  for (size_t i = 0; i < _headerKeysCount; ++i) {
+  for (size_t i = 0; i < _currentHeaders.size(); ++i) {
     if (_currentHeaders[i].key.equalsIgnoreCase(name)) {
       return _currentHeaders[i].value;
     }
@@ -1036,25 +1041,25 @@ String HTTPClient::header(const char *name) {
 }
 
 String HTTPClient::header(size_t i) {
-  if (i < _headerKeysCount) {
+  if (i < _currentHeaders.size()) {
     return _currentHeaders[i].value;
   }
   return String();
 }
 
 String HTTPClient::headerName(size_t i) {
-  if (i < _headerKeysCount) {
+  if (i < _currentHeaders.size()) {
     return _currentHeaders[i].key;
   }
   return String();
 }
 
 int HTTPClient::headers() {
-  return _headerKeysCount;
+  return _currentHeaders.size();
 }
 
 bool HTTPClient::hasHeader(const char *name) {
-  for (size_t i = 0; i < _headerKeysCount; ++i) {
+  for (size_t i = 0; i < _currentHeaders.size(); ++i) {
     if ((_currentHeaders[i].key.equalsIgnoreCase(name)) && (_currentHeaders[i].value.length() > 0)) {
       return true;
     }
@@ -1238,17 +1243,14 @@ int HTTPClient::handleHeaderResponse() {
           setCookie(date, headerValue);
         }
 
-        for (size_t i = 0; i < _headerKeysCount; i++) {
-          if (_currentHeaders[i].key.equalsIgnoreCase(headerName)) {
-            // Uncomment the following lines if you need to add support for multiple headers with the same key:
-            // if (!_currentHeaders[i].value.isEmpty()) {
-            //     // Existing value, append this one with a comma
-            //     _currentHeaders[i].value += ',';
-            //     _currentHeaders[i].value += headerValue;
-            // } else {
-            _currentHeaders[i].value = headerValue;
-            // }
-            break;  // We found a match, stop looking
+        if (_collectAllHeaders && headerName.length() > 0) {
+          _currentHeaders.emplace_back(headerName, headerValue);
+        } else {
+          for (size_t i = 0; i < _currentHeaders.size(); ++i) {
+            if (_currentHeaders[i].key.equalsIgnoreCase(headerName)) {
+              _currentHeaders[i].value = headerValue;
+              break;  // We found a match, stop looking
+            }
           }
         }
       }

libraries/HTTPClient/src/HTTPClient.h

@@ -38,7 +38,7 @@
 #include <NetworkClientSecure.h>
 #endif  // HTTPCLIENT_NOSECURE
 
-/// Cookie jar support
+/// Cookie jar and header support
 #include <vector>
 
 #define HTTPCLIENT_DEFAULT_TCP_TIMEOUT (5000)
@@ -238,6 +238,7 @@ class HTTPClient {
   void addHeader(const String &name, const String &value, bool first = false, bool replace = true);
 
   /// Response handling
+  void collectAllHeaders(bool collectAll = true);
   void collectHeaders(const char *headerKeys[], const size_t headerKeysCount);
   String header(const char *name);   // get request header value by name
   String header(size_t i);           // get request header value by number
@@ -294,6 +295,7 @@ class HTTPClient {
   uint16_t _tcpTimeout = HTTPCLIENT_DEFAULT_TCP_TIMEOUT;
   bool _useHTTP10 = false;
   bool _secure = false;
+  bool _collectAllHeaders = false;
 
   String _uri;
   String _protocol;
@@ -304,8 +306,7 @@ class HTTPClient {
   String _acceptEncoding = "identity;q=1,chunked;q=0.1,*;q=0";
 
   /// Response handling
-  RequestArgument *_currentHeaders = nullptr;
-  size_t _headerKeysCount = 0;
+  std::vector<RequestArgument> _currentHeaders;
 
   int _returnCode = 0;
   int _size = -1;