Revision 23:15fa2726f793, committed 2018-01-11

Comitter:

Jan Jongboom

Date:

Thu Jan 11 13:49:06 2018 +0100

Parent:

22:71fc1b1894f8

Child:

24:6c1651fd26b9

Commit message:

Implement Transfer-Encoding: chunked on HTTP and HTTPS requests, allows streaming large blocks of data over HTTP

Changed in this revision

--- a/source/http_request.h	Wed Jan 03 11:23:22 2018 +0000
+++ b/source/http_request.h	Thu Jan 11 13:49:06 2018 +0100
@@ -112,36 +112,160 @@
 
     /**
      * Execute the request and receive the response.
+     * This adds a Content-Length header to the request (when body_size is set), and sends the data to the server.
+     * @param body Pointer to the body to be sent
+     * @param body_size Size of the body to be sent
+     * @return An HttpResponse pointer on success, or NULL on failure.
+     *         See get_error() for the error code.
      */
     HttpResponse* send(const void* body = NULL, nsapi_size_t body_size = 0) {
-        if (response != NULL) {
-            // already executed this response
-            error = -2100; // @todo, make a lookup table with errors
-            return NULL;
-        }
-
-        error = 0;
+        nsapi_size_or_error_t ret = open_socket();
 
-        if (we_created_socket) {
-            nsapi_error_t open_result = socket->open(network);
-            if (open_result != 0) {
-                error = open_result;
-                return NULL;
-            }
-
-            nsapi_error_t connection_result = socket->connect(parsed_url->host(), parsed_url->port());
-            if (connection_result != 0) {
-                error = connection_result;
-                return NULL;
-            }
+        if (ret != NSAPI_ERROR_OK) {
+            error = ret;
+            return NULL;
         }
 
         size_t request_size = 0;
         char* request = request_builder->build(body, body_size, request_size);
 
+        ret = send_buffer(request, request_size);
+
+        free(request);
+
+        if (ret < 0) {
+            error = ret;
+            return NULL;
+        }
+
+        return create_http_response();
+    }
+
+    /**
+     * Execute the request and receive the response.
+     * This sends the request through chunked-encoding.
+     * @param body_cb Callback which generates the next chunk of the request
+     * @return An HttpResponse pointer on success, or NULL on failure.
+     *         See get_error() for the error code.
+     */
+    HttpResponse* send(Callback<const void*(size_t*)> body_cb) {
+
+        nsapi_error_t ret;
+
+        if ((ret = open_socket()) != NSAPI_ERROR_OK) {
+            error = ret;
+            return NULL;
+        }
+
+        set_header("Transfer-Encoding", "chunked");
+
+        size_t request_size = 0;
+        char* request = request_builder->build(NULL, 0, request_size);
+
+        // first... send this request headers without the body
+        nsapi_size_or_error_t total_send_count = send_buffer(request, request_size);
+
+        if (total_send_count < 0) {
+            free(request);
+            error = total_send_count;
+            return NULL;
+        }
+
+        // ok... now it's time to start sending chunks...
+        while (1) {
+            size_t size;
+            const void *buffer = body_cb(&size);
+
+            if (size == 0) break;
+
+            // so... size in HEX, \r\n, data, \r\n again
+            char size_buff[10]; // if sending length of more than 8 digits, you have another problem on a microcontroller...
+            size_t size_buff_size = sprintf(size_buff, "%X\r\n", size);
+            if ((total_send_count = send_buffer(size_buff, size_buff_size)) < 0) {
+                free(request);
+                error = total_send_count;
+                return NULL;
+            }
+
+            // now send the normal buffer... and then \r\n at the end
+            total_send_count = send_buffer((char*)buffer, size);
+            if (total_send_count < 0) {
+                free(request);
+                error = total_send_count;
+                return NULL;
+            }
+
+            // and... \r\n
+            const char* rn = "\r\n";
+            if ((total_send_count = send_buffer((char*)rn, 2)) < 0) {
+                free(request);
+                error = total_send_count;
+                return NULL;
+            }
+        }
+
+        // finalize...?
+        const char* fin = "0\r\n\r\n";
+        if ((total_send_count = send_buffer((char*)fin, strlen(fin))) < 0) {
+            free(request);
+            error = total_send_count;
+            return NULL;
+        }
+
+        free(request);
+
+        return create_http_response();
+    }
+
+    /**
+     * Set a header for the request.
+     *
+     * The 'Host' and 'Content-Length' headers are set automatically.
+     * Setting the same header twice will overwrite the previous entry.
+     *
+     * @param[in] key Header key
+     * @param[in] value Header value
+     */
+    void set_header(string key, string value) {
+        request_builder->set_header(key, value);
+    }
+
+    /**
+     * Get the error code.
+     *
+     * When send() fails, this error is set.
+     */
+    nsapi_error_t get_error() {
+        return error;
+    }
+
+private:
+    nsapi_error_t open_socket() {
+        if (response != NULL) {
+            // already executed this response
+            return -2100; // @todo, make a lookup table with errors
+        }
+
+
+        if (we_created_socket) {
+            nsapi_error_t open_result = socket->open(network);
+            if (open_result != NSAPI_ERROR_OK) {
+                return open_result;
+            }
+
+            nsapi_error_t connection_result = socket->connect(parsed_url->host(), parsed_url->port());
+            if (connection_result != NSAPI_ERROR_OK) {
+                return connection_result;
+            }
+        }
+
+        return NSAPI_ERROR_OK;
+    }
+
+    nsapi_size_or_error_t send_buffer(char* buffer, size_t buffer_size) {
         nsapi_size_or_error_t total_send_count = 0;
-        while (total_send_count < request_size) {
-            nsapi_size_or_error_t send_result = socket->send(request + total_send_count, request_size - total_send_count);
+        while (total_send_count < buffer_size) {
+            nsapi_size_or_error_t send_result = socket->send(buffer + total_send_count, buffer_size - total_send_count);
 
             if (send_result < 0) {
                 total_send_count = send_result;
@@ -155,14 +279,10 @@
             total_send_count += send_result;
         }
 
-
-        free(request);
+        return total_send_count;
+    }
 
-        if (total_send_count < 0) {
-            error = total_send_count;
-            return NULL;
-        }
-
+    HttpResponse* create_http_response() {
         // Create a response object
         response = new HttpResponse();
         // And a response parser
@@ -209,29 +329,7 @@
         return response;
     }
 
-    /**
-     * Set a header for the request.
-     *
-     * The 'Host' and 'Content-Length' headers are set automatically.
-     * Setting the same header twice will overwrite the previous entry.
-     *
-     * @param[in] key Header key
-     * @param[in] value Header value
-     */
-    void set_header(string key, string value) {
-        request_builder->set_header(key, value);
-    }
 
-    /**
-     * Get the error code.
-     *
-     * When send() fails, this error is set.
-     */
-    nsapi_error_t get_error() {
-        return error;
-    }
-
-private:
     NetworkInterface* network;
     TCPSocket* socket;
     http_method method;

--- a/source/http_request_builder.h	Wed Jan 03 11:23:22 2018 +0000
+++ b/source/http_request_builder.h	Thu Jan 11 13:49:06 2018 +0100
@@ -58,10 +58,12 @@
         }
     }
 
-    char* build(const void* body, size_t body_size, size_t &size) {
+    char* build(const void* body, size_t body_size, size_t &size, bool skip_content_length = false) {
         const char* method_str = http_method_str(method);
 
-        if (method == HTTP_POST || method == HTTP_PUT || method == HTTP_DELETE || body_size > 0) {
+        bool is_chunked = has_header("Transfer-Encoding", "chunked");
+
+        if (!is_chunked && (method == HTTP_POST || method == HTTP_PUT || method == HTTP_DELETE || body_size > 0)) {
             char buffer[10];
             snprintf(buffer, 10, "%d", body_size);
             set_header("Content-Length", string(buffer));
@@ -82,11 +84,13 @@
         // then the body, first an extra newline
         size += 2;
 
-        // body
-        size += body_size;
+        if (!is_chunked) {
+            // body
+            size += body_size;
 
-        // extra newline
-        size += 2;
+            // extra newline
+            size += 2;
+        }
 
         // Now let's print it
         char* req = (char*)calloc(size + 1, 1);
@@ -114,8 +118,10 @@
         }
         req += body_size;
 
-        sprintf(req, "\r\n");
-        req += 2;
+        if (!is_chunked) {
+            sprintf(req, "\r\n");
+            req += 2;
+        }
 
         // Uncomment to debug...
         // printf("----- BEGIN REQUEST -----\n");
@@ -126,6 +132,19 @@
     }
 
 private:
+    bool has_header(const char* key, const char* value = NULL) {
+        typedef map<string, string>::iterator it_type;
+        for(it_type it = headers.begin(); it != headers.end(); it++) {
+            if (strcmp(it->first.c_str(), key) == 0) { // key matches
+                if (value == NULL || (strcmp(it->second.c_str(), value) == 0)) { // value is NULL or matches
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
     http_method method;
     ParsedUrl* parsed_url;
     map<string, string> headers;

--- a/source/https_request.h	Wed Jan 03 11:23:22 2018 +0000
+++ b/source/https_request.h	Thu Jan 11 13:49:06 2018 +0100
@@ -121,91 +121,102 @@
      *         See get_error() for the error code.
      */
     HttpResponse* send(const void* body = NULL, nsapi_size_t body_size = 0) {
-        // not tried to connect before?
-        if (_tlssocket->error() != 0) {
-            _error = _tlssocket->error();
+        nsapi_size_or_error_t ret = open_socket();
+
+        if (ret != NSAPI_ERROR_OK) {
+            _error = ret;
             return NULL;
         }
 
-        bool socket_was_open = _tlssocket->connected();
-
-        if (!socket_was_open) {
-            nsapi_error_t r = _tlssocket->connect();
-            if (r != 0) {
-                _error = r;
-                return NULL;
-            }
-        }
-
-        int ret;
-
         size_t request_size = 0;
         char* request = _request_builder->build(body, body_size, request_size);
 
-        ret = mbedtls_ssl_write(_tlssocket->get_ssl_context(), (const unsigned char *) request, request_size);
+        ret = send_buffer((const unsigned char*)request, request_size);
 
         free(request);
 
         if (ret < 0) {
-            if (ret != MBEDTLS_ERR_SSL_WANT_READ &&
-                ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
-                print_mbedtls_error("mbedtls_ssl_write", ret);
-                onError(_tlssocket->get_tcp_socket(), -1 );
-            }
-            else {
-                _error = ret;
-            }
+            _error = ret;
+            return NULL;
+        }
+
+        return create_http_response();
+    }
+
+    /**
+     * Execute the HTTPS request.
+     * This sends the request through chunked-encoding.
+     * @param body_cb Callback which generates the next chunk of the request
+     * @return An HttpResponse pointer on success, or NULL on failure.
+     *         See get_error() for the error code.
+     */
+    HttpResponse* send(Callback<const void*(size_t*)> body_cb) {
+
+        nsapi_error_t ret;
+
+        if ((ret = open_socket()) != NSAPI_ERROR_OK) {
+            _error = ret;
+            return NULL;
+        }
+
+        set_header("Transfer-Encoding", "chunked");
+
+        size_t request_size = 0;
+        char* request = _request_builder->build(NULL, 0, request_size);
+
+        // first... send this request headers without the body
+        nsapi_size_or_error_t total_send_count = send_buffer((unsigned char*)request, request_size);
+
+        if (total_send_count < 0) {
+            free(request);
+            _error = total_send_count;
             return NULL;
         }
 
-        // Create a response object
-        _response = new HttpResponse();
-        // And a response parser
-        HttpParser parser(_response, HTTP_RESPONSE, _body_callback);
+        // ok... now it's time to start sending chunks...
+        while (1) {
+            size_t size;
+            const void *buffer = body_cb(&size);
 
-        // Set up a receive buffer (on the heap)
-        uint8_t* recv_buffer = (uint8_t*)malloc(HTTP_RECEIVE_BUFFER_SIZE);
+            if (size == 0) break;
 
-        /* Read data out of the socket */
-        while ((ret = mbedtls_ssl_read(_tlssocket->get_ssl_context(), (unsigned char *) recv_buffer, HTTP_RECEIVE_BUFFER_SIZE)) > 0) {
-            // Don't know if this is actually needed, but OK
-            size_t _bpos = static_cast<size_t>(ret);
-            recv_buffer[_bpos] = 0;
-
-            size_t nparsed = parser.execute((const char*)recv_buffer, _bpos);
-            if (nparsed != _bpos) {
-                print_mbedtls_error("parser_error", nparsed);
-                // parser error...
-                _error = -2101;
-                free(recv_buffer);
+            // so... size in HEX, \r\n, data, \r\n again
+            char size_buff[10]; // if sending length of more than 8 digits, you have another problem on a microcontroller...
+            size_t size_buff_size = sprintf(size_buff, "%X\r\n", size);
+            if ((total_send_count = send_buffer((const unsigned char*)size_buff, size_buff_size)) < 0) {
+                free(request);
+                _error = total_send_count;
                 return NULL;
             }
 
-            if (_response->is_message_complete()) {
-                break;
+            // now send the normal buffer... and then \r\n at the end
+            total_send_count = send_buffer((const unsigned char*)buffer, size);
+            if (total_send_count < 0) {
+                free(request);
+                _error = total_send_count;
+                return NULL;
+            }
+
+            // and... \r\n
+            const char* rn = "\r\n";
+            if ((total_send_count = send_buffer((const unsigned char*)rn, 2)) < 0) {
+                free(request);
+                _error = total_send_count;
+                return NULL;
             }
         }
-        if (ret < 0) {
-            if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
-                print_mbedtls_error("mbedtls_ssl_read", ret);
-                onError(_tlssocket->get_tcp_socket(), -1 );
-            }
-            else {
-                _error = ret;
-            }
-            free(recv_buffer);
+
+        // finalize...?
+        const char* fin = "0\r\n\r\n";
+        if ((total_send_count = send_buffer((const unsigned char*)fin, strlen(fin))) < 0) {
+            free(request);
+            _error = total_send_count;
             return NULL;
         }
 
-        parser.finish();
+        free(request);
 
-        if (!socket_was_open) {
-            _tlssocket->get_tcp_socket()->close();
-        }
-
-        free(recv_buffer);
-
-        return _response;
+        return create_http_response();
     }
 
     /**
@@ -264,6 +275,100 @@
         _error = error;
     }
 
+    nsapi_error_t onErrorAndReturn(TCPSocket *s, int error) {
+        s->close();
+        return error;
+    }
+
+private:
+    nsapi_error_t open_socket() {
+        // not tried to connect before?
+        if (_tlssocket->error() != 0) {
+            return _tlssocket->error();
+        }
+
+        _socket_was_open = _tlssocket->connected();
+
+        if (!_socket_was_open) {
+            nsapi_error_t r = _tlssocket->connect();
+            if (r != NSAPI_ERROR_OK) {
+                return r;
+            }
+        }
+
+        return NSAPI_ERROR_OK;
+    }
+
+    nsapi_size_or_error_t send_buffer(const unsigned char *buffer, size_t buffer_size) {
+        nsapi_size_or_error_t ret = mbedtls_ssl_write(_tlssocket->get_ssl_context(), (const unsigned char *) buffer, buffer_size);
+
+        if (ret < 0) {
+            if (ret != MBEDTLS_ERR_SSL_WANT_READ &&
+                ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
+                print_mbedtls_error("mbedtls_ssl_write", ret);
+                return onErrorAndReturn(_tlssocket->get_tcp_socket(), -1 );
+            }
+            else {
+                return ret;
+            }
+        }
+
+        return NSAPI_ERROR_OK;
+    }
+
+    HttpResponse* create_http_response() {
+        nsapi_size_or_error_t ret;
+
+        // Create a response object
+        _response = new HttpResponse();
+        // And a response parser
+        HttpParser parser(_response, HTTP_RESPONSE, _body_callback);
+
+        // Set up a receive buffer (on the heap)
+        uint8_t* recv_buffer = (uint8_t*)malloc(HTTP_RECEIVE_BUFFER_SIZE);
+
+        /* Read data out of the socket */
+        while ((ret = mbedtls_ssl_read(_tlssocket->get_ssl_context(), (unsigned char *) recv_buffer, HTTP_RECEIVE_BUFFER_SIZE)) > 0) {
+            // Don't know if this is actually needed, but OK
+            size_t _bpos = static_cast<size_t>(ret);
+            recv_buffer[_bpos] = 0;
+
+            size_t nparsed = parser.execute((const char*)recv_buffer, _bpos);
+            if (nparsed != _bpos) {
+                print_mbedtls_error("parser_error", nparsed);
+                // parser error...
+                _error = -2101;
+                free(recv_buffer);
+                return NULL;
+            }
+
+            if (_response->is_message_complete()) {
+                break;
+            }
+        }
+        if (ret < 0) {
+            if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE) {
+                print_mbedtls_error("mbedtls_ssl_read", ret);
+                onError(_tlssocket->get_tcp_socket(), -1 );
+            }
+            else {
+                _error = ret;
+            }
+            free(recv_buffer);
+            return NULL;
+        }
+
+        parser.finish();
+
+        if (!_socket_was_open) {
+            _tlssocket->get_tcp_socket()->close();
+        }
+
+        free(recv_buffer);
+
+        return _response;
+    }
+
 protected:
     TLSSocket* _tlssocket;
     bool _we_created_the_socket;
@@ -273,6 +378,8 @@
     HttpRequestBuilder* _request_builder;
     HttpResponse* _response;
 
+    bool _socket_was_open;
+
     nsapi_error_t _error;
     bool _debug;