Implemented CURL-based HTTP client.

Only supports GET requests, but that's more than enough for the time being.

Other changes include:

* OptionsTab: added an option to update the NSWDB XML (not implemented yet).
* config: add missing flag check in getters and setters.
* Move default socket initialization from utils.c to services.c.

http.c

@@ -1,23 +0,0 @@
-/*
- * http.c
- *
- * Copyright (c) 2020-2021, DarkMatterCore <pabloacurielz@gmail.com>.
- *
- * This file is part of nxdumptool (https://github.com/DarkMatterCore/nxdumptool).
- *
- * nxdumptool is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * nxdumptool is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-#include "nxdt_utils.h"
-#include "http.h"

include/core/http.h

@@ -25,15 +25,53 @@
 #define __HTTP_H__
 
 #include <curl/curl.h>
-/*
+
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+/// Callback definition to write downloaded data.
+typedef curl_write_callback HttpWriteCallback;
 
+/// Callback definition to handle progress updates.
+typedef curl_xferinfo_callback HttpProgressCallback;
+
+/// Used by httpWriteBufferCallback().
+typedef struct {
+    char *data;     ///< Dynamically allocated buffer.
+    size_t size;    ///< Buffer size.
+} HttpBuffer;
+
+/// Initializes the HTTP client interface.
+bool httpInitialize(void);
+
+/// Closes the HTTP client interface.
+void httpExit(void);
+
+/// Writes downloaded data to an output file. May be used as the write callback for httpPerformGetRequest().
+/// Expects 'outstream' / 'write_ptr' to be a FILE pointer.
+size_t httpWriteFileCallback(char *buffer, size_t size, size_t nitems, void *outstream);
+
+/// Writes downloaded data to an output buffer. May be used as the write callback for httpPerformGetRequest().
+/// Expects 'outstream' / 'write_ptr' to be a pointer to a HttpBuffer element. Its 'data' member is reallocated throughout the download process.
+size_t httpWriteBufferCallback(char *buffer, size_t size, size_t nitems, void *outstream);
+
+/// Performs a HTTP GET request. Blocks the calling thread until the whole transfer is complete.
+/// Callbacks are optional, but they should be provided to save downloaded data and/or handle progress updates.
+/// If 'outsize' is provided, the download size will be stored in it if the request succeeds.
+bool httpPerformGetRequest(const char *url, bool force_https, size_t *outsize, HttpWriteCallback write_cb, void *write_ptr, HttpProgressCallback progress_cb, void *progress_ptr);
+
+/// Wrapper for httpPerformGetRequest() + httpWriteFileCallback() that opens/closes the output file on its own.
+/// Returns false if the request fails.
+bool httpDownloadFile(const char *path, const char *url, bool force_https, HttpProgressCallback progress_cb, void *progress_ptr);
+
+/// Wrapper for httpPerformGetRequest() + httpWriteBufferCallback() that manages a HttpBuffer element on its own.
+/// Returns a pointer to a dynamically allocated buffer that holds the downloaded data, which must be freed by the user.
+/// Providing 'outsize' is mandatory. Returns NULL if the request fails.
+char *httpDownloadData(size_t *outsize, const char *url, bool force_https, HttpProgressCallback progress_cb, void *progress_ptr);
 
 #ifdef __cplusplus
 }
 #endif
-*/
+
 #endif /* __HTTP_H__ */

include/defines.h

@@ -85,21 +85,23 @@
 #define NRO_NAME                        APP_TITLE ".nro"
 #define NRO_PATH                        APP_BASE_PATH NRO_NAME
 
-#define NSWDB_XML_PATH                  APP_BASE_PATH "NSWreleases.xml"
-
 #define KEYS_FILE_PATH                  "sdmc:" HBMENU_BASE_PATH "prod.keys"                                                    /* Location used by Lockpick_RCM. */
 
 #define LOG_FILE_NAME                   APP_TITLE ".log"
 #define LOG_BUF_SIZE                    0x400000                                                                                /* 4 MiB. */
 #define LOG_FORCE_FLUSH                 0                                                                                       /* Forces a log buffer flush each time the logfile is written to. */
 
-#define NXLINK_TIMEOUT                  2000
-
 #define BIS_SYSTEM_PARTITION_MOUNT_NAME "sys:"
 
+#define HTTP_USER_AGENT                 APP_TITLE "/" APP_VERSION " (Nintendo Switch)"
+#define HTTP_CONNECT_TIMEOUT            10L                                                                                     /* 10 seconds. */
+#define HTTP_BUFFER_SIZE                131072L                                                                                 /* 128 KiB. */
+
 #define GITHUB_REPOSITORY_URL           "https://github.com/DarkMatterCore/nxdumptool"
 #define GITHUB_NEW_ISSUE_URL            GITHUB_REPOSITORY_URL "/issues/new/choose"
 
+#define NSWDB_XML_PATH                  APP_BASE_PATH "NSWreleases.xml"
+
 #define BOREALIS_URL                    "https://github.com/natinusala/borealis"
 #define LIBUSBHSFS_URL                  "https://github.com/DarkMatterCore/libusbhsfs"
 #define FATFS_URL                       "http://elm-chan.org/fsw/ff/00index_e.html"

romfs/i18n/en-US/options_tab.json

@@ -15,6 +15,11 @@
         "value_01": "ID and version only"
     },
     
+    "update_nswdb_xml": {
+        "label": "Update NSWDB XML",
+        "description": "Retrieves the latest NSWDB XML, which can be optionally used to validate checksums from gamecard dumps. Requires Internet connectivity."
+    },
+    
     "update_app": {
         "label": "Update application",
         "description": "Checks if an update is available in nxdumptool's GitHub repository. Requires Internet connectivity."

source/core/config.c

@@ -41,6 +41,7 @@ if (!strcmp(key, #name)) { \
 vartype configGet##functype(const char *path) { \
     vartype ret = (vartype)0; \
     SCOPED_LOCK(&g_configMutex) { \
+        if (!g_configInterfaceInit) break; \
         struct json_object *obj = configGetJsonObjectByPath(g_configJson, path); \
         if (!obj || !configValidateJson##functype(obj, ##__VA_ARGS__)) break; \
         ret = (vartype)json_object_get_##jsontype(obj); \
@@ -51,6 +52,7 @@ vartype configGet##functype(const char *path) { \
 #define JSON_SETTER(functype, vartype, jsontype, ...) \
 void configSet##functype(const char *path, vartype value) { \
     SCOPED_LOCK(&g_configMutex) { \
+        if (!g_configInterfaceInit) break; \
         struct json_object *obj = configGetJsonObjectByPath(g_configJson, path); \
         if (!obj || !configValidateJson##functype(obj, ##__VA_ARGS__)) break; \
         if (json_object_set_##jsontype(obj, value)) { \

source/core/http.c

@@ -0,0 +1,235 @@
+/*
+ * http.c
+ *
+ * Copyright (c) 2020-2021, DarkMatterCore <pabloacurielz@gmail.com>.
+ *
+ * This file is part of nxdumptool (https://github.com/DarkMatterCore/nxdumptool).
+ *
+ * nxdumptool is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nxdumptool is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "nxdt_utils.h"
+#include "http.h"
+
+/* Global variables. */
+
+static Mutex g_httpMutex = 0;
+static bool g_httpInterfaceInit = false;
+
+bool httpInitialize(void)
+{
+    bool ret = false;
+    
+    SCOPED_LOCK(&g_httpMutex)
+    {
+        ret = g_httpInterfaceInit;
+        if (ret) break;
+        
+        /* Initialize CURL. */
+        CURLcode res = curl_global_init(CURL_GLOBAL_ALL);
+        if (res != CURLE_OK)
+        {
+            LOG_MSG("%s", curl_easy_strerror(res));
+            break;
+        }
+        
+        /* Update flags. */
+        ret = g_httpInterfaceInit = true;
+    }
+    
+    return ret;
+}
+
+void httpExit(void)
+{
+    SCOPED_LOCK(&g_httpMutex)
+    {
+        /* Cleanup CURL. */
+        curl_global_cleanup();
+        
+        /* Update flag. */
+        g_httpInterfaceInit = false;
+    }
+}
+
+size_t httpWriteFileCallback(char *buffer, size_t size, size_t nitems, void *outstream)
+{
+    size_t total_size = (size * nitems);
+    return (total_size ? fwrite(buffer, 1, total_size, (FILE*)outstream) : 0);
+}
+
+size_t httpWriteBufferCallback(char *buffer, size_t size, size_t nitems, void *outstream)
+{
+    size_t total_size = (size * nitems);
+    HttpBuffer *http_buffer = (HttpBuffer*)outstream;
+    
+    if (!total_size) return 0;
+    
+    char *data_tmp = realloc(http_buffer->data, http_buffer->size + total_size);
+    if (!data_tmp) return 0;
+    
+    http_buffer->data = data_tmp;
+    data_tmp = NULL;
+    
+    memcpy(http_buffer->data + http_buffer->size, buffer, total_size);
+    http_buffer->size += total_size;
+    
+    return total_size;
+}
+
+bool httpPerformGetRequest(const char *url, bool force_https, size_t *outsize, HttpWriteCallback write_cb, void *write_ptr, HttpProgressCallback progress_cb, void *progress_ptr)
+{
+    bool ret = false;
+    
+    SCOPED_LOCK(&g_httpMutex)
+    {
+        if (!g_httpInterfaceInit || !url || !*url)
+        {
+            LOG_MSG("Invalid parameters!");
+            break;
+        }
+        
+        CURL *curl = NULL;
+        CURLcode res = CURLE_OK;
+        long http_code = 0;
+        curl_off_t download_size = 0, content_length = 0;
+        char curl_err_buf[CURL_ERROR_SIZE] = {0};
+        
+        /* Start CURL session. */
+        curl = curl_easy_init();
+        if (!curl)
+        {
+            LOG_MSG("Failed to start CURL session for \"%s\"!", url);
+            break;
+        }
+        
+        /* Set CURL options. */
+        curl_easy_setopt(curl, CURLOPT_URL, url);
+        curl_easy_setopt(curl, CURLOPT_USERAGENT, HTTP_USER_AGENT);
+        curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_err_buf);
+        curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
+        curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
+        curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 50L);
+        curl_easy_setopt(curl, CURLOPT_TCP_KEEPALIVE, 1L);
+        curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING, "");
+        curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, HTTP_CONNECT_TIMEOUT);
+        curl_easy_setopt(curl, CURLOPT_BUFFERSIZE, HTTP_BUFFER_SIZE);
+        curl_easy_setopt(curl, CURLOPT_HTTP_VERSION, (long)(force_https ? CURL_HTTP_VERSION_2TLS : CURL_HTTP_VERSION_1_1));
+        
+        if (write_cb) curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_cb);
+        if (write_ptr) curl_easy_setopt(curl, CURLOPT_WRITEDATA, write_ptr);
+        
+        if (progress_cb)
+        {
+            curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);
+            curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION, progress_cb);
+        }
+        
+        if (progress_ptr) curl_easy_setopt(curl, CURLOPT_XFERINFODATA, progress_ptr);
+        
+        /* Perform GET request. */
+        res = curl_easy_perform(curl);
+        
+        /* Get HTTP request properties. */
+        curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &http_code);
+        curl_easy_getinfo(curl, CURLINFO_SIZE_DOWNLOAD_T, &download_size);
+        curl_easy_getinfo(curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD_T, &content_length);
+        
+        /* End CURL session. */
+        curl_easy_cleanup(curl);
+        
+        /* Update return value. */
+        ret = (res == CURLE_OK && http_code >= 200 && http_code <= 299 && (content_length <= 0 || download_size == content_length));
+        if (ret)
+        {
+            /* Update output size. */
+            if (outsize) *outsize = (size_t)download_size;
+        } else {
+            LOG_MSG("curl_easy_perform failed for \"%s\"! (res %d, HTTP code %ld, download %ld, length %ld).", url, res, http_code, download_size, content_length);
+            if (res != CURLE_OK)
+            {
+                /* Log CURL error info. */
+                if (*curl_err_buf)
+                {
+                    size_t curl_err_buf_len = strlen(curl_err_buf);
+                    
+                    if (curl_err_buf[curl_err_buf_len - 1] == '\n') curl_err_buf[--curl_err_buf_len] = '\0';
+                    if (curl_err_buf[curl_err_buf_len - 1] == '\r') curl_err_buf[--curl_err_buf_len] = '\0';
+                    
+                    LOG_MSG("%s", curl_err_buf);
+                } else {
+                    LOG_MSG("%s", curl_easy_strerror(res));
+                }
+            }
+        }
+    }
+    
+    return ret;
+}
+
+bool httpDownloadFile(const char *path, const char *url, bool force_https, HttpProgressCallback progress_cb, void *progress_ptr)
+{
+    if (!path || !*path)
+    {
+        LOG_MSG("Invalid parameters!");
+        return false;
+    }
+    
+    FILE *fd = NULL;
+    bool ret = false;
+    
+    /* Open output file. */
+    fd = fopen(path, "wb");
+    if (!fd)
+    {
+        LOG_MSG("Failed to open \"%s\" for writing!", path);
+        return false;
+    }
+    
+    /* Perform HTTP GET request. */
+    ret = httpPerformGetRequest(url, force_https, NULL, httpWriteFileCallback, fd, progress_cb, progress_ptr);
+    
+    /* Close output file. */
+    fclose(fd);
+    
+    /* Delete output file if the request failed. */
+    if (!ret) remove(path);
+    
+    return ret;
+}
+
+char *httpDownloadData(size_t *outsize, const char *url, bool force_https, HttpProgressCallback progress_cb, void *progress_ptr)
+{
+    if (!outsize)
+    {
+        LOG_MSG("Invalid parameters!");
+        return NULL;
+    }
+    
+    HttpBuffer http_buffer = {0};
+    bool ret = false;
+    
+    /* Perform HTTP GET request. */
+    ret = httpPerformGetRequest(url, force_https, outsize, httpWriteBufferCallback, &http_buffer, progress_cb, progress_ptr);
+    
+    /* Free output buffer if the request failed. */
+    if (!ret && http_buffer.data)
+    {
+        free(http_buffer.data);
+        http_buffer.data = NULL;
+    }
+    
+    return http_buffer.data;
+}

source/core/nxdt_utils.c

@@ -146,6 +146,10 @@ bool utilsInitializeResources(const int program_argc, const char **program_argv)
         /* Create output directories (SD card only). */
         utilsCreateOutputDirectories(NULL);
         
+        /* Initialize HTTP interface. */
+        /* CURL must be initialized before starting any other threads. */
+        if (!httpInitialize()) break;
+        
         /* Initialize USB interface. */
         if (!usbInitialize()) break;
         
@@ -189,7 +193,7 @@ bool utilsInitializeResources(const int program_argc, const char **program_argv)
             break;
         }
         
-        /* Load configuration. */
+        /* Initialize configuration interface. */
         if (!configInitialize()) break;
         
         /* Overclock system. */
@@ -210,17 +214,6 @@ bool utilsInitializeResources(const int program_argc, const char **program_argv)
         /* TODO: only use this function while dealing with a dump process - make sure to handle power button presses as well. */
         appletSetMediaPlaybackState(true);
         
-        /* Initialize socket driver. */
-        rc = socketInitializeDefault();
-        if (R_FAILED(rc))
-        {
-            LOG_MSG("socketInitializeDefault failed! (0x%08X).", rc);
-            break;
-        }
-        
-        /* Initialize CURL. */
-        curl_global_init(CURL_GLOBAL_ALL);
-        
         /* Redirect stdout and stderr over network to nxlink. */
         g_nxLinkSocketFd = nxlinkConnectToHost(true, true);
         
@@ -237,9 +230,6 @@ void utilsCloseResources(void)
 {
     SCOPED_LOCK(&g_resourcesMutex)
     {
-        /* Cleanup CURL. */
-        curl_global_cleanup();
-        
         /* Close nxlink socket. */
         if (g_nxLinkSocketFd >= 0)
         {
@@ -247,9 +237,6 @@ void utilsCloseResources(void)
             g_nxLinkSocketFd = -1;
         }
         
-        /* Deinitialize socket driver. */
-        socketExit();
-        
         /* Enable screen dimming and auto sleep. */
         /* TODO: only use this function while dealing with a dump process - make sure to handle power button presses as well. */
         appletSetMediaPlaybackState(false);
@@ -273,7 +260,7 @@ void utilsCloseResources(void)
         utilsUnmountEmmcBisSystemPartitionStorage();
         
         /* Deinitialize BFSAR interface. */
-        bfsarExit();
+        //bfsarExit();
         
         /* Deinitialize BFTTF interface. */
         bfttfExit();
@@ -293,6 +280,9 @@ void utilsCloseResources(void)
         /* Close USB interface. */
         usbExit();
         
+        /* Close HTTP interface. */
+        httpExit();
+        
         /* Close initialized services. */
         servicesClose();
         

source/core/services.c

@@ -63,7 +63,8 @@ static ServiceInfo g_serviceInfo[] = {
     { false, "es", NULL, &esInitialize, &esExit },
     { false, "set", NULL, &setInitialize, &setExit },
     { false, "set:sys", NULL, &setsysInitialize, &setsysExit },
-    { false, "set:cal", NULL, &setcalInitialize, &setcalExit }
+    { false, "set:cal", NULL, &setcalInitialize, &setcalExit },
+    { false, "bsd:u", NULL, &socketInitializeDefault, &socketExit }                                     /* socketInitialize*() functions take care of initializing bsd:* too. */
 };
 
 static const u32 g_serviceInfoCount = MAX_ELEMENTS(g_serviceInfo);

source/options_tab.cpp

@@ -75,6 +75,13 @@ namespace nxdt::views
         });
         this->addView(naming_convention);
         
+        /* Update NSWDB XML. */
+        brls::ListItem *update_nswdb_xml = new brls::ListItem("options_tab/update_nswdb_xml/label"_i18n, "options_tab/update_nswdb_xml/description"_i18n);
+        update_nswdb_xml->getClickEvent()->subscribe([this](brls::View* view) {
+            this->DisplayNotification("Not implemented.");
+        });
+        this->addView(update_nswdb_xml);
+        
         /* Update application. */
         brls::ListItem *update_app = new brls::ListItem("options_tab/update_app/label"_i18n, "options_tab/update_app/description"_i18n);
         update_app->getClickEvent()->subscribe([this](brls::View* view) {