From ac0e0430a0310b1f7fa2fe041f788066df0ce897 Mon Sep 17 00:00:00 2001 From: "Dmitry Pereverzev (SoUrcerer)" Date: Fri, 7 Jun 2013 16:48:19 +0000 Subject: [PATCH] Updated cURL emulation git-svn-id: svn://kolibrios.org@3616 a494cfbc-eb01-0410-851d-a64ba20cac60 --- .../netsurf/netsurf/content/fetchers/curl.c | 504 ++++++++---------- 1 file changed, 216 insertions(+), 288 deletions(-) diff --git a/programs/network/netsurf/netsurf/content/fetchers/curl.c b/programs/network/netsurf/netsurf/content/fetchers/curl.c index 9620460cd4..90dd29474e 100644 --- a/programs/network/netsurf/netsurf/content/fetchers/curl.c +++ b/programs/network/netsurf/netsurf/content/fetchers/curl.c @@ -63,97 +63,36 @@ #define FETCHER_CURLL_SCHEDULED 1 */ -typedef int X509_STORE_CTX; -typedef int X509; -#define CURL_ERROR_SIZE 0 -typedef int CURLcode; -typedef int curl_infotype; -/** SSL certificate info */ -struct cert_info { - X509 *cert; /**< Pointer to certificate */ - long err; /**< OpenSSL error code */ + +struct fetch_curl_context { + struct fetch_curl_context *r_next, *r_prev; + + struct fetch *fetchh; /**< Handle for this fetch */ + + bool aborted; /**< Flag indicating fetch has been aborted */ + bool locked; /**< Flag indicating entry is already entered */ + + nsurl *url; /**< The full url the fetch refers to */ + char *path; /**< The actual path to be used with open() */ + + time_t file_etag; /**< Request etag for file (previous st.m_time) */ }; -/** Information for a single fetch. */ -struct curl_fetch_info { - struct fetch *fetch_handle; /**< The fetch handle we're parented by. */ - CURL * curl_handle; /**< cURL handle if being fetched, or 0. */ - bool had_headers; /**< Headers have been processed. */ - bool abort; /**< Abort requested. */ - bool stopped; /**< Download stopped on purpose. */ - bool only_2xx; /**< Only HTTP 2xx responses acceptable. */ - bool downgrade_tls; /**< Downgrade to TLS <= 1.0 */ - nsurl *url; /**< URL of this fetch. */ - lwc_string *host; /**< The hostname of this fetch. */ - struct curl_slist *headers; /**< List of request headers. */ - char *location; /**< Response Location header, or 0. */ - unsigned long content_length; /**< Response Content-Length, or 0. */ - char *cookie_string; /**< Cookie string for this fetch */ - char *realm; /**< HTTP Auth Realm */ - char *post_urlenc; /**< Url encoded POST string, or 0. */ - long http_code; /**< HTTP result code from cURL. */ - struct curl_httppost *post_multipart; /**< Multipart post data, or 0. */ -#define MAX_CERTS 10 - struct cert_info cert_data[MAX_CERTS]; /**< HTTPS certificate data */ - unsigned int last_progress_update; /**< Time of last progress update */ -}; +static struct fetch_curl_context *ring = NULL; -struct cache_handle { - CURL *handle; /**< The cached cURL handle */ - lwc_string *host; /**< The host for which this handle is cached */ - struct cache_handle *r_prev; /**< Previous cached handle in ring. */ - struct cache_handle *r_next; /**< Next cached handle in ring. */ -}; - -CURLM *fetch_curl_multi; /**< Global cURL multi handle. */ -/** Curl handle with default options set; not used for transfers. */ -static CURL *fetch_blank_curl; -static struct cache_handle *curl_handle_ring = 0; /**< Ring of cached handles */ -static int curl_fetchers_registered = 0; -static bool curl_with_openssl; - -static char fetch_error_buffer[CURL_ERROR_SIZE]; /**< Error buffer for cURL. */ -static char fetch_proxy_userpwd[100]; /**< Proxy authentication details. */ - -static bool fetch_curl_initialise(lwc_string *scheme); -static void fetch_curl_finalise(lwc_string *scheme); -static bool fetch_curl_can_fetch(const nsurl *url); +static bool fetch_curl_initialise(lwc_string *scheme); //here +static void fetch_curl_finalise(lwc_string *scheme); //here +static bool fetch_curl_can_fetch(const nsurl *url); //here static void * fetch_curl_setup(struct fetch *parent_fetch, nsurl *url, bool only_2xx, bool downgrade_tls, const char *post_urlenc, const struct fetch_multipart_data *post_multipart, - const char **headers); -static bool fetch_curl_start(void *vfetch); -static bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, - CURL *handle); -static CURL *fetch_curl_get_handle(lwc_string *host); -static void fetch_curl_cache_handle(CURL *handle, lwc_string *host); -static CURLcode fetch_curl_set_options(struct curl_fetch_info *f); -static CURLcode fetch_curl_sslctxfun(CURL *curl_handle, void *_sslctx, - void *p); -static void fetch_curl_abort(void *vf); -static void fetch_curl_stop(struct curl_fetch_info *f); -static void fetch_curl_free(void *f); -static void fetch_curl_poll(lwc_string *scheme_ignored); -static void fetch_curl_done(CURL *curl_handle, CURLcode result); -static int fetch_curl_progress(void *clientp, double dltotal, double dlnow, - double ultotal, double ulnow); -static int fetch_curl_ignore_debug(CURL *handle, - curl_infotype type, - char *data, - size_t size, - void *userptr); -static size_t fetch_curl_data(char *data, size_t size, size_t nmemb, - void *_f); -static size_t fetch_curl_header(char *data, size_t size, size_t nmemb, - void *_f); -static bool fetch_curl_process_headers(struct curl_fetch_info *f); -static struct curl_httppost *fetch_curl_post_convert( - const struct fetch_multipart_data *control); -static int fetch_curl_verify_callback(int preverify_ok, - X509_STORE_CTX *x509_ctx); -static int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, - void *parm); + const char **headers); //here +static bool fetch_curl_start(void *vfetch); //here + +static void fetch_curl_abort(void *vf); //here +static void fetch_curl_free(void *f); //here +static void fetch_curl_poll(lwc_string *scheme_ignored); //here /** @@ -173,18 +112,18 @@ LOG(("curl register\n")); lwc_intern_string("http", SLEN("http"), &scheme); if (!fetch_add_fetcher(scheme, - fetch_curl_initialise, - fetch_curl_can_fetch, + fetch_curl_initialise, //here + fetch_curl_can_fetch, //here fetch_curl_setup, fetch_curl_start, - fetch_curl_abort, - fetch_curl_free, + fetch_curl_abort, //here + fetch_curl_free, //here #ifdef FETCHER_CURLL_SCHEDULED NULL, #else - fetch_curl_poll, + fetch_curl_poll, //here #endif - fetch_curl_finalise)) { + fetch_curl_finalise)) { //here LOG(("Unable to register cURL fetcher for HTTP")); } @@ -231,10 +170,10 @@ void fetch_curl_finalise(lwc_string *scheme) LOG(("curl finali\n")); } -bool fetch_curl_can_fetch(const nsurl *url) +static bool fetch_curl_can_fetch(const nsurl *url) { LOG(("curl can fetch\n")); - return false; + return true; //let's lie a bit } /** @@ -259,14 +198,38 @@ bool fetch_curl_can_fetch(const nsurl *url) * callbacks will contain this. */ -void * fetch_curl_setup(struct fetch *parent_fetch, nsurl *url, - bool only_2xx, bool downgrade_tls, const char *post_urlenc, +void * fetch_curl_setup (struct fetch *fetchh, + nsurl *url, + bool only_2xx, + bool downgrade_tls, + const char *post_urlenc, const struct fetch_multipart_data *post_multipart, const char **headers) { -LOG(("curl setup\n")); -return; + LOG(("curl setup\n")); + + struct fetch_curl_context *ctx; + int i; + + ctx = calloc(1, sizeof(*ctx)); + if (ctx == NULL) + return NULL; + + ctx->path = url_to_path(nsurl_access(url)); + if (ctx->path == NULL) { + free(ctx); + return NULL; + } + + ctx->url = nsurl_ref(url); + + + ctx->fetchh = fetchh; + + RING_INSERT(ring, ctx); + + return ctx; } @@ -276,93 +239,27 @@ return; bool fetch_curl_start(void *vfetch) { LOG(("curl start\n")); - return 0; + return true; } -/** - * Initiate a fetch from the queue. - * - * Called with a fetch structure and a CURL handle to be used to fetch the - * content. - * - * This will return whether or not the fetch was successfully initiated. - */ - -bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handle) -{ - LOG(("curl initi fetch\n")); - return 0; -} -/** - * Find a CURL handle to use to dispatch a job - */ - -CURL *fetch_curl_get_handle(lwc_string *host) -{ -LOG(("curl get handle\n")); - return 0; -} - - -/** - * Cache a CURL handle for the provided host (if wanted) - */ - -void fetch_curl_cache_handle(CURL *handle, lwc_string *host) -{ - LOG(("curl cache handle\n")); -} - - -/** - * Set options specific for a fetch. - */ - -typedef int CURLcode; - -CURLcode -fetch_curl_set_options(struct curl_fetch_info *f) -{ - LOG(("curl set options\n")); - return -1; -} - - -/** - * cURL SSL setup callback - */ - -CURLcode -fetch_curl_sslctxfun(CURL *curl_handle, void *_sslctx, void *parm) -{ - LOG(("curlcodessl\n")); - - return -1; -} /** * Abort a fetch. */ -void fetch_curl_abort(void *vf) +void fetch_curl_abort(void *ctx) { -LOG(("curl abort\n")); -} + struct fetch_curl_context *c = ctx; - -/** - * Clean up the provided fetch object and free it. - * - * Will prod the queue afterwards to allow pending requests to be initiated. - */ - -void fetch_curl_stop(struct curl_fetch_info *f) -{ - LOG(("curl stop\n")); + /* To avoid the poll loop having to deal with the fetch context + * disappearing from under it, we simply flag the abort here. + * The poll loop itself will perform the appropriate cleanup. + */ + c->aborted = true; } @@ -370,9 +267,119 @@ void fetch_curl_stop(struct curl_fetch_info *f) * Free a fetch structure and associated resources. */ -void fetch_curl_free(void *vf) +void fetch_curl_free(void *ctx) { - LOG(("curl free\n")); + struct fetch_curl_context *c = ctx; + nsurl_unref(c->url); + free(c->path); + RING_REMOVE(ring, c); + free(ctx); +} + +static inline bool fetch_curl_send_callback(const fetch_msg *msg, + struct fetch_curl_context *ctx) +{ + ctx->locked = true; + fetch_send_callback(msg, ctx->fetchh); + ctx->locked = false; + + return ctx->aborted; +} + +static bool fetch_curl_send_header(struct fetch_curl_context *ctx, + const char *fmt, ...) +{ + fetch_msg msg; + char header[64]; + va_list ap; + + va_start(ap, fmt); + + vsnprintf(header, sizeof header, fmt, ap); + + va_end(ap); + + msg.type = FETCH_HEADER; + msg.data.header_or_data.buf = (const uint8_t *) header; + msg.data.header_or_data.len = strlen(header); + fetch_curl_send_callback(&msg, ctx); + + return ctx->aborted; +} + +static void fetch_curl_process_error(struct fetch_curl_context *ctx, int code) +{ + fetch_msg msg; + char buffer[1024]; + const char *title; + char key[8]; + + /* content is going to return error code */ + fetch_set_http_code(ctx->fetchh, code); + + /* content type */ + if (fetch_curl_send_header(ctx, "Content-Type: text/html")) + goto fetch_file_process_error_aborted; + + snprintf(key, sizeof key, "HTTP%03d", code); + title = messages_get(key); + + snprintf(buffer, sizeof buffer, "%s" + "

%s

" + "

Error %d while fetching file %s

", + title, title, code, nsurl_access(ctx->url)); + + msg.type = FETCH_DATA; + msg.data.header_or_data.buf = (const uint8_t *) buffer; + msg.data.header_or_data.len = strlen(buffer); + if (fetch_curl_send_callback(&msg, ctx)) + goto fetch_file_process_error_aborted; + + msg.type = FETCH_FINISHED; + fetch_curl_send_callback(&msg, ctx); + +fetch_file_process_error_aborted: + return; +} + + +static void fetch_curl_process(struct fetch_curl_context *ctx) { + + fetch_msg msg; + const char * buf = "

Hello, file fetcher!

"; +// fetch_curl_process_error(ctx, 501); +// return; + +/* fetch is going to be successful */ + fetch_set_http_code(ctx->fetchh, 200); + + /* Any callback can result in the fetch being aborted. + * Therefore, we _must_ check for this after _every_ call to + * fetch_file_send_callback(). + */ + + /* content type */ + if (fetch_curl_send_header(ctx, "Content-Type: text/html")) + goto fetch_file_process_aborted; + + + /* main data loop */ + + msg.type = FETCH_DATA; + msg.data.header_or_data.buf = (const uint8_t *) buf; + msg.data.header_or_data.len = strlen(buf); + fetch_curl_send_callback(&msg, ctx); + + + + if (ctx->aborted == false) { + msg.type = FETCH_FINISHED; + fetch_curl_send_callback(&msg, ctx); + } + +fetch_file_process_aborted: +return; + } @@ -385,125 +392,46 @@ void fetch_curl_free(void *vf) void fetch_curl_poll(lwc_string *scheme_ignored) { LOG(("curl poll\n")); -} - - -/** - * Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()). - * - * \param curl_handle curl easy handle of fetch - */ - -void fetch_curl_done(CURL *curl_handle, CURLcode result) -{ - LOG(("curl done\n")); -} - - -/** - * Callback function for fetch progress. - */ - -int fetch_curl_progress(void *clientp, double dltotal, double dlnow, - double ultotal, double ulnow) -{ - LOG(("curl progress\n")); - return 0; -} - - - -/** - * Ignore everything given to it. - * - * Used to ignore cURL debug. - */ - -int fetch_curl_ignore_debug(CURL *handle, - curl_infotype type, - char *data, - size_t size, - void *userptr) -{ - LOG(("curl igdebug\n")); - return 0; -} - - -/** - * Callback function for cURL. - */ - -size_t fetch_curl_data(char *data, size_t size, size_t nmemb, - void *_f) -{ - LOG(("curl callback\n")); - return 0; -} - - -/** - * Callback function for headers. - * - * See RFC 2616 4.2. - */ - -size_t fetch_curl_header(char *data, size_t size, size_t nmemb, - void *_f) - -{ - LOG(("curl header\n")); - return 0; -} - -/** - * Find the status code and content type and inform the caller. - * - * Return true if the fetch is being aborted. - */ - -bool fetch_curl_process_headers(struct curl_fetch_info *f) -{ -LOG(("curl proc head\n")); - return false; -} - - -/** - * Convert a list of struct ::fetch_multipart_data to a list of - * struct curl_httppost for libcurl. - */ -struct curl_httppost * -fetch_curl_post_convert(const struct fetch_multipart_data *control) -{ - struct curl_httppost *post = 0; - LOG(("curl post - FAIL\n")); - return post; + struct fetch_curl_context *c, *next; + + if (ring == NULL) return; + + /* Iterate over ring, processing each pending fetch */ + c = ring; + do { + /* Ignore fetches that have been flagged as locked. + * This allows safe re-entrant calls to this function. + * Re-entrancy can occur if, as a result of a callback, + * the interested party causes fetch_poll() to be called + * again. + */ + if (c->locked == true) { + next = c->r_next; + continue; + } + + /* Only process non-aborted fetches */ + if (c->aborted == false) { + /* file fetches can be processed in one go */ + fetch_curl_process(c); + } + + /* Compute next fetch item at the last possible moment as + * processing this item may have added to the ring. + */ + next = c->r_next; + + fetch_remove_from_queues(c->fetchh); + fetch_free(c->fetchh); + + /* Advance to next ring entry, exiting if we've reached + * the start of the ring or the ring has become empty + */ + } while ( (c = next) != ring && ring != NULL); + } -/** - * OpenSSL Certificate verification callback - * Stores certificate details in fetch struct. - */ - -int fetch_curl_verify_callback(int preverify_ok, X509_STORE_CTX *x509_ctx) -{ - - LOG(("curl verify\n")); - return 0; -} -/** - * OpenSSL certificate chain verification callback - * Verifies certificate chain, setting up context for fetch_curl_verify_callback - */ - -int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, void *parm) -{ - -LOG(("curl cert verify\n")); - return 0; -}