httpclient.c

00001 #include "config.h"
00002 
00003 #ifdef _WINDOWS
00004 #include <Winsock2.h>
00005 #endif
00006 
00007 #ifdef CP_HAS_STRNDUP
00008 #ifndef _GNU_SOURCE /* required for strndup */
00009 #define _GNU_SOURCE
00010 #endif
00011 #endif /* CP_HAS_STRNDUP */
00012 #include <string.h>
00013 
00014 #include <stdio.h>
00015 #include <stdlib.h>
00016 #include <errno.h>
00017 
00018 
00019 #ifdef CP_HAS_STRINGS_H
00020 #include <strings.h>
00021 #endif
00022 
00023 #if defined(unix) || defined(__unix__) || defined(__MACH__)
00024 #include <unistd.h>
00025 #endif
00026 
00027 #ifdef CP_HAS_REGEX_H
00028 #include <regex.h>
00029 #else
00030 #include <pcreposix.h>
00031 #endif /* CP_HAS_REGEX_H */
00032 
00033 /* if poll(2) is available, use it. otherwise, use select (2) */
00034 #ifdef CP_HAS_POLL
00035 #include <sys/poll.h>
00036 #else
00037 #if defined(sun) || defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__APPLE__)
00038 #include <sys/types.h>
00039 #include <sys/socket.h>
00040 #endif /* defined */
00041 #ifdef CP_HAS_SYS_SELECT_H
00042 #include <sys/select.h>
00043 #endif /* CP_HAS_SYS_SELECT_H */
00044 #endif /* CP_HAS_POLL */
00045 
00046 #include "httpclient.h"
00047 #include "hashtable.h"
00048 #include "hashlist.h"
00049 #include "thread.h"
00050 #include "util.h"
00051 
00052 #define RESPONSE_INIT         0
00053 #define RESPONSE_STATUS_LINE  1
00054 #define RESPONSE_HEADERS      2
00055 #define RESPONSE_BODY         3
00056 #define RESPONSE_DONE         4
00057 
00058 static char *re_status_line_lit = 
00059     "^HTTP/([^[:space:]]*)[[:space:]]+([0-9]{3,})[[:space:]]+(.*)$";
00060 static char *re_response_header_lit = 
00061     "^([[:alnum:][:punct:]]+):[[:space:]]*(.*[^[:space:]])[[:space:]]*$";
00062 static char *re_url_lit = 
00063     "^(http(s)?:\\/\\/)?([^:\\/]+)(\\:[0-9]{1,5})?(\\/.*)?$";
00064 #ifdef CP_USE_COOKIES
00065 static char *re_cookie_lit = "(;[[:space:]]*(([^=;\n]+)(=([^=;\n]*))?))";
00066 #endif /* CP_USE_COOKIES */
00067 
00068 static regex_t re_status_line;
00069 static regex_t re_response_header;
00070 static regex_t re_url;
00071 #ifdef CP_USE_COOKIES
00072 static regex_t re_cookie;
00073 
00074 static cp_trie *master_cookie_jar = NULL;
00075 #endif /* CP_USE_COOKIES */
00076 
00077 static cp_httpclient_result *
00078     cp_httpclient_result_new(cp_http_transfer_descriptor *desc,
00079                              cp_http_result_status status, 
00080                              cp_http_response *response);
00081 
00082 static cp_httpclient_ctl *async_stack = NULL;
00083 static cp_httpclient_ctl *async_bg_stack = NULL;
00084 
00085 #define ASYNC_BG_THREAD_MAX 1
00086 static int init_async_bg();
00087 static void stop_async_bg();
00088 
00089 volatile short initialized = 0;
00090 
00091 int cp_httpclient_init()
00092 {
00093     int rc;
00094 
00095     if (initialized) return 1;
00096     initialized = 1;
00097 
00098     cp_client_init();
00099 
00100     if ((rc = regcomp(&re_status_line, 
00101                     re_status_line_lit, REG_EXTENDED | REG_NEWLINE)))
00102     {
00103         log_regex_compilation_error(rc, "compiling status line expression");
00104         goto INIT_ERROR;
00105     }
00106 
00107     if ((rc = regcomp(&re_response_header, 
00108                     re_response_header_lit, REG_EXTENDED | REG_NEWLINE)))
00109     {
00110         log_regex_compilation_error(rc, "compiling response header expression");
00111         goto INIT_ERROR;
00112     }
00113 
00114     if ((rc = regcomp(&re_url, re_url_lit, REG_EXTENDED | REG_NEWLINE)))
00115     {
00116         log_regex_compilation_error(rc, "compiling url expression");
00117         goto INIT_ERROR;
00118     }
00119 
00120 #ifdef CP_USE_COOKIES
00121     if ((rc = regcomp(&re_cookie, re_cookie_lit, REG_EXTENDED | REG_NEWLINE)))
00122     {
00123         log_regex_compilation_error(rc, "compiling cookie expression");
00124         goto INIT_ERROR;
00125     }
00126 
00127     master_cookie_jar = 
00128         cp_trie_create_trie(COLLECTION_MODE_DEEP, NULL, 
00129                             (cp_destructor_fn) cp_trie_destroy);
00130     if (master_cookie_jar == NULL)
00131     {
00132         cp_error(CP_MEMORY_ALLOCATION_FAILURE, "can\'t allocate cookie jar");
00133         goto INIT_ERROR;
00134     }
00135 #endif /* CP_USE_COOKIES */
00136     
00137     async_stack = cp_httpclient_ctl_create(0);
00138     if (async_stack == NULL)
00139     {
00140         cp_error(CP_MEMORY_ALLOCATION_FAILURE, 
00141                  "can\'t instantiate asynchronous transfer stack");
00142         goto INIT_ERROR;
00143     }
00144 
00145     return 0;
00146 
00147 INIT_ERROR:
00148     initialized = 0;
00149     return -1;
00150 }
00151 
00152 int cp_httpclient_shutdown()
00153 {
00154     if (initialized == 0) return 1;
00155     initialized = 0;
00156 
00157     if (async_bg_stack)
00158         stop_async_bg();
00159 
00160     regfree(&re_status_line);
00161     regfree(&re_response_header);
00162     regfree(&re_url);
00163 
00164 #ifdef CP_USE_COOKIES
00165     regfree(&re_cookie);
00166 
00167     cp_trie_destroy(master_cookie_jar);
00168 #endif /* CP_USE_COOKIES */
00169 
00170     cp_httpclient_ctl_destroy(async_stack);
00171 
00172     /* if using ssl, free ssl context objects that may have been allocated */
00173 #ifdef CP_USE_SSL
00174     cp_socket_shutdown();
00175 #endif /* CP_USE_SSL */
00176     cp_client_shutdown();
00177 
00178     return 0;
00179 }
00180 
00181 #define HTTP_RE_MATCHMAX 10
00182 
00183 /* ------------------------------------------------------------------------ *
00184  *                                cookies                                   *
00185  * ------------------------------------------------------------------------ */
00186 #ifdef CP_USE_COOKIES
00187 
00188 #ifndef CP_HAS_STRPTIME
00189 //  strptime(expires, "%a, %d-%b-%Y %H:%M:%S GMT", &c->expires);
00190 void parse_cookie_time(char *str, struct tm *res)
00191 {
00192     char day[8];
00193     sscanf(str, "%s, %2d-%2d-%4d %2d:%2d:%2d GMT", day, &res->tm_mday, &res->tm_mon, 
00194         &res->tm_year, &res->tm_hour, &res->tm_min, &res->tm_sec);
00195 }
00196 #endif /* CP_HAS_STRPTIME */
00197 
00201 cp_http_cookie *cp_http_cookie_create(char *content, 
00202                                       char *domain, 
00203                                       char *path, 
00204                                       char *expires, 
00205                                       int secure,
00206                                       int version,
00207                                       int http_only)
00208 {
00209     cp_http_cookie *c = calloc(1, sizeof(cp_http_cookie));
00210     if (c)
00211     {
00212         c->content = content;
00213         c->domain = domain;
00214         c->path = path;
00215         if (expires)
00216 #ifdef CP_HAS_STRPTIME
00217             strptime(expires, "%a, %d-%b-%Y %H:%M:%S GMT", &c->expires);
00218 #else
00219             parse_cookie_time(expires, &c->expires);        
00220 #endif
00221         c->secure = secure;
00222         c->version = version;
00223         c->http_only = http_only;
00224     }
00225 
00226     return c;
00227 }
00228 
00232 cp_http_cookie *cp_http_cookie_parse(char *src)
00233 {
00234     cp_http_cookie *ck;
00235     char *content;
00236     char *ptr = strchr(src, ';');
00237     
00238     if (ptr == NULL) 
00239         ck = cp_http_cookie_create(strdup(src), NULL, NULL, NULL, 0, 0, 0);
00240     else
00241     {
00242         regmatch_t rm[HTTP_RE_MATCHMAX];
00243         char *fld;
00244         char *domain = NULL;
00245         char *expires = NULL;
00246         char *path = NULL;
00247         int secure = 0;
00248         int version = 0;
00249         int http_only = 0;
00250         cp_hashtable *unknown = NULL;
00251 
00252         *ptr = '\0';
00253         content = strdup(src);
00254         *ptr = ';';
00255 
00256         while (regexec(&re_cookie, ptr, HTTP_RE_MATCHMAX, rm, 0) == 0)
00257         {
00258             fld = strndup(&ptr[rm[3].rm_so], rm[3].rm_eo - rm[3].rm_so);
00259 
00260             if (strcasecmp(fld, "domain") == 0 && rm[5].rm_so > -1)
00261                 domain = strndup(&ptr[rm[5].rm_so], rm[5].rm_eo - rm[5].rm_so);
00262             else if (strcasecmp(fld, "expires") == 0 && rm[5].rm_so > -1)
00263                 expires = strndup(&ptr[rm[5].rm_so], rm[5].rm_eo - rm[5].rm_so);
00264             else if (strcasecmp(fld, "path") == 0 && rm[5].rm_so > -1)
00265                 path = strndup(&ptr[rm[5].rm_so], rm[5].rm_eo - rm[5].rm_so);
00266             else if (strcasecmp(fld, "secure") == 0)
00267                 secure = 1;
00268             else if (strcasecmp(fld, "httpOnly") == 0)
00269                 http_only = 1;
00270             else if (strcasecmp(fld, "version") == 0 && rm[5].rm_so > -1)
00271                 version = atoi(&ptr[rm[5].rm_so]);
00272             else 
00273             {
00274                 char *value;
00275                 if (unknown == NULL) 
00276                     unknown = 
00277                         cp_hashtable_create_by_option(COLLECTION_MODE_DEEP | 
00278                                                       COLLECTION_MODE_NOSYNC,
00279                                                       1,
00280                                                       cp_hash_string,
00281                                                       cp_hash_compare_string,
00282                                                       NULL, free, NULL, free);
00283                 if (rm[5].rm_so > -1) 
00284                     value = strndup(&ptr[rm[5].rm_so], 
00285                                     rm[5].rm_eo - rm[5].rm_so);
00286                 else
00287                     value = strdup("1");
00288 
00289                 cp_hashtable_put(unknown, fld, value);
00290                 fld = NULL;
00291             }
00292             if (fld) free(fld);
00293             ptr = &ptr[rm[0].rm_eo];
00294         }
00295 
00296         ck = cp_http_cookie_create(content, domain, path, expires, 
00297                                    secure, version, http_only);
00298         if (expires) free(expires);
00299 
00300         ck->fld = unknown;
00301     }
00302     
00303     return ck;
00304 }
00305 
00307 void cp_http_cookie_destroy(cp_http_cookie *c)
00308 {
00309     if (c)
00310     {
00311         if (c->content) free(c->content);
00312         if (c->domain) free(c->domain);
00313         if (c->path) free(c->path);
00314         if (c->fld) cp_hashtable_destroy(c->fld);
00315         free(c);
00316     }
00317 }
00318 
00320 void cp_http_cookie_dump(cp_http_cookie *c)
00321 {
00322     char dbuf[0x100];
00323     
00324     cp_info("Cookie\n======\n");
00325     cp_info("content:   %s\n", c->content);
00326     cp_info("domain:    %s\n", c->domain ? c->domain : "");
00327     cp_info("path:      %s\n", c->path ? c->path : "");
00328     if (c->expires.tm_year)
00329     {
00330         strftime(dbuf, 0x100, "%a, %d-%b-%Y %H:%M:%S GMT", &c->expires);
00331         cp_info("expires:   %s\n", dbuf);
00332     }
00333     cp_info("secure:    %s\n", c->secure ? "yes" : "no");
00334     if (c->version) cp_info("version:   %d\n", c->version);
00335     if (c->http_only) cp_info("http only: %s\n", c->http_only ? "yes" : "no");
00336     if (c->fld)
00337     {
00338         int i;
00339         char **key;
00340         char *value;
00341         key = (char **) cp_hashtable_get_keys(c->fld);
00342         for (i = 0; i < cp_hashtable_count(c->fld); i++)
00343         {
00344             value = cp_hashtable_get(c->fld, key[i]);
00345             cp_info("%s:   %s\n", key[i], value);
00346         }
00347         free(key);
00348     }
00349 }
00350 
00351 void cp_httpclient_set_cookie_jar(cp_httpclient *client, cp_trie *jar)
00352 {
00353     client->cookie_jar = jar;
00354 }
00355 
00356 #endif /* CP_USE_COOKIES */
00357 
00358 cp_httpclient *cp_httpclient_allocate(char *host, int port)
00359 {
00360     cp_httpclient *client = calloc(1, sizeof(cp_httpclient));
00361     if (client == NULL) goto ALLOC_ERROR;
00362 
00363     client->header = 
00364         cp_hashtable_create_by_option(COLLECTION_MODE_COPY | 
00365                                       COLLECTION_MODE_DEEP | 
00366                                       COLLECTION_MODE_NOSYNC,
00367                                       5,
00368                                       cp_hash_istring,
00369                                       cp_hash_compare_istring,
00370                                       (cp_copy_fn) strdup, free,
00371                                       (cp_copy_fn) strdup, free);
00372     if (client->header == NULL) goto ALLOC_ERROR;
00373 
00374     client->host = host ? strdup(host) : NULL;
00375     client->port = port;
00376     client->version = HTTP_1_1;
00377     client->type = GET;
00378     client->pipeline_requests = 0;
00379     client->follow_redirects = 1;
00380     client->auto_drop_parameters = 1;
00381     client->auto_drop_headers = 0;
00382     client->max_redirects = DEFAULT_MAX_REDIRECTS;
00383 #ifdef CP_USE_COOKIES
00384     client->cookie_jar = master_cookie_jar;
00385 #endif
00386 
00387     return client;
00388 
00389 ALLOC_ERROR:
00390     cp_httpclient_destroy(client);
00391     return NULL;
00392 }
00393 
00394 cp_httpclient *cp_httpclient_create(char *host, int port)
00395 {
00396     cp_httpclient *client = cp_httpclient_allocate(host, port);
00397     if (client == NULL) goto CREATE_ERROR;
00398 
00399     client->socket = cp_client_create(host, port);
00400     if (client->socket == NULL) goto CREATE_ERROR;
00401 
00402     return client;
00403 
00404 CREATE_ERROR:
00405     cp_httpclient_destroy(client);
00406     return NULL;
00407 }
00408 
00409 cp_httpclient *
00410     cp_httpclient_create_proxy(char *host, int port, 
00411                                char *proxy_host, int proxy_port)
00412 {
00413     cp_httpclient *client = cp_httpclient_allocate(host, port);
00414     if (client == NULL) goto CREATE_ERROR;
00415 
00416     client->proxy_host = proxy_host ? strdup(proxy_host) : NULL;
00417     if (client->proxy_host == NULL) goto CREATE_ERROR;
00418     client->proxy_port = proxy_port;
00419 
00420     client->socket = cp_client_create(proxy_host, proxy_port);
00421     if (client->socket == NULL) goto CREATE_ERROR;
00422 
00423     return client;
00424 
00425 CREATE_ERROR:
00426     cp_httpclient_destroy(client);
00427     return NULL;
00428 }
00429 
00430 #ifdef CP_USE_SSL
00431 cp_httpclient *cp_httpclient_create_ssl(char *host, int port, 
00432                                         char *CA_file, char *CA_path, 
00433                                         int verification_mode)
00434 {
00435     cp_httpclient *client = cp_httpclient_allocate(host, port);
00436     if (client == NULL) goto CREATE_ERROR;
00437 
00438     client->socket = 
00439         cp_client_create_ssl(host, port, CA_file, CA_path, verification_mode);
00440     if (client->socket == NULL) goto CREATE_ERROR;
00441 
00442     return client;
00443 
00444 CREATE_ERROR:
00445     cp_httpclient_destroy(client);
00446     return NULL;
00447 }
00448 
00449 cp_httpclient *
00450     cp_httpclient_create_proxy_ssl(char *host, int port, 
00451                                    char *proxy_host, int proxy_port, 
00452                                    char *CA_file, char *CA_path, 
00453                                    int verification_mode)
00454 {
00455     cp_httpclient *client = cp_httpclient_allocate(host, port);
00456     if (client == NULL) goto CREATE_ERROR;
00457 
00458     if (client)
00459     {
00460         client->proxy_host = proxy_host ? strdup(proxy_host) : NULL;
00461         client->proxy_port = proxy_port;
00462     }
00463 
00464     client->socket = 
00465         cp_client_create_ssl(proxy_host, proxy_port, 
00466                              CA_file, CA_path, verification_mode);
00467 
00468     client->proxy_ssl = 1;
00469 
00470     return client;
00471 
00472 CREATE_ERROR:
00473     cp_httpclient_destroy(client);
00474     return NULL;
00475 }
00476 #endif /* CP_USE_SSL */
00477 
00478 void cp_httpclient_destroy(cp_httpclient *client)
00479 {
00480     if (client)
00481     {
00482         if (client->connected)
00483             cp_client_close(client->socket);
00484         if (client->socket)
00485             cp_client_destroy(client->socket);
00486         if (client->header)
00487             cp_hashtable_destroy(client->header);
00488         if (client->cgi_parameters)
00489             cp_hashtable_destroy(client->cgi_parameters);
00490         if (client->host) free(client->host);
00491         if (client->proxy_host) free(client->proxy_host);
00492         free(client);
00493     }
00494 }
00495 
00496 void cp_httpclient_set_http_version(cp_httpclient *client, 
00497                                      cp_http_version version)
00498 {
00499     client->version = version;
00500 }
00501 
00502 void cp_httpclient_set_request_type(cp_httpclient *client, 
00503                                      cp_http_request_type type)
00504 {
00505     client->type = type;
00506 }
00507 
00508 void cp_httpclient_set_header(cp_httpclient *client, char *header, char *value)
00509 {
00510     if (value != NULL)
00511         cp_hashtable_put(client->header, header, value);
00512     else
00513         cp_hashtable_remove(client->header, header);
00514 }
00515 
00516 void cp_httpclient_drop_headers(cp_httpclient *client)
00517 {
00518     if (client->header) cp_hashtable_remove_all(client->header);
00519 }
00520         
00521 void cp_httpclient_set_auto_drop_headers(cp_httpclient *client, short mode)
00522 {
00523     client->auto_drop_headers = mode;
00524 }
00525 
00526 void cp_httpclient_set_user_agent(cp_httpclient *client, char *agent)
00527 {
00528     cp_hashtable_put(client->header, "User-Agent", agent);
00529 }
00530 
00531 void cp_httpclient_set_timeout(cp_httpclient *client, 
00532                                int sec, int usec)
00533 {
00534     cp_client_set_timeout(client->socket, sec, usec);
00535 }
00536 
00537 void cp_httpclient_set_retry(cp_httpclient *client, int retry_count)
00538 {
00539     cp_client_set_retry(client->socket, retry_count);
00540 }
00541 
00542 void cp_httpclient_allow_redirects(cp_httpclient *client, int mode)
00543 {
00544     client->follow_redirects = mode;
00545 }
00546 
00547 void cp_httpclient_set_max_redirects(cp_httpclient *client, int max)
00548 {
00549     client->max_redirects = max;
00550 }
00551 
00552 int cp_httpclient_connect(cp_httpclient *client)
00553 {
00554     return cp_client_connect(client->socket);
00555 }
00556 
00557 #define LINEBUFSIZE 0x1000
00558 
00559 static cp_string *cgi_param_string(cp_httpclient *client)
00560 {
00561     cp_string *str = NULL;
00562 
00563     if (client->cgi_parameters)
00564     {
00565         int len = cp_hashtable_count(client->cgi_parameters);
00566         if (len)
00567         {
00568             int i;
00569             char *value;
00570             char **keys = (char **) cp_hashtable_get_keys(client->cgi_parameters);
00571             str = cp_string_create("", 0);
00572             for (i = 0; i < len; i++)
00573             {
00574                 if (i > 0) cp_string_append_char(str, '&');
00575                 value = cp_hashtable_get(client->cgi_parameters, keys[i]);
00576                 cp_string_cstrcat(str, keys[i]);
00577                 cp_string_append_char(str, '=');
00578                 cp_string_cstrcat(str, value);
00579             }
00580             free(keys);
00581         }
00582     }
00583     
00584     return str;
00585 }
00586 
00587 void *cp_httpclient_set_parameter(cp_httpclient *client, 
00588                                   char *name, char *value)
00589 {
00590     if (value == NULL)
00591     {
00592         if (client->cgi_parameters)
00593             return cp_hashtable_remove(client->cgi_parameters, name);
00594         return NULL;
00595     }
00596 
00597     if (client->cgi_parameters == NULL)
00598     {
00599         client->cgi_parameters = 
00600             cp_hashtable_create_by_option(COLLECTION_MODE_DEEP | 
00601                                           COLLECTION_MODE_COPY | 
00602                                           COLLECTION_MODE_NOSYNC, 
00603                                           1, 
00604                                           cp_hash_string, 
00605                                           cp_hash_compare_string, 
00606                                           (cp_copy_fn) strdup, free,
00607                                           (cp_copy_fn) strdup, free);
00608         if (client->cgi_parameters == NULL) return NULL;
00609     }
00610 
00611     return cp_hashtable_put(client->cgi_parameters, name, value);
00612 }
00613         
00614 void cp_httpclient_drop_parameters(cp_httpclient *client)
00615 {
00616     if (client->cgi_parameters) cp_hashtable_remove_all(client->cgi_parameters);
00617 }
00618         
00619 void cp_httpclient_set_auto_drop_parameters(cp_httpclient *client, short mode)
00620 {
00621     client->auto_drop_parameters = mode;
00622 }
00623 
00624 static void append_header(cp_string *buf, char *key, char *value)
00625 {
00626     cp_string_cstrcat(buf, key);
00627     cp_string_cstrcat(buf, ": ");
00628     cp_string_cstrcat(buf, value);
00629     cp_string_cstrcat(buf, "\r\n");
00630 }
00631 
00632 static void append_host_header(cp_httpclient *client, cp_string *request)
00633 {
00634     if (client->socket->host) 
00635     {
00636         int need_port = 0;
00637 #if CP_USE_SSL
00638         if (client->socket->use_ssl && client->port != 443)
00639             need_port = 1;
00640         else
00641 #endif /* CP_USE_SSL */
00642         if (client->port != 80)
00643             need_port = 1;
00644 
00645         if (need_port)
00646         {
00647             char *host;
00648             size_t len = strlen(client->host) + 7;
00649             host = (char *) malloc(len);
00650             if (host == NULL) return;
00651 #ifdef CP_HAS_SNPRINTF
00652             snprintf(host, len, "%s:%d", client->host, client->port);
00653 #else
00654             sprintf(host, "%s:%d", client->host, client->port);
00655 #endif /* CP_HAS_SNPRINTF */
00656             append_header(request, "Host", host);
00657             free(host);
00658         }
00659         else
00660             append_header(request, "Host", client->host);
00661     }
00662 }
00663 
00664 cp_string *cp_httpclient_format_request(cp_httpclient *client, char *uri)
00665 {
00666     char buf[LINEBUFSIZE];
00667     cp_string *request;
00668     int len;
00669 
00670     /* print request line */
00671     if (client->proxy_host == NULL)
00672 #ifdef CP_HAS_SNPRINTF
00673         snprintf(buf, LINEBUFSIZE, "%s %s", 
00674             get_http_request_type_lit(client->type), uri);
00675 #else
00676         sprintf(buf, "%s %s", get_http_request_type_lit(client->type), uri);
00677 #endif /* CP_HAS_SNPRINTF */
00678     else
00679     {
00680         char *method = "http";
00681         char port[8];
00682 #ifdef CP_USE_SSL
00683         if (client->socket->use_ssl) 
00684         {
00685             method = "https";
00686             if (client->port == 443) 
00687                 port[0] = '\0';
00688             else
00689 #ifdef CP_HAS_SNPRINTF
00690                 snprintf(port, 8, ":%d", client->proxy_port);
00691 #else
00692                 sprintf(port, ":%d", client->proxy_port);
00693 #endif /* CP_HAS_SNPRINTF */
00694         }
00695         else
00696 #endif /* CP_USE_SSL */
00697         {
00698             if (client->port == 80) 
00699                 port[0] = '\0';
00700             else
00701 #ifdef CP_HAS_SNPRINTF
00702                 snprintf(port, 8, ":%d", client->proxy_port);
00703 #else
00704                 sprintf(port, ":%d", client->proxy_port);
00705 #endif /* CP_HAS_SNPRINTF */
00706         }
00707 
00708 #ifdef CP_HAS_SNPRINTF
00709         snprintf(buf, LINEBUFSIZE, 
00710                  "%s %s://%s%s%s", get_http_request_type_lit(client->type), 
00711                  method, client->host, port, uri);
00712 #else
00713         sprintf(buf, "%s %s://%s%s%s", get_http_request_type_lit(client->type), 
00714                 method, client->host, port, uri);
00715 #endif /* CP_HAS_SNPRINTF */
00716     }
00717             
00718     request = cp_string_create(buf, strlen(buf));
00719 
00720     if (client->type == GET)
00721     {
00722         cp_string *prm = cgi_param_string(client);
00723         if (prm) /* may be null if no parameters */
00724         {
00725             cp_string_append_char(request, '?');
00726             cp_string_cat(request, prm);
00727             cp_string_destroy(prm);
00728         }
00729     }
00730 
00731     cp_string_cstrcat(request, " HTTP/");
00732     cp_string_cstrcat(request, client->version == HTTP_1_1 ? "1.1" : "1.0");
00733     cp_string_cstrcat(request, "\r\n");
00734     
00735     /* print headers */
00736     append_host_header(client, request);
00737     if (cp_hashtable_get(client->header, "User-Agent") == NULL)
00738         append_header(request, "User-Agent", DEFAULT_SERVER_NAME);
00739 
00740     if ((len = cp_hashtable_count(client->header)))
00741     {
00742         int i;
00743         char **key = (char **) cp_hashtable_get_keys(client->header);
00744         char *value;
00745 
00746         for (i = 0; i < len; i++)
00747         {
00748             value = cp_hashtable_get(client->header, key[i]);
00749             if (value) append_header(request, key[i], value);
00750         }
00751     }
00752     
00753     /* set Connection header if necessary */
00754     if (cp_hashtable_get(client->header, "Connection") == NULL)
00755     {
00756         if (client->version == HTTP_1_1)
00757         {
00758             append_header(request, "Connection", "keep-alive");
00759             append_header(request, "Keep-Alive", "300"); //~~
00760         }
00761         else
00762             append_header(request, "Connection", "close");
00763     }
00764     
00765     /* check cookie jar for applicable cookies */
00766 #ifdef CP_USE_COOKIES
00767     {
00768         cp_string *cookie_lit = NULL;
00769         char *domain = reverse_string(client->socket->host);
00770         cp_vector *urilist = cp_trie_fetch_matches(client->cookie_jar, domain);
00771         if (urilist)
00772         {
00773             int i, j;
00774             int len, jlen;
00775             cp_vector *cooklist;
00776             cp_http_cookie *ck;
00777 
00778             len = cp_vector_size(urilist);
00779             cookie_lit = cp_string_create_empty(80);
00780             for (i = 0; i < len; i++)
00781             {
00782                 cooklist = cp_trie_fetch_matches(client->cookie_jar, uri);
00783                 if (cooklist == NULL) continue;
00784                 jlen = cp_vector_size(cooklist);
00785                 for (j = 0; j < jlen; j++)
00786                 {
00787                     ck = cp_vector_element_at(cooklist, j);
00788                     if (cookie_lit->len > 0)
00789                         cp_string_cstrcat(cookie_lit, "; ");
00790                     cp_string_cstrcat(cookie_lit, ck->content);
00791                 }
00792                 if (cookie_lit->len > 0)
00793                     append_header(request, "Cookie", cookie_lit->data);
00794             }
00795             cp_string_destroy(cookie_lit);
00796             cp_vector_destroy(urilist);
00797         }
00798         free(domain);
00799     }
00800 #endif /* CP_USE_COOKIES */
00801     
00802     /* for POST requests, print request body */
00803     if (client->type == POST)
00804     {
00805         cp_string *prm = cgi_param_string(client);
00806         if (prm)
00807         {
00808             char len[16];
00809 #ifdef CP_HAS_SNPRINTF
00810             snprintf(len, 16, "%d", prm->len);
00811 #else
00812             sprintf(len, "%d", prm->len);
00813 #endif /* CP_HAS_SNPRINTF */
00814             append_header(request, 
00815                     "Content-Type", "application/x-www-form-urlencoded");
00816             append_header(request, "Content-Length", len);
00817             cp_string_cstrcat(request, "\r\n\r\n");
00818             cp_string_cat(request, prm);
00819             cp_string_destroy(prm);
00820         }
00821     }
00822 
00823     if (client->content)
00824     {
00825         char len[16];
00826 #ifdef CP_HAS_SNPRINTF
00827         snprintf(len, 16, "%d", client->content->len);
00828 #else
00829         sprintf(len, "%d", client->content->len);
00830 #endif /* CP_HAS_SNPRINTF */
00831         append_header(request, "Content-Length", len);
00832         cp_string_cstrcat(request, "\r\n\r\n");
00833         cp_string_cat(request, client->content);
00834     }
00835     
00836     cp_string_cstrcat(request, "\r\n");
00837     return request;
00838 }
00839 
00840 #ifdef CP_USE_SSL
00841 //~~ the right way to do this would be to add modes for 'PROXY_CONNECT_WRITE' 
00842 //~~ and 'PROXY_CONNECT_READ' to prevent blocking. coming in the next version.
00843 static int do_proxy_ssl_connect(cp_httpclient *client)
00844 {
00845     char cbuf[0x400];
00846     char *pbuf;
00847     int total;
00848     int len;
00849     
00850     int rc;
00851     cp_client *sock = client->socket;
00852     sock->use_ssl = 0;
00853     
00854     rc = cp_client_connect(sock);
00855     sock->use_ssl = 1;
00856     if (rc) return rc;
00857 
00858     if (client->port != 443)
00859 #ifdef CP_HAS_SNPRINTF
00860         snprintf(cbuf, 0x400, "CONNECT %s:%d HTTP/1.0\r\n\r\n", 
00861                 client->host, client->port);
00862 #else
00863         sprintf(cbuf, "CONNECT %s:%d HTTP/1.0\r\n\r\n", 
00864                 client->host, client->port);
00865 #endif /* CP_HAS_SNPRINTF */
00866     else
00867 #ifdef CP_HAS_SNPRINTF
00868         snprintf(cbuf, 0x400,  "CONNECT %s HTTP/1.0\r\n\r\n", client->host);
00869 #else
00870         sprintf(cbuf, "CONNECT %s HTTP/1.0\r\n\r\n", client->host);
00871 #endif /* CP_HAS_SNPRINTF */
00872 
00873     len = strlen(cbuf);
00874 
00875     for (total = 0; total < len; total+= rc)
00876     {
00877         rc = send(sock->fd, cbuf, len - total, 0);
00878         if (rc <= 0) break;
00879     }
00880     if (rc == 0) return -1;
00881     if (rc < 0) return rc;
00882 
00883     pbuf = cbuf;
00884     cbuf[0] = '\0';
00885     len = 0;
00886     do
00887     {
00888         rc = recv(sock->fd, pbuf, 0x400 - len, 0);
00889         if (rc == -1 && (errno == EAGAIN || errno == EINTR)) continue;
00890         if (rc <= 0) break;
00891         pbuf[rc] = '\0';
00892         sock->bytes_read += rc;
00893 #ifdef _HTTP_DUMP
00894         DEBUGMSG("\n---------------------------------------------------------------------------" "\n%s\n" "---------------------------------------------------------------------------", pbuf); 
00895 #endif /* _HTTP_DUMP */
00896         pbuf = &pbuf[rc];
00897     } while (strstr(cbuf, "\r\n\r\n") == NULL);
00898 
00899     if (rc < 0) return rc;
00900     if (strstr(cbuf, "HTTP/1.0 200 Connection established") == NULL)
00901         return -1;
00902 
00903     return cp_client_connect_ssl(sock);
00904 }
00905 #endif /* CP_USE_SSL */
00906 
00907 static int do_connect(cp_httpclient *client)
00908 {
00909     int rc;
00910     
00911 #ifdef CP_USE_SSL
00912     if (client->proxy_ssl)
00913         rc = do_proxy_ssl_connect(client);
00914     else
00915 #endif /* CP_USE_SSL */
00916     rc = cp_client_connect(client->socket);
00917     if (rc)
00918     {
00919 #ifdef CP_USE_SSL
00920         if (rc > 0)
00921         {
00922             cp_error(CP_SSL_VERIFICATION_ERROR, "%s:%d : %s", 
00923                      client->socket->host, client->socket->port, 
00924                      ssl_verification_error_str(rc));
00925         }
00926         else
00927 #endif
00928         {
00929             cp_error(CP_IO_ERROR, "can\'t connect to %s on port %d", 
00930                      client->socket->host, client->socket->port);
00931             return -1;
00932         }
00933     }
00934     client->connected = 1;
00935     return 0;
00936 }
00937 
00938 #ifdef __OpenBSD__
00939 #define HTTP_PARSE_BUFSIZE 0x1000
00940 #else
00941 #define HTTP_PARSE_BUFSIZE 0xFFFF
00942 #endif
00943 
00944 cp_http_response *cp_http_client_response_create()
00945 {
00946     cp_http_response *res = NULL;
00947     
00948     if ((res = (cp_http_response *) calloc(1, sizeof(cp_http_response))))
00949     {
00950         res->header = 
00951             cp_hashtable_create_by_option(COLLECTION_MODE_NOSYNC |
00952                                           COLLECTION_MODE_COPY |
00953                                           COLLECTION_MODE_DEEP,
00954                                           10, 
00955                                           cp_hash_istring, 
00956                                           cp_hash_compare_istring,
00957                                           (cp_copy_fn) strdup, free, 
00958                                           (cp_copy_fn) strdup, free);
00959         if (res->header == NULL)
00960         {
00961             free(res);
00962             return NULL;
00963         }
00964     }
00965 
00966     return res;
00967 }
00968 
00969 cp_url_descriptor *
00970     cp_url_descriptor_create(char *host, short ssl, int port, char *uri)
00971 {
00972     cp_url_descriptor *desc = calloc(1, sizeof(cp_url_descriptor));
00973     if (desc == NULL) return NULL;
00974 
00975     desc->host = host;
00976     desc->uri = uri;
00977 
00978     desc->ssl = ssl;
00979     desc->port = port;
00980 
00981     return desc;
00982 }
00983 
00984 void cp_url_descriptor_destroy(cp_url_descriptor *desc)
00985 {
00986     if (desc)
00987     {
00988         if (desc->host) free(desc->host);
00989         if (desc->uri) free(desc->uri);
00990         free(desc);
00991     }
00992 }
00993 
00994 short cp_url_descriptor_ssl(cp_url_descriptor *desc)
00995 {
00996     return desc->ssl;
00997 }
00998 
00999 char *cp_url_descriptor_host(cp_url_descriptor *desc)
01000 {
01001     return desc->host;
01002 }
01003 
01004 int cp_url_descriptor_port(cp_url_descriptor *desc)
01005 {
01006     return desc->port;
01007 }
01008 
01009 char *cp_url_descriptor_uri(cp_url_descriptor *desc)
01010 {
01011     return desc->uri;
01012 }
01013 
01014 cp_url_descriptor *cp_url_descriptor_parse(char *url)
01015 {
01016     cp_string *host;
01017     char *chost;
01018     short ssl;
01019     int port;
01020     cp_string *uri;
01021     char *curi;
01022     regmatch_t rm[6];
01023     
01024     if (url[0] == '/') /* it's not a url, it's a uri */
01025         return cp_url_descriptor_create(NULL, 0, 0, strdup(url));
01026     
01027     if (regexec(&re_url, url, 6, rm, 0))
01028     {
01029         cp_error(CP_HTTP_INVALID_URL, "can\'t parse url: %s\n", url);
01030         return NULL;
01031     }
01032     
01033     ssl = rm[2].rm_so != -1;
01034     host = cp_string_create(&url[rm[3].rm_so], rm[3].rm_eo - rm[3].rm_so);
01035     if (rm[4].rm_so != -1)
01036         port = atoi(&url[rm[4].rm_so + 1]);
01037     else
01038         port = ssl ? 443 : 80;
01039     if (rm[5].rm_so != -1)
01040         uri = cp_string_create(&url[rm[5].rm_so], rm[5].rm_eo - rm[5].rm_so);
01041     else
01042         uri = cp_string_create("/", 1);
01043 
01044     chost = cp_string_tocstr(host);
01045     free(host);
01046     curi = cp_string_tocstr(uri);
01047     free(uri);
01048 
01049     return cp_url_descriptor_create(chost, ssl, port, curi);
01050 }
01051 
01052 int read_status_line(cp_http_response *res, char **ptr, int len)
01053 {
01054     regmatch_t rm[HTTP_RE_MATCHMAX];
01055 
01056     if (regexec(&re_status_line, *ptr, HTTP_RE_MATCHMAX, rm, 0))
01057     {
01058         cp_error(CP_HTTP_INVALID_STATUS_LINE, "can\'t parse server response");
01059         return CP_HTTP_INVALID_STATUS_LINE;
01060     }
01061 
01062     if (strncmp(&(*ptr)[rm[1].rm_so], "1.1", 3) == 0)
01063         res->version = HTTP_1_1;
01064     else
01065         res->version = HTTP_1_0;
01066 
01067     res->status = (cp_http_status_code) atoi(&(*ptr)[rm[2].rm_so]);
01068     if (rm[3].rm_so > -1)
01069     {
01070         cp_string *status_lit = 
01071             cp_string_create(&(*ptr)[rm[3].rm_so], rm[3].rm_eo - rm[3].rm_so);
01072         res->status_lit = status_lit->data;
01073         free(status_lit);
01074     }
01075 
01076     *ptr = &(*ptr)[rm[0].rm_eo];
01077     return 0;
01078 }
01079 
01080 static char *find_headers_end(char *buf)
01081 {
01082     char *end = strstr(buf, "\r\n\r\n");
01083     if (end == NULL)
01084         end = strstr(buf, "\n\n");
01085     if (end == NULL)
01086         end = strstr(buf, "\n\r\n\r");
01087 
01088     return end;
01089 }
01090 
01091 int read_headers(cp_httpclient *client, cp_http_response *res, 
01092                  char *uri, char **ptr, int srclen)
01093 {
01094     char *curr = *ptr;
01095     int len;
01096     regmatch_t rm[HTTP_RE_MATCHMAX];
01097     char *name, *value;
01098     char *headers_end;
01099     char ch;
01100 
01101     while (*curr == '\r' || *curr == '\n') curr++; //~~ place curr properly 
01102 
01103     headers_end = find_headers_end(curr);
01104     if (headers_end == NULL) return EAGAIN;
01105     ch = *headers_end;
01106     *headers_end = '\0';
01107 
01108     while (regexec(&re_response_header, curr, HTTP_RE_MATCHMAX, rm, 0) == 0)
01109     {
01110         len = rm[1].rm_eo - rm[1].rm_so;
01111         if ((name = (char *) malloc((len + 1) * sizeof(char))) == NULL) 
01112             return CP_MEMORY_ALLOCATION_FAILURE;
01113         strncpy(name, curr + rm[1].rm_so, len);
01114         name[len] = '\0';
01115 
01116         len = rm[2].rm_eo - rm[2].rm_so;
01117         if ((value = (char *) malloc((len + 1) * sizeof(char))) == NULL) 
01118         {
01119             free(name);
01120             return CP_MEMORY_ALLOCATION_FAILURE;
01121         }
01122         strncpy(value, curr + rm[2].rm_so, len);
01123         value[len] = '\0';
01124 
01125 #ifdef CP_USE_COOKIES
01126         if (strcasecmp(name, "Set-Cookie") == 0)
01127         {
01128             cp_http_cookie *ck = cp_http_cookie_parse(value);
01129             cp_trie *uri_trie;
01130             if (ck)
01131             {
01132                 char *domain = NULL;
01133                 if (ck->domain)
01134                     domain = reverse_string(ck->domain);
01135                 else
01136                     domain = reverse_string(client->socket->host);
01137                 uri_trie = 
01138                     cp_trie_exact_match(client->cookie_jar, domain);
01139                 if (uri_trie == NULL)
01140                 {
01141                     uri_trie = 
01142                         cp_trie_create_trie(COLLECTION_MODE_DEEP, NULL,
01143                                 (cp_destructor_fn) cp_http_cookie_destroy);
01144                     cp_trie_add(client->cookie_jar, domain, uri_trie);
01145                 }
01146                 free(domain);
01147                 if (ck->path)
01148                     cp_trie_add(uri_trie, ck->path, ck);
01149                 else
01150                     cp_trie_add(uri_trie, uri, ck);
01151             }
01152             free(name);
01153             free(value);
01154         }
01155         else
01156 #endif /* CP_USE_COOKIES */
01157         cp_hashtable_put_by_option(res->header, name, value, 
01158             (cp_hashtable_get_mode(res->header) | COLLECTION_MODE_COPY) 
01159                 ^ COLLECTION_MODE_COPY);
01160         curr = &curr[rm[0].rm_eo];
01161     }
01162     *ptr = curr;
01163 
01164     *headers_end = ch;
01165     return 0;
01166 }
01167 
01168 static char *find_response_end(char *curr)
01169 {
01170     char *src = strstr(curr, "\r\n\r\n");
01171     if (src) 
01172         src += 4;
01173     else 
01174     {
01175         src = strstr(curr, "\n\n");
01176         if (src) src += 2;
01177     }
01178 
01179     return src;
01180 }
01181 
01182 // #if 0
01183 static int read_chunk(cp_httpclient *client, char *buf, int len)
01184 {
01185     int rc;
01186     while (1)
01187     {
01188 #ifdef CP_USE_SSL
01189         if (client->socket->use_ssl)
01190             rc = SSL_read(client->socket->ssl, buf, len);
01191         else
01192 #endif /* CP_USE_SSL */
01193         rc = recv(client->socket->fd, buf, len, 0);
01194         if (rc == -1 && (errno == EAGAIN || errno == EINTR)) continue;
01195         if (rc >= 0) 
01196         {
01197             buf[rc] = '\0';
01198             client->socket->bytes_read += rc;
01199 #ifdef _HTTP_DUMP
01200             cp_info("\n---------------------------------------------------------------------------" "\n%s\n" "---------------------------------------------------------------------------", buf); 
01201 #endif /* _HTTP_DUMP */
01202         }
01203         break;
01204     }
01205 
01206     return rc;
01207 }
01208 // #endif
01209 
01210 #define C2HEX(p) ((p) >= 'a' && (p) <= 'f' ? (p) - 'a' : \
01211                     (p) >= 'A' && (p) <= 'F' ? (p) - 'A' : (p) - '0')
01212 
01213 /* xtoid - xtoi with diagnostic: convert hex string to integer or return -1 if 
01214  * no conversion was done
01215  */
01216 int xtoid(char *p)
01217 {
01218     int curr;
01219     int res = -1;
01220 
01221     while (*p)
01222     {
01223         curr = C2HEX(*p);
01224         if (curr == -1) break;
01225         res = res * 0x10 + curr;
01226         p++;
01227     }
01228     
01229     return res;
01230 }
01231 
01232 static int read_chunked(cp_httpclient *client, 
01233                         cp_http_response *res, 
01234                         char *tmpbuf, 
01235                         int buflen,
01236                         char *curr)
01237 {
01238     int chunk;
01239     int inc, cat;
01240     int len = -1;
01241     res->content = NULL;
01242 
01243     while (1)
01244     {
01245         while (*curr == '\r' || *curr == '\n') curr++;
01246         if (*curr == 0)
01247         {
01248             curr = tmpbuf;
01249             buflen = read_chunk(client, tmpbuf, HTTP_PARSE_BUFSIZE);
01250 //          buflen = cp_client_read(client->socket, tmpbuf, HTTP_PARSE_BUFSIZE);
01251             if (buflen <= 0) break;
01252         }
01253 
01254         len = xtoi(curr);
01255         if (len == 0) break;
01256 
01257         curr = strchr(curr, '\n');
01258         if (curr == NULL) return -1; /* bad chunk format */
01259         curr++;
01260         inc = buflen - (curr - tmpbuf); 
01261         cat = (len < inc) ? len : inc;      
01262         if (res->content)
01263             cp_string_cat_bin(res->content, curr, cat);
01264         else
01265             res->content = cp_string_create(curr, cat);
01266         if (len < inc) curr = &curr[cat];
01267         len -= cat;
01268 
01269         for (chunk = 0; chunk < len; )
01270         {
01271             curr = tmpbuf;
01272             buflen = read_chunk(client, tmpbuf, HTTP_PARSE_BUFSIZE);
01273 //          buflen = cp_client_read(client->socket, tmpbuf, HTTP_PARSE_BUFSIZE);
01274             chunk += buflen;
01275             if (chunk <= len)
01276                 cp_string_cat_bin(res->content, tmpbuf, buflen);
01277             else
01278             {
01279                 cp_string_cat_bin(res->content, tmpbuf, buflen - (chunk - len));
01280                 curr = &tmpbuf[buflen - (chunk - len)];
01281             }
01282         }
01283     }
01284 
01285     return len;
01286 }
01287 
01288 static cp_http_response *read_response(cp_httpclient *client, char *uri)
01289 {
01290     char tmpbuf[HTTP_PARSE_BUFSIZE + 1];
01291     cp_string *buf = cp_string_create_empty(HTTP_PARSE_BUFSIZE);
01292     char *curr;
01293     cp_http_response *res = cp_http_client_response_create();
01294     int stage = 0;
01295     int rc;
01296     int skip = 0;
01297     
01298     curr = buf->data;
01299 
01300     while (stage < RESPONSE_DONE)
01301     {
01302         rc = read_chunk(client, tmpbuf, HTTP_PARSE_BUFSIZE);
01303 //      rc = cp_client_read(client->socket, tmpbuf, HTTP_PARSE_BUFSIZE);
01304         if (rc <= 0) break;
01305 
01306         cp_string_cat_bin(buf, tmpbuf, rc);
01307         buf->data[buf->len] = '\0';
01308         
01309         if (stage == RESPONSE_INIT)
01310         {
01311             if (read_status_line(res, 
01312                     &curr, buf->len - (curr - buf->data))) break;
01313             stage = RESPONSE_HEADERS;
01314         }
01315 
01316         if (stage == RESPONSE_HEADERS)
01317         {
01318             int rv;
01319             char *prm;
01320 
01321             rv = read_headers(client, res, uri, &curr, 
01322                              buf->len + skip - (curr - buf->data));
01323             if (rv == -1) 
01324                 break;
01325             else if (rv == EAGAIN) 
01326             {
01327                 skip += rc;
01328                 continue;
01329             }
01330 
01331             if ((prm = cp_hashtable_get(res->header, "Transfer-Encoding"))
01332                     && strcasecmp(prm, "chunked") == 0)
01333             {
01334                 int len;
01335                 curr = &tmpbuf[curr - buf->data];
01336                 len = read_chunked(client, res, tmpbuf, rc, curr);
01337                 if (len == -1) break;
01338                 if (len == 0) stage = RESPONSE_DONE;
01339                 break;
01340             }
01341             else if ((prm = cp_hashtable_get(res->header, "Content-Length")))
01342             {
01343                 int len;
01344                 char *src = find_response_end(curr);
01345                 if (src == NULL) break;
01346                 curr = src;
01347                 res->len = atoi(prm);
01348                 stage = RESPONSE_BODY;
01349                 res->content = 
01350                     cp_string_create(curr, rc + skip - (curr - buf->data));
01351                 len = res->len - res->content->len;
01352 
01353                 if (len > 0)
01354                     cp_client_read_string(client->socket, res->content, len);
01355                 stage = RESPONSE_DONE;
01356                 break;
01357             }
01358             /* some servers neglect to specify content length */
01359             else if (cp_hashtable_get(res->header, "Content-Type"))
01360             {
01361                 char *src = find_response_end(curr);
01362                 if (src == NULL) break; 
01363                 curr = src;
01364                 stage = RESPONSE_BODY;
01365                 res->content = 
01366                     cp_string_create(curr, rc + skip - (curr - buf->data));
01367                 continue;
01368             }
01369             else 
01370                 stage = RESPONSE_DONE;
01371         }
01372 
01373         if (stage == RESPONSE_BODY)
01374             cp_string_cat_bin(res->content, tmpbuf, rc);
01375     }
01376 
01377     cp_string_destroy(buf);
01378     
01379     if ((stage == RESPONSE_DONE) || (stage == RESPONSE_BODY && res->len == 0))
01380         return res;
01381 
01382     cp_error(CP_IO_ERROR, "error %s", 
01383             stage == RESPONSE_INIT ? "initializing response" : 
01384             stage == RESPONSE_HEADERS ? "reading headers" : "reading content");
01385 
01386     cp_http_response_delete(res);
01387     return NULL;
01388 }
01389 
01390 cp_httpclient_result *cp_httpclient_fetch(cp_httpclient *client, char *uri)
01391 {
01392     cp_httpclient_result *res;
01393     cp_string *request = cp_httpclient_format_request(client, uri);
01394     cp_http_response *response;
01395     int redirects = 0;
01396 
01397     res = cp_httpclient_result_create(client);
01398     if (res == NULL) return NULL;
01399     
01400     if (!client->connected) 
01401         if (do_connect(client))
01402         {
01403             res->result = CP_HTTP_RESULT_CONNECTION_FAILED;
01404             return res;
01405         }
01406     
01407     if (cp_client_write_string(client->socket, request) == 0)
01408     {
01409         if (do_connect(client)) 
01410         {
01411             res->result = CP_HTTP_RESULT_CONNECTION_FAILED;
01412             return res;
01413         }
01414         if (cp_client_write_string(client->socket, request) == 0)
01415         {
01416             cp_string_destroy(request);
01417             res->result = CP_HTTP_RESULT_WRITE_ERROR;
01418             return res;
01419         }
01420     }
01421     
01422     response = read_response(client, uri);
01423     if (response == NULL)
01424     {
01425         res->result = CP_HTTP_RESULT_READ_ERROR;
01426         return res;
01427     }
01428 
01429     while (client->follow_redirects && (response->status == HTTP_302_FOUND ||
01430             response->status == HTTP_301_MOVED_PERMANENTLY ||
01431             response->status == HTTP_300_MULTIPLE_CHOICES) && 
01432             redirects < client->max_redirects)
01433     {
01434         int rc;
01435         int retry = 3;
01436         cp_http_response *redirected;
01437         cp_url_descriptor *udesc;
01438         char *url = cp_hashtable_get(response->header, "Location");
01439 
01440         if (url == NULL) break;
01441         udesc = cp_url_descriptor_parse(url);
01442         if (udesc == NULL) break;
01443         cp_string_destroy(request);
01444 
01445         if (udesc->host == NULL) /* client returned a uri */
01446         {
01447             udesc->host = strdup(client->host);
01448             udesc->port = client->port;
01449 #ifdef CP_USE_SSL
01450             udesc->ssl = client->socket->use_ssl;
01451 #endif /* CP_USE_SSL */
01452         }
01453         else if (strcmp(udesc->host, client->host) || (udesc->port != client->port)) 
01454         {
01455             free(client->host);
01456             client->host = strdup(udesc->host);
01457             if (!client->proxy_host)/* if using proxy connection doesn't change */
01458                 cp_client_reopen(client->socket, udesc->host, udesc->port);
01459             //~~ what if redirecting to https address?
01460         }
01461         
01462         request = cp_httpclient_format_request(client, udesc->uri);
01463         while (cp_client_write_string(client->socket, request) == 0)
01464         {
01465             rc = errno;
01466             if (rc == EINTR || rc == EAGAIN) continue;
01467             if (rc == EPIPE && retry-- > 0) 
01468             {
01469                 cp_client_reopen(client->socket, udesc->host, udesc->port);
01470                 continue;
01471             }
01472             cp_string_destroy(request);
01473             res->result = CP_HTTP_RESULT_WRITE_ERROR;
01474             return res;
01475         }
01476 
01477         redirected = read_response(client, udesc->uri);
01478         if (redirected)
01479         {
01480             cp_http_response_delete(response);
01481             response = redirected;
01482             redirects++;
01483         }
01484         cp_url_descriptor_destroy(udesc);
01485     }
01486 
01487     cp_string_destroy(request);
01488 
01489     if (client->header && client->auto_drop_headers)
01490         cp_httpclient_drop_headers(client);
01491 
01492     if (client->cgi_parameters && client->auto_drop_parameters)
01493         cp_httpclient_drop_parameters(client);
01494 
01495     if (response == NULL)
01496     {
01497         res->result = CP_HTTP_RESULT_READ_ERROR;
01498         return res;
01499     }
01500 
01501     res->response = response;
01502     return res;
01503 }
01504 
01505 int cp_httpclient_reopen(cp_httpclient *client, char *host, int port)
01506 {
01507     client->connected = 0;
01508     return cp_client_reopen(client->socket, host, port);
01509 }
01510 
01511 /* ------------------------------------------------------------------------ *
01512  *                                async stack                               *
01513  * ------------------------------------------------------------------------ */
01514 
01515 #define TRANSFER_STAGE_CONNECT      1
01516 #define TRANSFER_STAGE_WRITE        2
01517 #define TRANSFER_STAGE_READ         3
01518 #define TRANSFER_STAGE_DONE         4
01519 
01520 #define TRANSFER_TYPE_LENGTH        1
01521 #define TRANSFER_TYPE_CHUNKED       2
01522 #define TRANSFER_TYPE_READ_TO_EOF   3
01523 
01524 static int set_async_stack_size(cp_httpclient_ctl *ctl, int size)
01525 {
01526     void *p;
01527     if (size > ctl->size)
01528     {
01529         ctl->size = size * 2;
01530 #ifdef CP_HAS_POLL
01531         p = realloc(ctl->ufds, ctl->size * sizeof(struct pollfd));
01532         if (p == NULL) return -1;
01533         ctl->ufds = p;
01534 // #else
01535 //      p = realloc(ctl->ufds, ctl->size * sizeof(int));
01536 #endif
01537         p = realloc(ctl->desc, 
01538                     ctl->size * sizeof(cp_http_transfer_descriptor *));
01539         if (p == NULL) return -1;
01540         ctl->desc = p;
01541     }
01542     ctl->nfds = size;
01543     return 0;
01544 }
01545 
01546 cp_httpclient_result *cp_httpclient_result_create(cp_httpclient *client)
01547 {
01548     cp_httpclient_result *res = calloc(1, sizeof(cp_httpclient_result));
01549     if (res == NULL) return NULL;
01550     res->client = client;
01551     return res;
01552 }
01553 
01554 static cp_httpclient_result *
01555     cp_httpclient_result_new(cp_http_transfer_descriptor *desc,
01556                              cp_http_result_status status, 
01557                              cp_http_response *response)
01558 {
01559     cp_httpclient_result *res = calloc(1, sizeof(cp_httpclient_result));
01560     if (res == NULL) goto RESULT_ERROR;
01561     res->id = desc->id;
01562     res->client = desc->owner;
01563     res->result = status;
01564     res->response = response;
01565 
01566     return res;
01567 
01568 RESULT_ERROR:
01569     cp_httpclient_result_destroy(res);
01570     return NULL;
01571 }
01572 
01573 void cp_httpclient_result_destroy(cp_httpclient_result *res)
01574 {
01575     if (res) 
01576     {
01577         if (res->response) cp_http_response_delete(res->response);
01578         free(res);
01579     }
01580 }
01581 
01582 void *cp_httpclient_result_id(cp_httpclient_result *res)
01583 {
01584     return res->id;
01585 }
01586 
01587 cp_http_result_status cp_httpclient_result_status(cp_httpclient_result *res)
01588 {
01589     return res->result;
01590 }
01591 
01592 cp_http_response *cp_httpclient_result_get_response(cp_httpclient_result *res)
01593 {
01594     return res->response;
01595 }
01596 
01597 cp_httpclient *cp_httpclient_result_get_client(cp_httpclient_result *res)
01598 {
01599     return res->client;
01600 }
01601 
01602 static void 
01603     cp_http_transfer_descriptor_callback(cp_http_transfer_descriptor *desc)
01604 {
01605     cp_httpclient_result *res;
01606     if (desc->callback)
01607     {
01608         res = cp_httpclient_result_new(desc,
01609             (cp_http_result_status) desc->status, desc->res);
01610         if (res) //~~ handle error
01611         {
01612             desc->callback(res);
01613             cp_httpclient_result_destroy(res);
01614         }
01615     }
01616 }
01617 
01618 static void prepare_async_stack_fds(cp_httpclient_ctl *ctl)
01619 {
01620     cp_hashlist_iterator *i;
01621     int j;
01622     int rc;
01623     cp_http_transfer_descriptor *desc;
01624 
01625     i = cp_hashlist_create_iterator(ctl->stack, COLLECTION_LOCK_READ);
01626     j = 0;
01627 #ifndef CP_HAS_POLL
01628     FD_ZERO(&ctl->ufds_r);
01629     FD_ZERO(&ctl->ufds_w);
01630     ctl->fdmax = 0;
01631 #endif
01632     while ((desc = cp_hashlist_iterator_next_value(i)) != NULL)
01633     {
01634         if (desc->stage == TRANSFER_STAGE_CONNECT)
01635         {
01636             if (!desc->owner->connected)
01637             {
01638                 rc = do_connect(desc->owner);
01639                 if (rc)
01640                 {
01641                     cp_hashlist_iterator_remove(i);
01642                     desc->stage = TRANSFER_STAGE_DONE;
01643                     desc->status = CP_HTTP_RESULT_CONNECTION_FAILED;
01644                     continue;
01645                 }
01646             }
01647             desc->stage = TRANSFER_STAGE_WRITE;
01648         }
01649 
01650         ctl->desc[j] = desc;
01651 #ifdef CP_HAS_POLL
01652         ctl->ufds[j].fd = desc->owner->socket->fd;
01653         ctl->ufds[j].events = 0;
01654         if (desc->stage == TRANSFER_STAGE_WRITE)
01655             ctl->ufds[j].events |= POLLOUT;
01656         else
01657             ctl->ufds[j].events |= POLLIN;
01658 #else
01659         if (desc->stage == TRANSFER_STAGE_WRITE)
01660             FD_SET(desc->owner->socket->fd, &ctl->ufds_w);
01661         else
01662             FD_SET(desc->owner->socket->fd, &ctl->ufds_r);
01663         if (desc->owner->socket->fd > ctl->fdmax)
01664             ctl->fdmax = desc->owner->socket->fd;
01665 #endif /* CP_HAS_POLL */
01666         j++;
01667     }
01668     ctl->nfds = j;
01669     cp_hashlist_iterator_destroy(i);
01670 }
01671 
01672 static void 
01673     cp_http_transfer_descriptor_do_write(cp_http_transfer_descriptor *desc)
01674 {
01675     int rc;
01676     char *index = &desc->buf->data[desc->buf->len - desc->remaining];
01677 #ifdef CP_USE_SSL
01678     if (desc->owner->socket->use_ssl)
01679         rc = SSL_write(desc->owner->socket->ssl, index, desc->remaining);
01680     else
01681 #endif /* CP_USE_SSL */
01682     rc = send(desc->owner->socket->fd, index, desc->remaining, 0);
01683 
01684     if (rc == -1)
01685     {
01686         int err = errno;
01687         if (!(err == EAGAIN || err == EINTR))
01688         {
01689             desc->stage = TRANSFER_STAGE_DONE;
01690             desc->status = CP_HTTP_RESULT_WRITE_ERROR;
01691             return;
01692         }
01693     }
01694     else
01695         desc->owner->socket->bytes_sent += rc;
01696 
01697     desc->remaining -= rc;
01698     if (desc->remaining == 0)
01699     {
01700         desc->stage = TRANSFER_STAGE_READ;
01701         desc->buf->len = 0;
01702     }
01703 }
01704 
01705 static void transfer_chunked(cp_http_transfer_descriptor *desc, 
01706                              char *tmpbuf, 
01707                              int index, 
01708                              int len)
01709 {
01710     char *curr = &tmpbuf[index];
01711     int read_len;
01712 
01713     if (desc->remaining) 
01714     {
01715         read_len = len - index;
01716         if (read_len <= desc->remaining) /* need another read before next chunk */
01717         {
01718             cp_string_cat_bin(desc->res->content, curr, read_len);
01719             desc->remaining -= read_len;
01720             return;
01721         }
01722         else
01723         {
01724             cp_string_cat_bin(desc->res->content, curr, desc->remaining);
01725             curr = &tmpbuf[index + desc->remaining];
01726             desc->remaining = 0;
01727         }
01728     }
01729     
01730     while (*curr == '\r' || *curr == '\n') curr++;
01731     if (*curr == 0) return; 
01732 
01733     read_len = xtoi(curr);
01734     if (read_len == 0) /* last chunk length is 0 - done reading */
01735     {
01736         desc->stage = TRANSFER_STAGE_DONE;
01737         desc->status = CP_HTTP_RESULT_SUCCESS;
01738         return;
01739     }
01740     else
01741     {
01742         char *str = strchr(curr, '\n');
01743         if (str == NULL) 
01744         {
01745             desc->stage = TRANSFER_STAGE_DONE;
01746             desc->status = CP_HTTP_RESULT_READ_ERROR;
01747         }
01748         curr = str + 1;
01749     }
01750     desc->remaining = read_len;
01751 
01752     
01753     /* recursive call for the case there's still more in the buffer */
01754     if (curr - tmpbuf < len)
01755         transfer_chunked(desc, tmpbuf, (curr - tmpbuf), len);
01756 }
01757 
01758 static void 
01759     cp_http_transfer_descriptor_do_read(cp_http_transfer_descriptor *desc)
01760 {
01761     char tmpbuf[HTTP_PARSE_BUFSIZE + 1];
01762     int read_len;
01763     int rc;
01764 
01765     /* perform a read */
01766     if (desc->remaining && desc->remaining < HTTP_PARSE_BUFSIZE)
01767         read_len = desc->remaining;
01768     else 
01769         read_len = HTTP_PARSE_BUFSIZE;
01770 
01771     rc = read_chunk(desc->owner, tmpbuf, read_len);
01772 //  rc = cp_client_read(desc->owner->socket, tmpbuf, read_len);
01773 
01774     if (rc < 0) goto READ_ERROR;
01775     if (rc == 0)
01776     {
01777         if (desc->transfer != TRANSFER_TYPE_READ_TO_EOF) goto READ_ERROR;
01778         desc->stage = TRANSFER_STAGE_DONE;
01779         desc->status = CP_HTTP_RESULT_SUCCESS;
01780         return;
01781     }
01782 
01783     /* handle response body according to transfer type */
01784     if (desc->read_stage == RESPONSE_BODY)
01785     {
01786         switch (desc->transfer)
01787         {
01788             case TRANSFER_TYPE_CHUNKED: 
01789                 transfer_chunked(desc, tmpbuf, 0, rc); 
01790                 break;
01791                 
01792             case TRANSFER_TYPE_LENGTH: 
01793                 cp_string_cat_bin(desc->res->content, tmpbuf, rc);
01794                 desc->remaining -= rc;
01795                 if (desc->remaining == 0)
01796                 {
01797                     desc->stage = TRANSFER_STAGE_DONE;
01798                     desc->status = CP_HTTP_RESULT_SUCCESS;
01799                 }
01800                 break;
01801 
01802             case TRANSFER_TYPE_READ_TO_EOF:
01803                 cp_string_cat_bin(desc->res->content, tmpbuf, rc);
01804                 break;
01805         }
01806 
01807         return;
01808     }
01809     
01810     /* still reading header - concatenate to header buffer */
01811     cp_string_cat_bin(desc->buf, tmpbuf, rc);
01812     desc->buf->data[desc->buf->len] = '\0';
01813 
01814     if (desc->read_stage == RESPONSE_INIT)
01815     {
01816         desc->read_ptr = desc->buf->data;
01817         desc->res = cp_http_client_response_create();
01818         if (read_status_line(desc->res, &desc->read_ptr, desc->buf->len)) 
01819             return;
01820         
01821         desc->read_stage = RESPONSE_HEADERS;
01822     }
01823 
01824     /* read headers, determing transfer type and begin transfer */
01825     if (desc->read_stage == RESPONSE_HEADERS)
01826     {
01827         int rv;
01828         char *prm;
01829 
01830         rv = read_headers(desc->owner, desc->res, desc->uri, &desc->read_ptr, 
01831                 desc->buf->len + desc->skip - 
01832                 (desc->read_ptr - desc->buf->data)); 
01833         if (rv == -1)
01834             return; //~~ fail
01835         else if (rv == EAGAIN) 
01836         {
01837             desc->skip += rc;
01838             return;
01839         }
01840 
01841         desc->read_stage = RESPONSE_BODY;
01842         if ((prm = cp_hashtable_get(desc->res->header, "Transfer-Encoding"))
01843             && strcasecmp(prm, "chunked") == 0)
01844         {
01845             desc->transfer = TRANSFER_TYPE_CHUNKED;
01846 
01847             desc->res->content = cp_string_create_empty(0x400);
01848             transfer_chunked(desc, tmpbuf, 
01849                              desc->read_ptr - desc->buf->data, rc);
01850             return;
01851         }
01852         else if ((prm = cp_hashtable_get(desc->res->header, "Content-Length")))
01853         {
01854             char *src = find_response_end(desc->read_ptr);
01855             if (src == NULL) goto READ_ERROR;
01856             desc->transfer = TRANSFER_TYPE_LENGTH;
01857 
01858             desc->read_ptr = src;
01859             desc->res->len = atoi(prm);
01860             desc->res->content = 
01861                 cp_string_create(desc->read_ptr, 
01862                              rc + desc->skip - 
01863                              (desc->read_ptr - desc->buf->data));
01864             desc->remaining = 
01865                 desc->res->len - desc->res->content->len;
01866             if (desc->remaining == 0)
01867             {
01868                 desc->stage = TRANSFER_STAGE_DONE;
01869                 desc->status = CP_HTTP_RESULT_SUCCESS;
01870                 return;
01871             }
01872         }
01873         else if (cp_hashtable_get(desc->res->header, "Content-Type"))
01874         {
01875             char *src = find_response_end(desc->read_ptr);
01876             if (src == NULL) goto READ_ERROR;
01877             desc->transfer = TRANSFER_TYPE_READ_TO_EOF;
01878 
01879             desc->read_ptr = src;
01880             desc->res->content = 
01881                 cp_string_create(desc->read_ptr, 
01882                          rc + desc->skip - (desc->read_ptr - desc->buf->data));
01883             return;
01884         }
01885     }
01886 
01887     return;
01888 
01889 READ_ERROR:
01890     desc->stage = TRANSFER_STAGE_DONE;
01891     desc->status = CP_HTTP_RESULT_READ_ERROR;
01892 }
01893 
01895 static void
01896     cp_http_transfer_descriptor_do_redirect(cp_http_transfer_descriptor *desc)
01897 {
01898     if (desc->owner->follow_redirects && 
01899         (desc->res->status == HTTP_302_FOUND ||
01900          desc->res->status == HTTP_301_MOVED_PERMANENTLY ||
01901          desc->res->status == HTTP_300_MULTIPLE_CHOICES) &&
01902         desc->redirects < desc->owner->max_redirects)
01903     {
01904         cp_url_descriptor *udesc;
01905         char *url = cp_hashtable_get(desc->res->header, "Location");
01906         if (url == NULL) return;
01907         udesc = cp_url_descriptor_parse(url);
01908         if (udesc == NULL) return;
01909 
01910         if (udesc->host == NULL) 
01911         {
01912             udesc->host = strdup(desc->owner->host);
01913             udesc->port = desc->owner->port;
01914 #ifdef CP_USE_SSL
01915             udesc->ssl = desc->owner->socket->use_ssl;
01916 #endif /* CP_USE_SSL */
01917         }
01918         else if (strcmp(udesc->host, desc->owner->host) || 
01919             udesc->port != desc->owner->port)
01920         {
01921             free(desc->owner->host);
01922             desc->owner->host = strdup(udesc->host);
01923             if (!desc->owner->proxy_host)
01924                 cp_client_reopen(desc->owner->socket, udesc->host, udesc->port);
01925             //~~ redirect to https issue
01926         }
01927 
01928         cp_string_destroy(desc->buf);
01929         desc->buf = cp_httpclient_format_request(desc->owner, udesc->uri);
01930         desc->remaining = desc->buf->len;
01931         desc->stage = TRANSFER_STAGE_WRITE; //~~ or STAGE_CONNECT 
01932         desc->read_stage = RESPONSE_INIT;
01933         cp_http_response_delete(desc->res);
01934         desc->res = NULL;
01935         desc->redirects++;
01936 
01937         cp_url_descriptor_destroy(udesc);
01938     }
01939 }
01940 
01941 
01943 typedef struct _async_struct_entry
01944 {
01945     cp_httpclient_ctl *ctl;
01946     int index;
01947 } async_struct_entry;
01948 
01949 static void *process_async_stack_entry(void *prm)
01950 {
01951     async_struct_entry *entry = prm;
01952     cp_http_transfer_descriptor *desc;
01953     cp_httpclient_ctl *ctl = entry->ctl;
01954     int i = entry->index;
01955 
01956     desc = ctl->desc[i];
01957     if (desc->stage == TRANSFER_STAGE_WRITE)
01958     {
01959 #ifdef CP_HAS_POLL
01960         if (ctl->ufds[i].revents & POLLOUT)
01961 #else
01962         if (FD_ISSET(desc->owner->socket->fd, &ctl->ufds_w))
01963 #endif /* CP_HAS_POLL */
01964             cp_http_transfer_descriptor_do_write(desc);
01965     }
01966     /* don't try to read before making another poll()/select() */
01967     else if (desc->stage == TRANSFER_STAGE_READ)
01968     {
01969 #ifdef CP_HAS_POLL
01970         if (ctl->ufds[i].revents & (POLLIN | POLLPRI))
01971 #else
01972         if (FD_ISSET(desc->owner->socket->fd, &ctl->ufds_r))
01973 #endif /* CP_HAS_POLL */
01974         {
01975             cp_http_transfer_descriptor_do_read(desc);
01976             if (desc->res)
01977                 cp_http_transfer_descriptor_do_redirect(desc);
01978         }
01979     }
01980 
01981     /* no need to poll()/select() again if already done */
01982     if (desc->stage == TRANSFER_STAGE_DONE)
01983     {
01984         cp_hashlist_remove(ctl->stack, desc);
01985         cp_http_transfer_descriptor_callback(desc);
01986     }
01987     
01988     free(entry);
01989 
01990     return NULL;
01991 }
01992 
01993 static void scan_async_stack(cp_httpclient_ctl *ctl)
01994 {
01995     int i;
01996     cp_http_transfer_descriptor *desc;
01997 
01998     if (ctl->pool)
01999     {
02000         for (i = 0; i < ctl->nfds; i++)
02001         {
02002             async_struct_entry *entry = calloc(1, sizeof(async_struct_entry));
02003             entry->ctl = ctl;
02004             entry->index = i;
02005             cp_thread_pool_get(ctl->pool, process_async_stack_entry, entry);
02006         }
02007         cp_thread_pool_wait(ctl->pool);
02008         return;
02009     }
02010         
02011     for (i = 0; i < ctl->nfds; i++)
02012     {
02013         desc = ctl->desc[i];
02014         if (desc->stage == TRANSFER_STAGE_WRITE)
02015         {
02016 #ifdef CP_HAS_POLL
02017             if (ctl->ufds[i].revents & POLLOUT)
02018 #else
02019             if (FD_ISSET(desc->owner->socket->fd, &ctl->ufds_w))
02020 #endif /* CP_HAS_POLL */
02021                 cp_http_transfer_descriptor_do_write(desc);
02022         }
02023         /* don't try to read before making another poll()/select() */
02024         else if (desc->stage == TRANSFER_STAGE_READ)
02025         {
02026 #ifdef CP_HAS_POLL
02027             if (ctl->ufds[i].revents & (POLLIN | POLLPRI))
02028 #else
02029             if (FD_ISSET(desc->owner->socket->fd, &ctl->ufds_r))
02030 #endif /* CP_HAS_POLL */
02031             {
02032                 cp_http_transfer_descriptor_do_read(desc);
02033                 if (desc->res)
02034                     cp_http_transfer_descriptor_do_redirect(desc);
02035             }
02036         }
02037 
02038         /* no need to poll()/select() again if already done */
02039         if (desc->stage == TRANSFER_STAGE_DONE)
02040         {
02041             cp_hashlist_remove(ctl->stack, desc);
02042             cp_http_transfer_descriptor_callback(desc);
02043             cp_http_transfer_descriptor_destroy(desc);
02044         }
02045     }
02046 }
02047 
02048 static void *async_bg_thread_run(void *prm)
02049 {
02050     int err;
02051     int rcount;
02052     cp_httpclient_ctl *ctl = prm;
02053 #ifndef CP_HAS_POLL
02054     struct timeval delay;
02055     delay.tv_sec = 0;
02056     delay.tv_usec = 50;
02057 #endif /* ndef CP_HAS_POLL */
02058 
02059     while (ctl->running)
02060     {
02061         cp_mutex_lock(&ctl->bg_lock);
02062         while ((rcount = cp_hashlist_item_count(ctl->stack) == 0) &&
02063                 ctl->running)
02064             cp_cond_wait(&ctl->bg_cond, &ctl->bg_lock);
02065         cp_mutex_unlock(&ctl->bg_lock);
02066 
02067         if (ctl->running == 0) break; /* shutdown */
02068             
02069         if (set_async_stack_size(ctl, rcount))
02070         {
02071             cp_error(CP_MEMORY_ALLOCATION_FAILURE, "background fetch failed");
02072             break;
02073         }
02074 
02075         prepare_async_stack_fds(ctl);
02076         err = 0;
02077         do
02078         {
02079 #ifdef CP_HAS_POLL
02080             if (poll(ctl->ufds, ctl->nfds, 0) == -1)
02081                 err = errno;
02082 #else
02083             if (select(ctl->fdmax + 1, &ctl->ufds_r, &ctl->ufds_w, NULL, 
02084                         &delay) == -1)
02085                 err = errno;
02086 #endif /* CP_HAS_POLL */
02087         } while (err == EINTR);
02088         scan_async_stack(ctl);
02089     }
02090 
02091     return NULL;
02092 }
02093 
02094 int cp_httpclient_fetch_nb_exec()
02095 {
02096     return cp_httpclient_fetch_ctl_exec(async_stack);
02097 }
02098 
02099 int cp_httpclient_fetch_ctl_exec(cp_httpclient_ctl *ctl)
02100 {
02101     int rcount;
02102     int err;
02103 #ifndef CP_HAS_POLL
02104     struct timeval delay;
02105     delay.tv_sec = 0;
02106     delay.tv_usec = 50;
02107 #endif /* ndef CP_HAS_POLL */
02108     
02109     rcount = cp_hashlist_item_count(ctl->stack);
02110     if (rcount == 0) return 0;
02111     
02112     if (set_async_stack_size(ctl, rcount))
02113     {
02114         cp_error(CP_MEMORY_ALLOCATION_FAILURE, "asynchronous fetch failed");
02115         return -1;
02116     }
02117 
02118     prepare_async_stack_fds(ctl);
02119     
02120     err = 0;
02121     do
02122     {
02123 #ifdef CP_HAS_POLL
02124         if (poll(ctl->ufds, ctl->nfds, 0) == -1)
02125             err = errno;
02126 #else
02127         if (select(ctl->fdmax + 1, &ctl->ufds_r, &ctl->ufds_w, NULL, 
02128                     &delay) == -1)
02129             err = errno;
02130 #endif /* CP_HAS_POLL */
02131     } while (err == EINTR);
02132     scan_async_stack(ctl);
02133 
02134     return cp_hashlist_item_count(ctl->stack);
02135 }
02136 
02137 cp_httpclient_ctl *cp_httpclient_ctl_create(int background)
02138 {
02139     cp_httpclient_ctl *ctl = calloc(1, sizeof(cp_httpclient_ctl));
02140     if (ctl == NULL) goto CREATE_ERROR;
02141     ctl->stack = cp_hashlist_create(1, cp_hash_addr, cp_hash_compare_addr);
02142     if (ctl->stack == NULL) goto CREATE_ERROR;
02143     ctl->size = 10;
02144 #ifdef CP_HAS_POLL
02145     ctl->ufds = malloc(10 * sizeof(struct pollfd));
02146     if (ctl->ufds == NULL) goto CREATE_ERROR;
02147 // #else
02148 //  FD_ZERO(&ctl->ufds_r);
02149 //  FD_ZERO(&ctl->ufds_w);
02150 #endif /* CP_HAS_POLL */
02151     ctl->desc = malloc(10 * sizeof(cp_http_transfer_descriptor *));
02152     if (ctl->desc == NULL) goto CREATE_ERROR;
02153 
02154     if (background)
02155     {
02156         int rc;
02157         if ((rc = cp_mutex_init(&ctl->bg_lock, NULL)))
02158             goto CREATE_ERROR;
02159         cp_cond_init(&ctl->bg_cond, NULL);
02160         ctl->running = 1;
02161         if (background > 1)
02162         {
02163             ctl->pool = cp_thread_pool_create(1, background);
02164             if (ctl->pool == NULL) goto CREATE_ERROR;
02165         }
02166         if (cp_thread_create(ctl->bg, NULL, async_bg_thread_run, ctl) != 0)
02167             goto CREATE_ERROR;
02168     }
02169     
02170     return ctl;
02171 
02172 CREATE_ERROR:
02173     cp_httpclient_ctl_destroy(ctl);
02174     return NULL;
02175 }
02176 
02177 void cp_httpclient_ctl_destroy(cp_httpclient_ctl *ctl)
02178 {
02179     if (ctl)
02180     {
02181         if (ctl->running)
02182         {
02183             cp_mutex_lock(&ctl->bg_lock);
02184             ctl->running = 0;
02185             cp_cond_signal(&ctl->bg_cond);
02186             cp_mutex_unlock(&ctl->bg_lock);
02187 
02188             if (ctl->pool)
02189             {
02190                 cp_thread_pool_stop(ctl->pool);
02191                 cp_thread_pool_destroy(ctl->pool);
02192             }
02193             
02194             cp_thread_join(ctl->bg, NULL);
02195             cp_mutex_destroy(&ctl->bg_lock);
02196             cp_cond_destroy(&ctl->bg_cond);
02197         }
02198             
02199         cp_hashlist_destroy_custom(ctl->stack, NULL, 
02200                        (cp_destructor_fn) cp_http_transfer_descriptor_destroy);
02201 #ifdef CP_HAS_POLL
02202         if (ctl->ufds) free(ctl->ufds);
02203 #endif
02204         if (ctl->desc) free(ctl->desc);
02205         free(ctl);
02206     }
02207 }
02208 
02209 #ifdef CP_HAS_POLL
02210 struct pollfd *cp_httpclient_ctl_default_get_pollfds(int *num)
02211 {
02212     return cp_httpclient_ctl_get_pollfds(async_stack, num);
02213 }
02214 
02215 struct pollfd *cp_httpclient_ctl_get_pollfds(cp_httpclient_ctl *ctl, int *num)
02216 {
02217     prepare_async_stack_fds(ctl);
02218     if (num) *num = cp_hashlist_item_count(ctl->stack);
02219     return ctl->ufds;
02220 }
02221 #else
02222 fd_set *cp_httpclient_ctl_default_get_read_fd_set(int *num)
02223 {
02224     return cp_httpclient_ctl_get_read_fd_set(async_stack, num);
02225 }
02226 
02227 fd_set *cp_httpclient_ctl_get_read_fd_set(cp_httpclient_ctl *ctl, int *num)
02228 {
02229     prepare_async_stack_fds(ctl);
02230     if (num) *num = ctl->fdmax;
02231     return &ctl->ufds_r;
02232 }
02233 
02234 fd_set *cp_httpclient_ctl_default_get_write_fd_set(int *num)
02235 {
02236     return cp_httpclient_ctl_get_write_fd_set(async_stack, num);
02237 }
02238 
02239 fd_set *cp_httpclient_ctl_get_write_fd_set(cp_httpclient_ctl *ctl, int *num)
02240 {
02241     prepare_async_stack_fds(ctl);
02242     if (num) *num = ctl->fdmax;
02243     return &ctl->ufds_w;
02244 }
02245 #endif /* CP_HAS_POLL */
02246 
02247 
02248 static int init_async_bg()
02249 {
02250     async_bg_stack = cp_httpclient_ctl_create(ASYNC_BG_THREAD_MAX);
02251     if (async_bg_stack == NULL) goto INIT_ERROR;
02252     return 0;
02253 
02254 INIT_ERROR:
02255     cp_error(CP_THREAD_CREATION_FAILURE, 
02256             "can\'t start background transfer handler");
02257     stop_async_bg();
02258     return -1;
02259 }
02260 
02261 static void stop_async_bg()
02262 {
02263     if (async_bg_stack)
02264         cp_httpclient_ctl_destroy(async_bg_stack);
02265 }
02266 
02267 
02268 cp_http_transfer_descriptor *
02269     cp_http_transfer_descriptor_create(void *id, 
02270                                        cp_httpclient *client, 
02271                                        cp_httpclient_callback callback,
02272                                        char *uri)
02273 {
02274     cp_http_transfer_descriptor *desc = 
02275         calloc(1, sizeof(cp_http_transfer_descriptor));
02276     if (desc == NULL) return NULL;
02277 
02278     desc->id = id;
02279     desc->owner = client;
02280     desc->callback = callback;
02281     desc->stage = TRANSFER_STAGE_CONNECT;
02282     desc->uri = strdup(uri);
02283 
02284     return desc;
02285 }
02286 
02287 void cp_http_transfer_descriptor_destroy(cp_http_transfer_descriptor *desc)
02288 {
02289     if (desc)
02290     {
02291         if (desc->buf) cp_string_destroy(desc->buf);
02292         if (desc->owner->cgi_parameters && desc->owner->auto_drop_parameters)
02293             cp_httpclient_drop_parameters(desc->owner);
02294         if (desc->owner->header && desc->owner->auto_drop_headers)
02295             cp_httpclient_drop_headers(desc->owner);
02296         if (desc->uri) free(desc->uri);
02297         free(desc);
02298     }
02299 }
02300 
02301 
02302 int cp_httpclient_fetch_nb(cp_httpclient *client, char *uri, void *id, 
02303                            cp_httpclient_callback callback, int background)
02304 {
02305     if (background && async_bg_stack == NULL)
02306         if (init_async_bg()) return -1;
02307 
02308     return cp_httpclient_fetch_ctl(background ? async_bg_stack : async_stack, 
02309             client, uri, id, callback);
02310 }
02311 
02312 int cp_httpclient_fetch_ctl(cp_httpclient_ctl *ctl, cp_httpclient *client, 
02313                             char *uri, void *id, 
02314                             cp_httpclient_callback callback)
02315 {
02316     cp_http_transfer_descriptor *desc = 
02317         cp_http_transfer_descriptor_create(id, client, callback, uri);
02318     if (desc == NULL) return -1;
02319 
02320     desc->buf = cp_httpclient_format_request(client, uri);
02321     desc->remaining = desc->buf->len;
02322 
02323     if (ctl->running) /* background transfer */
02324     {
02325         cp_hashlist_append(ctl->stack, desc, desc);
02326         cp_mutex_lock(&ctl->bg_lock);
02327         cp_cond_broadcast(&ctl->bg_cond);
02328         cp_mutex_unlock(&ctl->bg_lock);
02329     }
02330     else
02331         cp_hashlist_append(ctl->stack, desc, desc);
02332         
02333     return 0;
02334 }
02335 

Generated on Mon Dec 5 23:00:22 2011 for cprops by  doxygen 1.4.7