00001 #include "config.h"
00002
00003 #ifdef _WINDOWS
00004 #include <Winsock2.h>
00005 #endif
00006
00007 #ifdef CP_HAS_STRNDUP
00008 #ifndef _GNU_SOURCE
00009 #define _GNU_SOURCE
00010 #endif
00011 #endif
00012 #include <string.h>
00013
00014 #include <stdio.h>
00015 #include <stdlib.h>
00016 #include <errno.h>
00017
00018
00019 #ifdef CP_HAS_STRINGS_H
00020 #include <strings.h>
00021 #endif
00022
00023 #if defined(unix) || defined(__unix__) || defined(__MACH__)
00024 #include <unistd.h>
00025 #endif
00026
00027 #ifdef CP_HAS_REGEX_H
00028 #include <regex.h>
00029 #else
00030 #include <pcreposix.h>
00031 #endif
00032
00033
00034 #ifdef CP_HAS_POLL
00035 #include <sys/poll.h>
00036 #else
00037 #if defined(sun) || defined(__OpenBSD__) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__APPLE__)
00038 #include <sys/types.h>
00039 #include <sys/socket.h>
00040 #endif
00041 #ifdef CP_HAS_SYS_SELECT_H
00042 #include <sys/select.h>
00043 #endif
00044 #endif
00045
00046 #include "httpclient.h"
00047 #include "hashtable.h"
00048 #include "hashlist.h"
00049 #include "thread.h"
00050 #include "util.h"
00051
00052 #define RESPONSE_INIT 0
00053 #define RESPONSE_STATUS_LINE 1
00054 #define RESPONSE_HEADERS 2
00055 #define RESPONSE_BODY 3
00056 #define RESPONSE_DONE 4
00057
00058 static char *re_status_line_lit =
00059 "^HTTP/([^[:space:]]*)[[:space:]]+([0-9]{3,})[[:space:]]+(.*)$";
00060 static char *re_response_header_lit =
00061 "^([[:alnum:][:punct:]]+):[[:space:]]*(.*[^[:space:]])[[:space:]]*$";
00062 static char *re_url_lit =
00063 "^(http(s)?:\\/\\/)?([^:\\/]+)(\\:[0-9]{1,5})?(\\/.*)?$";
00064 #ifdef CP_USE_COOKIES
00065 static char *re_cookie_lit = "(;[[:space:]]*(([^=;\n]+)(=([^=;\n]*))?))";
00066 #endif
00067
00068 static regex_t re_status_line;
00069 static regex_t re_response_header;
00070 static regex_t re_url;
00071 #ifdef CP_USE_COOKIES
00072 static regex_t re_cookie;
00073
00074 static cp_trie *master_cookie_jar = NULL;
00075 #endif
00076
00077 static cp_httpclient_result *
00078 cp_httpclient_result_new(cp_http_transfer_descriptor *desc,
00079 cp_http_result_status status,
00080 cp_http_response *response);
00081
00082 static cp_httpclient_ctl *async_stack = NULL;
00083 static cp_httpclient_ctl *async_bg_stack = NULL;
00084
00085 #define ASYNC_BG_THREAD_MAX 1
00086 static int init_async_bg();
00087 static void stop_async_bg();
00088
00089 volatile short initialized = 0;
00090
00091 int cp_httpclient_init()
00092 {
00093 int rc;
00094
00095 if (initialized) return 1;
00096 initialized = 1;
00097
00098 cp_client_init();
00099
00100 if ((rc = regcomp(&re_status_line,
00101 re_status_line_lit, REG_EXTENDED | REG_NEWLINE)))
00102 {
00103 log_regex_compilation_error(rc, "compiling status line expression");
00104 goto INIT_ERROR;
00105 }
00106
00107 if ((rc = regcomp(&re_response_header,
00108 re_response_header_lit, REG_EXTENDED | REG_NEWLINE)))
00109 {
00110 log_regex_compilation_error(rc, "compiling response header expression");
00111 goto INIT_ERROR;
00112 }
00113
00114 if ((rc = regcomp(&re_url, re_url_lit, REG_EXTENDED | REG_NEWLINE)))
00115 {
00116 log_regex_compilation_error(rc, "compiling url expression");
00117 goto INIT_ERROR;
00118 }
00119
00120 #ifdef CP_USE_COOKIES
00121 if ((rc = regcomp(&re_cookie, re_cookie_lit, REG_EXTENDED | REG_NEWLINE)))
00122 {
00123 log_regex_compilation_error(rc, "compiling cookie expression");
00124 goto INIT_ERROR;
00125 }
00126
00127 master_cookie_jar =
00128 cp_trie_create_trie(COLLECTION_MODE_DEEP, NULL,
00129 (cp_destructor_fn) cp_trie_destroy);
00130 if (master_cookie_jar == NULL)
00131 {
00132 cp_error(CP_MEMORY_ALLOCATION_FAILURE, "can\'t allocate cookie jar");
00133 goto INIT_ERROR;
00134 }
00135 #endif
00136
00137 async_stack = cp_httpclient_ctl_create(0);
00138 if (async_stack == NULL)
00139 {
00140 cp_error(CP_MEMORY_ALLOCATION_FAILURE,
00141 "can\'t instantiate asynchronous transfer stack");
00142 goto INIT_ERROR;
00143 }
00144
00145 return 0;
00146
00147 INIT_ERROR:
00148 initialized = 0;
00149 return -1;
00150 }
00151
00152 int cp_httpclient_shutdown()
00153 {
00154 if (initialized == 0) return 1;
00155 initialized = 0;
00156
00157 if (async_bg_stack)
00158 stop_async_bg();
00159
00160 regfree(&re_status_line);
00161 regfree(&re_response_header);
00162 regfree(&re_url);
00163
00164 #ifdef CP_USE_COOKIES
00165 regfree(&re_cookie);
00166
00167 cp_trie_destroy(master_cookie_jar);
00168 #endif
00169
00170 cp_httpclient_ctl_destroy(async_stack);
00171
00172
00173 #ifdef CP_USE_SSL
00174 cp_socket_shutdown();
00175 #endif
00176 cp_client_shutdown();
00177
00178 return 0;
00179 }
00180
00181 #define HTTP_RE_MATCHMAX 10
00182
00183
00184
00185
00186 #ifdef CP_USE_COOKIES
00187
00188 #ifndef CP_HAS_STRPTIME
00189
00190 void parse_cookie_time(char *str, struct tm *res)
00191 {
00192 char day[8];
00193 sscanf(str, "%s, %2d-%2d-%4d %2d:%2d:%2d GMT", day, &res->tm_mday, &res->tm_mon,
00194 &res->tm_year, &res->tm_hour, &res->tm_min, &res->tm_sec);
00195 }
00196 #endif
00197
00201 cp_http_cookie *cp_http_cookie_create(char *content,
00202 char *domain,
00203 char *path,
00204 char *expires,
00205 int secure,
00206 int version,
00207 int http_only)
00208 {
00209 cp_http_cookie *c = calloc(1, sizeof(cp_http_cookie));
00210 if (c)
00211 {
00212 c->content = content;
00213 c->domain = domain;
00214 c->path = path;
00215 if (expires)
00216 #ifdef CP_HAS_STRPTIME
00217 strptime(expires, "%a, %d-%b-%Y %H:%M:%S GMT", &c->expires);
00218 #else
00219 parse_cookie_time(expires, &c->expires);
00220 #endif
00221 c->secure = secure;
00222 c->version = version;
00223 c->http_only = http_only;
00224 }
00225
00226 return c;
00227 }
00228
00232 cp_http_cookie *cp_http_cookie_parse(char *src)
00233 {
00234 cp_http_cookie *ck;
00235 char *content;
00236 char *ptr = strchr(src, ';');
00237
00238 if (ptr == NULL)
00239 ck = cp_http_cookie_create(strdup(src), NULL, NULL, NULL, 0, 0, 0);
00240 else
00241 {
00242 regmatch_t rm[HTTP_RE_MATCHMAX];
00243 char *fld;
00244 char *domain = NULL;
00245 char *expires = NULL;
00246 char *path = NULL;
00247 int secure = 0;
00248 int version = 0;
00249 int http_only = 0;
00250 cp_hashtable *unknown = NULL;
00251
00252 *ptr = '\0';
00253 content = strdup(src);
00254 *ptr = ';';
00255
00256 while (regexec(&re_cookie, ptr, HTTP_RE_MATCHMAX, rm, 0) == 0)
00257 {
00258 fld = strndup(&ptr[rm[3].rm_so], rm[3].rm_eo - rm[3].rm_so);
00259
00260 if (strcasecmp(fld, "domain") == 0 && rm[5].rm_so > -1)
00261 domain = strndup(&ptr[rm[5].rm_so], rm[5].rm_eo - rm[5].rm_so);
00262 else if (strcasecmp(fld, "expires") == 0 && rm[5].rm_so > -1)
00263 expires = strndup(&ptr[rm[5].rm_so], rm[5].rm_eo - rm[5].rm_so);
00264 else if (strcasecmp(fld, "path") == 0 && rm[5].rm_so > -1)
00265 path = strndup(&ptr[rm[5].rm_so], rm[5].rm_eo - rm[5].rm_so);
00266 else if (strcasecmp(fld, "secure") == 0)
00267 secure = 1;
00268 else if (strcasecmp(fld, "httpOnly") == 0)
00269 http_only = 1;
00270 else if (strcasecmp(fld, "version") == 0 && rm[5].rm_so > -1)
00271 version = atoi(&ptr[rm[5].rm_so]);
00272 else
00273 {
00274 char *value;
00275 if (unknown == NULL)
00276 unknown =
00277 cp_hashtable_create_by_option(COLLECTION_MODE_DEEP |
00278 COLLECTION_MODE_NOSYNC,
00279 1,
00280 cp_hash_string,
00281 cp_hash_compare_string,
00282 NULL, free, NULL, free);
00283 if (rm[5].rm_so > -1)
00284 value = strndup(&ptr[rm[5].rm_so],
00285 rm[5].rm_eo - rm[5].rm_so);
00286 else
00287 value = strdup("1");
00288
00289 cp_hashtable_put(unknown, fld, value);
00290 fld = NULL;
00291 }
00292 if (fld) free(fld);
00293 ptr = &ptr[rm[0].rm_eo];
00294 }
00295
00296 ck = cp_http_cookie_create(content, domain, path, expires,
00297 secure, version, http_only);
00298 if (expires) free(expires);
00299
00300 ck->fld = unknown;
00301 }
00302
00303 return ck;
00304 }
00305
00307 void cp_http_cookie_destroy(cp_http_cookie *c)
00308 {
00309 if (c)
00310 {
00311 if (c->content) free(c->content);
00312 if (c->domain) free(c->domain);
00313 if (c->path) free(c->path);
00314 if (c->fld) cp_hashtable_destroy(c->fld);
00315 free(c);
00316 }
00317 }
00318
00320 void cp_http_cookie_dump(cp_http_cookie *c)
00321 {
00322 char dbuf[0x100];
00323
00324 cp_info("Cookie\n======\n");
00325 cp_info("content: %s\n", c->content);
00326 cp_info("domain: %s\n", c->domain ? c->domain : "");
00327 cp_info("path: %s\n", c->path ? c->path : "");
00328 if (c->expires.tm_year)
00329 {
00330 strftime(dbuf, 0x100, "%a, %d-%b-%Y %H:%M:%S GMT", &c->expires);
00331 cp_info("expires: %s\n", dbuf);
00332 }
00333 cp_info("secure: %s\n", c->secure ? "yes" : "no");
00334 if (c->version) cp_info("version: %d\n", c->version);
00335 if (c->http_only) cp_info("http only: %s\n", c->http_only ? "yes" : "no");
00336 if (c->fld)
00337 {
00338 int i;
00339 char **key;
00340 char *value;
00341 key = (char **) cp_hashtable_get_keys(c->fld);
00342 for (i = 0; i < cp_hashtable_count(c->fld); i++)
00343 {
00344 value = cp_hashtable_get(c->fld, key[i]);
00345 cp_info("%s: %s\n", key[i], value);
00346 }
00347 free(key);
00348 }
00349 }
00350
00351 void cp_httpclient_set_cookie_jar(cp_httpclient *client, cp_trie *jar)
00352 {
00353 client->cookie_jar = jar;
00354 }
00355
00356 #endif
00357
00358 cp_httpclient *cp_httpclient_allocate(char *host, int port)
00359 {
00360 cp_httpclient *client = calloc(1, sizeof(cp_httpclient));
00361 if (client == NULL) goto ALLOC_ERROR;
00362
00363 client->header =
00364 cp_hashtable_create_by_option(COLLECTION_MODE_COPY |
00365 COLLECTION_MODE_DEEP |
00366 COLLECTION_MODE_NOSYNC,
00367 5,
00368 cp_hash_istring,
00369 cp_hash_compare_istring,
00370 (cp_copy_fn) strdup, free,
00371 (cp_copy_fn) strdup, free);
00372 if (client->header == NULL) goto ALLOC_ERROR;
00373
00374 client->host = host ? strdup(host) : NULL;
00375 client->port = port;
00376 client->version = HTTP_1_1;
00377 client->type = GET;
00378 client->pipeline_requests = 0;
00379 client->follow_redirects = 1;
00380 client->auto_drop_parameters = 1;
00381 client->auto_drop_headers = 0;
00382 client->max_redirects = DEFAULT_MAX_REDIRECTS;
00383 #ifdef CP_USE_COOKIES
00384 client->cookie_jar = master_cookie_jar;
00385 #endif
00386
00387 return client;
00388
00389 ALLOC_ERROR:
00390 cp_httpclient_destroy(client);
00391 return NULL;
00392 }
00393
00394 cp_httpclient *cp_httpclient_create(char *host, int port)
00395 {
00396 cp_httpclient *client = cp_httpclient_allocate(host, port);
00397 if (client == NULL) goto CREATE_ERROR;
00398
00399 client->socket = cp_client_create(host, port);
00400 if (client->socket == NULL) goto CREATE_ERROR;
00401
00402 return client;
00403
00404 CREATE_ERROR:
00405 cp_httpclient_destroy(client);
00406 return NULL;
00407 }
00408
00409 cp_httpclient *
00410 cp_httpclient_create_proxy(char *host, int port,
00411 char *proxy_host, int proxy_port)
00412 {
00413 cp_httpclient *client = cp_httpclient_allocate(host, port);
00414 if (client == NULL) goto CREATE_ERROR;
00415
00416 client->proxy_host = proxy_host ? strdup(proxy_host) : NULL;
00417 if (client->proxy_host == NULL) goto CREATE_ERROR;
00418 client->proxy_port = proxy_port;
00419
00420 client->socket = cp_client_create(proxy_host, proxy_port);
00421 if (client->socket == NULL) goto CREATE_ERROR;
00422
00423 return client;
00424
00425 CREATE_ERROR:
00426 cp_httpclient_destroy(client);
00427 return NULL;
00428 }
00429
00430 #ifdef CP_USE_SSL
00431 cp_httpclient *cp_httpclient_create_ssl(char *host, int port,
00432 char *CA_file, char *CA_path,
00433 int verification_mode)
00434 {
00435 cp_httpclient *client = cp_httpclient_allocate(host, port);
00436 if (client == NULL) goto CREATE_ERROR;
00437
00438 client->socket =
00439 cp_client_create_ssl(host, port, CA_file, CA_path, verification_mode);
00440 if (client->socket == NULL) goto CREATE_ERROR;
00441
00442 return client;
00443
00444 CREATE_ERROR:
00445 cp_httpclient_destroy(client);
00446 return NULL;
00447 }
00448
00449 cp_httpclient *
00450 cp_httpclient_create_proxy_ssl(char *host, int port,
00451 char *proxy_host, int proxy_port,
00452 char *CA_file, char *CA_path,
00453 int verification_mode)
00454 {
00455 cp_httpclient *client = cp_httpclient_allocate(host, port);
00456 if (client == NULL) goto CREATE_ERROR;
00457
00458 if (client)
00459 {
00460 client->proxy_host = proxy_host ? strdup(proxy_host) : NULL;
00461 client->proxy_port = proxy_port;
00462 }
00463
00464 client->socket =
00465 cp_client_create_ssl(proxy_host, proxy_port,
00466 CA_file, CA_path, verification_mode);
00467
00468 client->proxy_ssl = 1;
00469
00470 return client;
00471
00472 CREATE_ERROR:
00473 cp_httpclient_destroy(client);
00474 return NULL;
00475 }
00476 #endif
00477
00478 void cp_httpclient_destroy(cp_httpclient *client)
00479 {
00480 if (client)
00481 {
00482 if (client->connected)
00483 cp_client_close(client->socket);
00484 if (client->socket)
00485 cp_client_destroy(client->socket);
00486 if (client->header)
00487 cp_hashtable_destroy(client->header);
00488 if (client->cgi_parameters)
00489 cp_hashtable_destroy(client->cgi_parameters);
00490 if (client->host) free(client->host);
00491 if (client->proxy_host) free(client->proxy_host);
00492 free(client);
00493 }
00494 }
00495
00496 void cp_httpclient_set_http_version(cp_httpclient *client,
00497 cp_http_version version)
00498 {
00499 client->version = version;
00500 }
00501
00502 void cp_httpclient_set_request_type(cp_httpclient *client,
00503 cp_http_request_type type)
00504 {
00505 client->type = type;
00506 }
00507
00508 void cp_httpclient_set_header(cp_httpclient *client, char *header, char *value)
00509 {
00510 if (value != NULL)
00511 cp_hashtable_put(client->header, header, value);
00512 else
00513 cp_hashtable_remove(client->header, header);
00514 }
00515
00516 void cp_httpclient_drop_headers(cp_httpclient *client)
00517 {
00518 if (client->header) cp_hashtable_remove_all(client->header);
00519 }
00520
00521 void cp_httpclient_set_auto_drop_headers(cp_httpclient *client, short mode)
00522 {
00523 client->auto_drop_headers = mode;
00524 }
00525
00526 void cp_httpclient_set_user_agent(cp_httpclient *client, char *agent)
00527 {
00528 cp_hashtable_put(client->header, "User-Agent", agent);
00529 }
00530
00531 void cp_httpclient_set_timeout(cp_httpclient *client,
00532 int sec, int usec)
00533 {
00534 cp_client_set_timeout(client->socket, sec, usec);
00535 }
00536
00537 void cp_httpclient_set_retry(cp_httpclient *client, int retry_count)
00538 {
00539 cp_client_set_retry(client->socket, retry_count);
00540 }
00541
00542 void cp_httpclient_allow_redirects(cp_httpclient *client, int mode)
00543 {
00544 client->follow_redirects = mode;
00545 }
00546
00547 void cp_httpclient_set_max_redirects(cp_httpclient *client, int max)
00548 {
00549 client->max_redirects = max;
00550 }
00551
00552 int cp_httpclient_connect(cp_httpclient *client)
00553 {
00554 return cp_client_connect(client->socket);
00555 }
00556
00557 #define LINEBUFSIZE 0x1000
00558
00559 static cp_string *cgi_param_string(cp_httpclient *client)
00560 {
00561 cp_string *str = NULL;
00562
00563 if (client->cgi_parameters)
00564 {
00565 int len = cp_hashtable_count(client->cgi_parameters);
00566 if (len)
00567 {
00568 int i;
00569 char *value;
00570 char **keys = (char **) cp_hashtable_get_keys(client->cgi_parameters);
00571 str = cp_string_create("", 0);
00572 for (i = 0; i < len; i++)
00573 {
00574 if (i > 0) cp_string_append_char(str, '&');
00575 value = cp_hashtable_get(client->cgi_parameters, keys[i]);
00576 cp_string_cstrcat(str, keys[i]);
00577 cp_string_append_char(str, '=');
00578 cp_string_cstrcat(str, value);
00579 }
00580 free(keys);
00581 }
00582 }
00583
00584 return str;
00585 }
00586
00587 void *cp_httpclient_set_parameter(cp_httpclient *client,
00588 char *name, char *value)
00589 {
00590 if (value == NULL)
00591 {
00592 if (client->cgi_parameters)
00593 return cp_hashtable_remove(client->cgi_parameters, name);
00594 return NULL;
00595 }
00596
00597 if (client->cgi_parameters == NULL)
00598 {
00599 client->cgi_parameters =
00600 cp_hashtable_create_by_option(COLLECTION_MODE_DEEP |
00601 COLLECTION_MODE_COPY |
00602 COLLECTION_MODE_NOSYNC,
00603 1,
00604 cp_hash_string,
00605 cp_hash_compare_string,
00606 (cp_copy_fn) strdup, free,
00607 (cp_copy_fn) strdup, free);
00608 if (client->cgi_parameters == NULL) return NULL;
00609 }
00610
00611 return cp_hashtable_put(client->cgi_parameters, name, value);
00612 }
00613
00614 void cp_httpclient_drop_parameters(cp_httpclient *client)
00615 {
00616 if (client->cgi_parameters) cp_hashtable_remove_all(client->cgi_parameters);
00617 }
00618
00619 void cp_httpclient_set_auto_drop_parameters(cp_httpclient *client, short mode)
00620 {
00621 client->auto_drop_parameters = mode;
00622 }
00623
00624 static void append_header(cp_string *buf, char *key, char *value)
00625 {
00626 cp_string_cstrcat(buf, key);
00627 cp_string_cstrcat(buf, ": ");
00628 cp_string_cstrcat(buf, value);
00629 cp_string_cstrcat(buf, "\r\n");
00630 }
00631
00632 static void append_host_header(cp_httpclient *client, cp_string *request)
00633 {
00634 if (client->socket->host)
00635 {
00636 int need_port = 0;
00637 #if CP_USE_SSL
00638 if (client->socket->use_ssl && client->port != 443)
00639 need_port = 1;
00640 else
00641 #endif
00642 if (client->port != 80)
00643 need_port = 1;
00644
00645 if (need_port)
00646 {
00647 char *host;
00648 size_t len = strlen(client->host) + 7;
00649 host = (char *) malloc(len);
00650 if (host == NULL) return;
00651 #ifdef CP_HAS_SNPRINTF
00652 snprintf(host, len, "%s:%d", client->host, client->port);
00653 #else
00654 sprintf(host, "%s:%d", client->host, client->port);
00655 #endif
00656 append_header(request, "Host", host);
00657 free(host);
00658 }
00659 else
00660 append_header(request, "Host", client->host);
00661 }
00662 }
00663
00664 cp_string *cp_httpclient_format_request(cp_httpclient *client, char *uri)
00665 {
00666 char buf[LINEBUFSIZE];
00667 cp_string *request;
00668 int len;
00669
00670
00671 if (client->proxy_host == NULL)
00672 #ifdef CP_HAS_SNPRINTF
00673 snprintf(buf, LINEBUFSIZE, "%s %s",
00674 get_http_request_type_lit(client->type), uri);
00675 #else
00676 sprintf(buf, "%s %s", get_http_request_type_lit(client->type), uri);
00677 #endif
00678 else
00679 {
00680 char *method = "http";
00681 char port[8];
00682 #ifdef CP_USE_SSL
00683 if (client->socket->use_ssl)
00684 {
00685 method = "https";
00686 if (client->port == 443)
00687 port[0] = '\0';
00688 else
00689 #ifdef CP_HAS_SNPRINTF
00690 snprintf(port, 8, ":%d", client->proxy_port);
00691 #else
00692 sprintf(port, ":%d", client->proxy_port);
00693 #endif
00694 }
00695 else
00696 #endif
00697 {
00698 if (client->port == 80)
00699 port[0] = '\0';
00700 else
00701 #ifdef CP_HAS_SNPRINTF
00702 snprintf(port, 8, ":%d", client->proxy_port);
00703 #else
00704 sprintf(port, ":%d", client->proxy_port);
00705 #endif
00706 }
00707
00708 #ifdef CP_HAS_SNPRINTF
00709 snprintf(buf, LINEBUFSIZE,
00710 "%s %s://%s%s%s", get_http_request_type_lit(client->type),
00711 method, client->host, port, uri);
00712 #else
00713 sprintf(buf, "%s %s://%s%s%s", get_http_request_type_lit(client->type),
00714 method, client->host, port, uri);
00715 #endif
00716 }
00717
00718 request = cp_string_create(buf, strlen(buf));
00719
00720 if (client->type == GET)
00721 {
00722 cp_string *prm = cgi_param_string(client);
00723 if (prm)
00724 {
00725 cp_string_append_char(request, '?');
00726 cp_string_cat(request, prm);
00727 cp_string_destroy(prm);
00728 }
00729 }
00730
00731 cp_string_cstrcat(request, " HTTP/");
00732 cp_string_cstrcat(request, client->version == HTTP_1_1 ? "1.1" : "1.0");
00733 cp_string_cstrcat(request, "\r\n");
00734
00735
00736 append_host_header(client, request);
00737 if (cp_hashtable_get(client->header, "User-Agent") == NULL)
00738 append_header(request, "User-Agent", DEFAULT_SERVER_NAME);
00739
00740 if ((len = cp_hashtable_count(client->header)))
00741 {
00742 int i;
00743 char **key = (char **) cp_hashtable_get_keys(client->header);
00744 char *value;
00745
00746 for (i = 0; i < len; i++)
00747 {
00748 value = cp_hashtable_get(client->header, key[i]);
00749 if (value) append_header(request, key[i], value);
00750 }
00751 }
00752
00753
00754 if (cp_hashtable_get(client->header, "Connection") == NULL)
00755 {
00756 if (client->type == HTTP_1_1)
00757 {
00758 append_header(request, "Connection", "keep-alive");
00759 append_header(request, "Keep-Alive", "300");
00760 }
00761 else
00762 append_header(request, "Connection", "close");
00763 }
00764
00765
00766 #ifdef CP_USE_COOKIES
00767 {
00768 cp_string *cookie_lit = NULL;
00769 char *domain = reverse_string(client->socket->host);
00770 cp_vector *urilist = cp_trie_fetch_matches(client->cookie_jar, domain);
00771 if (urilist)
00772 {
00773 int i, j;
00774 int len, jlen;
00775 cp_trie *container;
00776 cp_vector *cooklist;
00777 cp_http_cookie *ck;
00778
00779 len = cp_vector_size(urilist);
00780 cookie_lit = cp_string_create_empty(80);
00781 for (i = 0; i < len; i++)
00782 {
00783 container = cp_vector_element_at(urilist, i);
00784 cooklist = cp_trie_fetch_matches(client->cookie_jar, uri);
00785 if (cooklist == NULL) continue;
00786 jlen = cp_vector_size(cooklist);
00787 for (j = 0; j < jlen; j++)
00788 {
00789 ck = cp_vector_element_at(cooklist, j);
00790 if (cookie_lit->len > 0)
00791 cp_string_cstrcat(cookie_lit, "; ");
00792 cp_string_cstrcat(cookie_lit, ck->content);
00793 }
00794 if (cookie_lit->len > 0)
00795 append_header(request, "Cookie", cookie_lit->data);
00796 }
00797 cp_string_destroy(cookie_lit);
00798 cp_vector_destroy(urilist);
00799 }
00800 free(domain);
00801 }
00802 #endif
00803
00804
00805 if (client->type == POST)
00806 {
00807 cp_string *prm = cgi_param_string(client);
00808 if (prm)
00809 {
00810 char len[16];
00811 #ifdef CP_HAS_SNPRINTF
00812 snprintf(len, 16, "%d", prm->len);
00813 #else
00814 sprintf(len, "%d", prm->len);
00815 #endif
00816 append_header(request,
00817 "Content-Type", "application/x-www-form-urlencoded");
00818 append_header(request, "Content-Length", len);
00819 cp_string_cstrcat(request, "\r\n\r\n");
00820 cp_string_cat(request, prm);
00821 cp_string_destroy(prm);
00822 }
00823 }
00824
00825 if (client->content)
00826 {
00827 char len[16];
00828 #ifdef CP_HAS_SNPRINTF
00829 snprintf(len, 16, "%d", client->content->len);
00830 #else
00831 sprintf(len, "%d", client->content->len);
00832 #endif
00833 append_header(request, "Content-Length", len);
00834 cp_string_cstrcat(request, "\r\n\r\n");
00835 cp_string_cat(request, client->content);
00836 }
00837
00838 cp_string_cstrcat(request, "\r\n");
00839 return request;
00840 }
00841
00842 #ifdef CP_USE_SSL
00843
00844
00845 static int do_proxy_ssl_connect(cp_httpclient *client)
00846 {
00847 char cbuf[0x400];
00848 char *pbuf;
00849 int total;
00850 int len;
00851
00852 int rc;
00853 cp_client *sock = client->socket;
00854 sock->use_ssl = 0;
00855
00856 rc = cp_client_connect(sock);
00857 sock->use_ssl = 1;
00858 if (rc) return rc;
00859
00860 if (client->port != 443)
00861 #ifdef CP_HAS_SNPRINTF
00862 snprintf(cbuf, 0x400, "CONNECT %s:%d HTTP/1.0\r\n\r\n",
00863 client->host, client->port);
00864 #else
00865 sprintf(cbuf, "CONNECT %s:%d HTTP/1.0\r\n\r\n",
00866 client->host, client->port);
00867 #endif
00868 else
00869 #ifdef CP_HAS_SNPRINTF
00870 snprintf(cbuf, 0x400, "CONNECT %s HTTP/1.0\r\n\r\n", client->host);
00871 #else
00872 sprintf(cbuf, "CONNECT %s HTTP/1.0\r\n\r\n", client->host);
00873 #endif
00874
00875 len = strlen(cbuf);
00876
00877 for (total = 0; total < len; total+= rc)
00878 {
00879 rc = send(sock->fd, cbuf, len - total, 0);
00880 if (rc <= 0) break;
00881 }
00882 if (rc == 0) return -1;
00883 if (rc < 0) return rc;
00884
00885 pbuf = cbuf;
00886 cbuf[0] = '\0';
00887 len = 0;
00888 do
00889 {
00890 rc = recv(sock->fd, pbuf, 0x400 - len, 0);
00891 if (rc == -1 && (errno == EAGAIN || errno == EINTR)) continue;
00892 if (rc <= 0) break;
00893 pbuf[rc] = '\0';
00894 sock->bytes_read += rc;
00895 #ifdef _HTTP_DUMP
00896 DEBUGMSG("\n---------------------------------------------------------------------------" "\n%s\n" "---------------------------------------------------------------------------", pbuf);
00897 #endif
00898 pbuf = &pbuf[rc];
00899 } while (strstr(cbuf, "\r\n\r\n") == NULL);
00900
00901 if (rc < 0) return rc;
00902 if (strstr(cbuf, "HTTP/1.0 200 Connection established") == NULL)
00903 return -1;
00904
00905 return cp_client_connect_ssl(sock);
00906 }
00907 #endif
00908
00909 static int do_connect(cp_httpclient *client)
00910 {
00911 int rc;
00912
00913 #ifdef CP_USE_SSL
00914 if (client->proxy_ssl)
00915 rc = do_proxy_ssl_connect(client);
00916 else
00917 #endif
00918 rc = cp_client_connect(client->socket);
00919 if (rc)
00920 {
00921 #ifdef CP_USE_SSL
00922 if (rc > 0)
00923 {
00924 cp_error(CP_SSL_VERIFICATION_ERROR, "%s:%d : %s",
00925 client->socket->host, client->socket->port,
00926 ssl_verification_error_str(rc));
00927 }
00928 else
00929 #endif
00930 {
00931 cp_error(CP_IO_ERROR, "can\'t connect to %s on port %d",
00932 client->socket->host, client->socket->port);
00933 return -1;
00934 }
00935 }
00936 client->connected = 1;
00937 return 0;
00938 }
00939
00940 #ifdef __OpenBSD__
00941 #define HTTP_PARSE_BUFSIZE 0x1000
00942 #else
00943 #define HTTP_PARSE_BUFSIZE 0xFFFF
00944 #endif
00945
00946 cp_http_response *cp_http_client_response_create()
00947 {
00948 cp_http_response *res = NULL;
00949
00950 if ((res = (cp_http_response *) calloc(1, sizeof(cp_http_response))))
00951 {
00952 res->header =
00953 cp_hashtable_create_by_option(COLLECTION_MODE_NOSYNC |
00954 COLLECTION_MODE_COPY |
00955 COLLECTION_MODE_DEEP,
00956 10,
00957 cp_hash_istring,
00958 cp_hash_compare_istring,
00959 (cp_copy_fn) strdup, free,
00960 (cp_copy_fn) strdup, free);
00961 if (res->header == NULL)
00962 {
00963 free(res);
00964 return NULL;
00965 }
00966 }
00967
00968 return res;
00969 }
00970
00971 cp_url_descriptor *
00972 cp_url_descriptor_create(char *host, short ssl, int port, char *uri)
00973 {
00974 cp_url_descriptor *desc = calloc(1, sizeof(cp_url_descriptor));
00975 if (desc == NULL) return NULL;
00976
00977 desc->host = host;
00978 desc->uri = uri;
00979
00980 desc->ssl = ssl;
00981 desc->port = port;
00982
00983 return desc;
00984 }
00985
00986 void cp_url_descriptor_destroy(cp_url_descriptor *desc)
00987 {
00988 if (desc)
00989 {
00990 if (desc->host) free(desc->host);
00991 if (desc->uri) free(desc->uri);
00992 free(desc);
00993 }
00994 }
00995
00996 short cp_url_descriptor_ssl(cp_url_descriptor *desc)
00997 {
00998 return desc->ssl;
00999 }
01000
01001 char *cp_url_descriptor_host(cp_url_descriptor *desc)
01002 {
01003 return desc->host;
01004 }
01005
01006 int cp_url_descriptor_port(cp_url_descriptor *desc)
01007 {
01008 return desc->port;
01009 }
01010
01011 char *cp_url_descriptor_uri(cp_url_descriptor *desc)
01012 {
01013 return desc->uri;
01014 }
01015
01016 cp_url_descriptor *cp_url_descriptor_parse(char *url)
01017 {
01018 cp_string *host;
01019 char *chost;
01020 short ssl;
01021 int port;
01022 cp_string *uri;
01023 char *curi;
01024 regmatch_t rm[6];
01025
01026 if (url[0] == '/')
01027 return cp_url_descriptor_create(NULL, 0, 0, strdup(url));
01028
01029 if (regexec(&re_url, url, 6, rm, 0))
01030 {
01031 cp_error(CP_HTTP_INVALID_URL, "can\'t parse url: %s\n", url);
01032 return NULL;
01033 }
01034
01035 ssl = rm[2].rm_so != -1;
01036 host = cp_string_create(&url[rm[3].rm_so], rm[3].rm_eo - rm[3].rm_so);
01037 if (rm[4].rm_so != -1)
01038 port = atoi(&url[rm[4].rm_so + 1]);
01039 else
01040 port = ssl ? 443 : 80;
01041 if (rm[5].rm_so != -1)
01042 uri = cp_string_create(&url[rm[5].rm_so], rm[5].rm_eo - rm[5].rm_so);
01043 else
01044 uri = cp_string_create("/", 1);
01045
01046 chost = cp_string_tocstr(host);
01047 free(host);
01048 curi = cp_string_tocstr(uri);
01049 free(uri);
01050
01051 return cp_url_descriptor_create(chost, ssl, port, curi);
01052 }
01053
01054 int read_status_line(cp_http_response *res, char **ptr, int len)
01055 {
01056 regmatch_t rm[HTTP_RE_MATCHMAX];
01057
01058 if (regexec(&re_status_line, *ptr, HTTP_RE_MATCHMAX, rm, 0))
01059 {
01060 cp_error(CP_HTTP_INVALID_STATUS_LINE, "can\'t parse server response");
01061 return CP_HTTP_INVALID_STATUS_LINE;
01062 }
01063
01064 if (strncmp(&(*ptr)[rm[1].rm_so], "1.1", 3) == 0)
01065 res->version = HTTP_1_1;
01066 else
01067 res->version = HTTP_1_0;
01068
01069 res->status = (cp_http_status_code) atoi(&(*ptr)[rm[2].rm_so]);
01070 if (rm[3].rm_so > -1)
01071 {
01072 cp_string *status_lit =
01073 cp_string_create(&(*ptr)[rm[3].rm_so], rm[3].rm_eo - rm[3].rm_so);
01074 res->status_lit = status_lit->data;
01075 free(status_lit);
01076 }
01077
01078 *ptr = &(*ptr)[rm[0].rm_eo];
01079 return 0;
01080 }
01081
01082 static char *find_headers_end(char *buf)
01083 {
01084 char *end = strstr(buf, "\r\n\r\n");
01085 if (end == NULL)
01086 end = strstr(buf, "\n\n");
01087 if (end == NULL)
01088 end = strstr(buf, "\n\r\n\r");
01089
01090 return end;
01091 }
01092
01093 int read_headers(cp_httpclient *client, cp_http_response *res,
01094 char *uri, char **ptr, int srclen)
01095 {
01096 char *curr = *ptr;
01097 int len;
01098 regmatch_t rm[HTTP_RE_MATCHMAX];
01099 char *name, *value;
01100 char *headers_end;
01101 char ch;
01102
01103 while (*curr == '\r' || *curr == '\n') curr++;
01104
01105 headers_end = find_headers_end(curr);
01106 if (headers_end == NULL) return EAGAIN;
01107 ch = *headers_end;
01108 *headers_end = '\0';
01109
01110 while (regexec(&re_response_header, curr, HTTP_RE_MATCHMAX, rm, 0) == 0)
01111 {
01112 len = rm[1].rm_eo - rm[1].rm_so;
01113 if ((name = (char *) malloc((len + 1) * sizeof(char))) == NULL)
01114 return CP_MEMORY_ALLOCATION_FAILURE;
01115 strncpy(name, curr + rm[1].rm_so, len);
01116 name[len] = '\0';
01117
01118 len = rm[2].rm_eo - rm[2].rm_so;
01119 if ((value = (char *) malloc((len + 1) * sizeof(char))) == NULL)
01120 {
01121 free(name);
01122 return CP_MEMORY_ALLOCATION_FAILURE;
01123 }
01124 strncpy(value, curr + rm[2].rm_so, len);
01125 value[len] = '\0';
01126
01127 #ifdef CP_USE_COOKIES
01128 if (strcasecmp(name, "Set-Cookie") == 0)
01129 {
01130 cp_http_cookie *ck = cp_http_cookie_parse(value);
01131 cp_trie *uri_trie;
01132 if (ck)
01133 {
01134 char *domain = NULL;
01135 if (ck->domain)
01136 domain = reverse_string(ck->domain);
01137 else
01138 domain = reverse_string(client->socket->host);
01139 uri_trie =
01140 cp_trie_exact_match(client->cookie_jar, domain);
01141 if (uri_trie == NULL)
01142 {
01143 uri_trie =
01144 cp_trie_create_trie(COLLECTION_MODE_DEEP, NULL,
01145 (cp_destructor_fn) cp_http_cookie_destroy);
01146 cp_trie_add(client->cookie_jar, domain, uri_trie);
01147 }
01148 free(domain);
01149 if (ck->path)
01150 cp_trie_add(uri_trie, ck->path, ck);
01151 else
01152 cp_trie_add(uri_trie, uri, ck);
01153 }
01154 free(name);
01155 free(value);
01156 }
01157 else
01158 #endif
01159 cp_hashtable_put_by_option(res->header, name, value,
01160 (cp_hashtable_get_mode(res->header) | COLLECTION_MODE_COPY)
01161 ^ COLLECTION_MODE_COPY);
01162 curr = &curr[rm[0].rm_eo];
01163 }
01164 *ptr = curr;
01165
01166 *headers_end = ch;
01167 return 0;
01168 }
01169
01170 static char *find_response_end(char *curr)
01171 {
01172 char *src = strstr(curr, "\r\n\r\n");
01173 if (src)
01174 src += 4;
01175 else
01176 {
01177 src = strstr(curr, "\n\n");
01178 if (src) src += 2;
01179 }
01180
01181 return src;
01182 }
01183
01184
01185 static int read_chunk(cp_httpclient *client, char *buf, int len)
01186 {
01187 int rc;
01188 while (1)
01189 {
01190 #ifdef CP_USE_SSL
01191 if (client->socket->use_ssl)
01192 rc = SSL_read(client->socket->ssl, buf, len);
01193 else
01194 #endif
01195 rc = recv(client->socket->fd, buf, len, 0);
01196 if (rc == -1 && (errno == EAGAIN || errno == EINTR)) continue;
01197 if (rc >= 0)
01198 {
01199 buf[rc] = '\0';
01200 client->socket->bytes_read += rc;
01201 #ifdef _HTTP_DUMP
01202 cp_info("\n---------------------------------------------------------------------------" "\n%s\n" "---------------------------------------------------------------------------", buf);
01203 #endif
01204 }
01205 break;
01206 }
01207
01208 return rc;
01209 }
01210
01211
01212 #define C2HEX(p) ((p) >= 'a' && (p) <= 'f' ? (p) - 'a' : \
01213 (p) >= 'A' && (p) <= 'F' ? (p) - 'A' : (p) - '0')
01214
01215
01216
01217
01218 int xtoid(char *p)
01219 {
01220 int curr;
01221 int res = -1;
01222
01223 while (*p)
01224 {
01225 curr = C2HEX(*p);
01226 if (curr == -1) break;
01227 res = res * 0x10 + curr;
01228 p++;
01229 }
01230
01231 return res;
01232 }
01233
01234 static int read_chunked(cp_httpclient *client,
01235 cp_http_response *res,
01236 char *tmpbuf,
01237 int buflen,
01238 char *curr)
01239 {
01240 int chunk;
01241 int inc, cat;
01242 int len = -1;
01243 res->content = NULL;
01244
01245 while (1)
01246 {
01247 while (*curr == '\r' || *curr == '\n') curr++;
01248 if (*curr == 0)
01249 {
01250 curr = tmpbuf;
01251 buflen = read_chunk(client, tmpbuf, HTTP_PARSE_BUFSIZE);
01252
01253 if (buflen <= 0) break;
01254 }
01255
01256 len = xtoi(curr);
01257 if (len == 0) break;
01258
01259 curr = strchr(curr, '\n');
01260 if (curr == NULL) return -1;
01261 curr++;
01262 inc = buflen - (curr - tmpbuf);
01263 cat = (len < inc) ? len : inc;
01264 if (res->content)
01265 cp_string_cat_bin(res->content, curr, cat);
01266 else
01267 res->content = cp_string_create(curr, cat);
01268 if (len < inc) curr = &curr[cat];
01269 len -= cat;
01270
01271 for (chunk = 0; chunk < len; )
01272 {
01273 curr = tmpbuf;
01274 buflen = read_chunk(client, tmpbuf, HTTP_PARSE_BUFSIZE);
01275
01276 chunk += buflen;
01277 if (chunk <= len)
01278 cp_string_cat_bin(res->content, tmpbuf, buflen);
01279 else
01280 {
01281 cp_string_cat_bin(res->content, tmpbuf, buflen - (chunk - len));
01282 curr = &tmpbuf[buflen - (chunk - len)];
01283 }
01284 }
01285 }
01286
01287 return len;
01288 }
01289
01290 static cp_http_response *read_response(cp_httpclient *client, char *uri)
01291 {
01292 char tmpbuf[HTTP_PARSE_BUFSIZE + 1];
01293 cp_string *buf = cp_string_create_empty(HTTP_PARSE_BUFSIZE);
01294 char *curr;
01295 cp_http_response *res = cp_http_client_response_create();
01296 int stage = 0;
01297 int rc;
01298 int skip = 0;
01299
01300 curr = buf->data;
01301
01302 while (stage < RESPONSE_DONE)
01303 {
01304 rc = read_chunk(client, tmpbuf, HTTP_PARSE_BUFSIZE);
01305
01306 if (rc <= 0) break;
01307
01308 cp_string_cat_bin(buf, tmpbuf, rc);
01309 buf->data[buf->len] = '\0';
01310
01311 if (stage == RESPONSE_INIT)
01312 {
01313 if (read_status_line(res,
01314 &curr, buf->len - (curr - buf->data))) break;
01315 stage = RESPONSE_HEADERS;
01316 }
01317
01318 if (stage == RESPONSE_HEADERS)
01319 {
01320 int rv;
01321 char *prm;
01322
01323 rv = read_headers(client, res, uri, &curr,
01324 buf->len + skip - (curr - buf->data));
01325 if (rv == -1)
01326 break;
01327 else if (rv == EAGAIN)
01328 {
01329 skip += rc;
01330 continue;
01331 }
01332
01333 if ((prm = cp_hashtable_get(res->header, "Transfer-Encoding"))
01334 && strcasecmp(prm, "chunked") == 0)
01335 {
01336 int len;
01337 curr = &tmpbuf[curr - buf->data];
01338 len = read_chunked(client, res, tmpbuf, rc, curr);
01339 if (len == -1) break;
01340 if (len == 0) stage = RESPONSE_DONE;
01341 break;
01342 }
01343 else if ((prm = cp_hashtable_get(res->header, "Content-Length")))
01344 {
01345 int len;
01346 char *src = find_response_end(curr);
01347 if (src == NULL) break;
01348 curr = src;
01349 res->len = atoi(prm);
01350 stage = RESPONSE_BODY;
01351 res->content =
01352 cp_string_create(curr, rc + skip - (curr - buf->data));
01353 len = res->len - res->content->len;
01354
01355 if (len > 0)
01356 cp_client_read_string(client->socket, res->content, len);
01357 stage = RESPONSE_DONE;
01358 break;
01359 }
01360
01361 else if (cp_hashtable_get(res->header, "Content-Type"))
01362 {
01363 char *src = find_response_end(curr);
01364 if (src == NULL) break;
01365 curr = src;
01366 stage = RESPONSE_BODY;
01367 res->content =
01368 cp_string_create(curr, rc + skip - (curr - buf->data));
01369 continue;
01370 }
01371 else
01372 stage = RESPONSE_DONE;
01373 }
01374
01375 if (stage == RESPONSE_BODY)
01376 cp_string_cat_bin(res->content, tmpbuf, rc);
01377 }
01378
01379 cp_string_destroy(buf);
01380
01381 if ((stage == RESPONSE_DONE) || (stage == RESPONSE_BODY && res->len == 0))
01382 return res;
01383
01384 cp_error(CP_IO_ERROR, "error %s",
01385 stage == RESPONSE_INIT ? "initializing response" :
01386 stage == RESPONSE_HEADERS ? "reading headers" : "reading content");
01387
01388 cp_http_response_delete(res);
01389 return NULL;
01390 }
01391
01392 cp_httpclient_result *cp_httpclient_fetch(cp_httpclient *client, char *uri)
01393 {
01394 cp_httpclient_result *res;
01395 cp_string *request = cp_httpclient_format_request(client, uri);
01396 cp_http_response *response;
01397 int redirects = 0;
01398
01399 res = cp_httpclient_result_create(client);
01400 if (res == NULL) return NULL;
01401
01402 if (!client->connected)
01403 if (do_connect(client))
01404 {
01405 res->result = CP_HTTP_RESULT_CONNECTION_FAILED;
01406 return res;
01407 }
01408
01409 if (cp_client_write_string(client->socket, request) == 0)
01410 {
01411 if (do_connect(client))
01412 {
01413 res->result = CP_HTTP_RESULT_CONNECTION_FAILED;
01414 return res;
01415 }
01416 if (cp_client_write_string(client->socket, request) == 0)
01417 {
01418 cp_string_destroy(request);
01419 res->result = CP_HTTP_RESULT_WRITE_ERROR;
01420 return res;
01421 }
01422 }
01423
01424 response = read_response(client, uri);
01425 if (response == NULL)
01426 {
01427 res->result = CP_HTTP_RESULT_READ_ERROR;
01428 return res;
01429 }
01430
01431 while (client->follow_redirects && (response->status == HTTP_302_FOUND ||
01432 response->status == HTTP_301_MOVED_PERMANENTLY ||
01433 response->status == HTTP_300_MULTIPLE_CHOICES) &&
01434 redirects < client->max_redirects)
01435 {
01436 int rc;
01437 int retry = 3;
01438 cp_http_response *redirected;
01439 cp_url_descriptor *udesc;
01440 char *url = cp_hashtable_get(response->header, "Location");
01441
01442 if (url == NULL) break;
01443 udesc = cp_url_descriptor_parse(url);
01444 if (udesc == NULL) break;
01445 cp_string_destroy(request);
01446
01447 if (udesc->host == NULL)
01448 {
01449 udesc->host = strdup(client->host);
01450 udesc->port = client->port;
01451 #ifdef CP_USE_SSL
01452 udesc->ssl = client->socket->use_ssl;
01453 #endif
01454 }
01455 else if (strcmp(udesc->host, client->host) || (udesc->port != client->port))
01456 {
01457 free(client->host);
01458 client->host = strdup(udesc->host);
01459 if (!client->proxy_host)
01460 cp_client_reopen(client->socket, udesc->host, udesc->port);
01461
01462 }
01463
01464 request = cp_httpclient_format_request(client, udesc->uri);
01465 while (cp_client_write_string(client->socket, request) == 0)
01466 {
01467 rc = errno;
01468 if (rc == EINTR || rc == EAGAIN) continue;
01469 if (rc == EPIPE && retry-- > 0)
01470 {
01471 cp_client_reopen(client->socket, udesc->host, udesc->port);
01472 continue;
01473 }
01474 cp_string_destroy(request);
01475 res->result = CP_HTTP_RESULT_WRITE_ERROR;
01476 return res;
01477 }
01478
01479 redirected = read_response(client, udesc->uri);
01480 if (redirected)
01481 {
01482 cp_http_response_delete(response);
01483 response = redirected;
01484 redirects++;
01485 }
01486 cp_url_descriptor_destroy(udesc);
01487 }
01488
01489 cp_string_destroy(request);
01490
01491 if (client->header && client->auto_drop_headers)
01492 cp_httpclient_drop_headers(client);
01493
01494 if (client->cgi_parameters && client->auto_drop_parameters)
01495 cp_httpclient_drop_parameters(client);
01496
01497 if (response == NULL)
01498 {
01499 res->result = CP_HTTP_RESULT_READ_ERROR;
01500 return res;
01501 }
01502
01503 res->response = response;
01504 return res;
01505 }
01506
01507 int cp_httpclient_reopen(cp_httpclient *client, char *host, int port)
01508 {
01509 client->connected = 0;
01510 return cp_client_reopen(client->socket, host, port);
01511 }
01512
01513
01514
01515
01516
01517 #define TRANSFER_STAGE_CONNECT 1
01518 #define TRANSFER_STAGE_WRITE 2
01519 #define TRANSFER_STAGE_READ 3
01520 #define TRANSFER_STAGE_DONE 4
01521
01522 #define TRANSFER_TYPE_LENGTH 1
01523 #define TRANSFER_TYPE_CHUNKED 2
01524 #define TRANSFER_TYPE_READ_TO_EOF 3
01525
01526 static int set_async_stack_size(cp_httpclient_ctl *ctl, int size)
01527 {
01528 void *p;
01529 if (size > ctl->size)
01530 {
01531 ctl->size = size * 2;
01532 #ifdef CP_HAS_POLL
01533 p = realloc(ctl->ufds, ctl->size * sizeof(struct pollfd));
01534 if (p == NULL) return -1;
01535 ctl->ufds = p;
01536
01537
01538 #endif
01539 p = realloc(ctl->desc,
01540 ctl->size * sizeof(cp_http_transfer_descriptor *));
01541 if (p == NULL) return -1;
01542 ctl->desc = p;
01543 }
01544 ctl->nfds = size;
01545 return 0;
01546 }
01547
01548 cp_httpclient_result *cp_httpclient_result_create(cp_httpclient *client)
01549 {
01550 cp_httpclient_result *res = calloc(1, sizeof(cp_httpclient_result));
01551 if (res == NULL) return NULL;
01552 res->client = client;
01553 return res;
01554 }
01555
01556 static cp_httpclient_result *
01557 cp_httpclient_result_new(cp_http_transfer_descriptor *desc,
01558 cp_http_result_status status,
01559 cp_http_response *response)
01560 {
01561 cp_httpclient_result *res = calloc(1, sizeof(cp_httpclient_result));
01562 if (res == NULL) goto RESULT_ERROR;
01563 res->id = desc->id;
01564 res->client = desc->owner;
01565 res->result = status;
01566 res->response = response;
01567
01568 return res;
01569
01570 RESULT_ERROR:
01571 cp_httpclient_result_destroy(res);
01572 return NULL;
01573 }
01574
01575 void cp_httpclient_result_destroy(cp_httpclient_result *res)
01576 {
01577 if (res)
01578 {
01579 if (res->response) cp_http_response_delete(res->response);
01580 free(res);
01581 }
01582 }
01583
01584 void *cp_httpclient_result_id(cp_httpclient_result *res)
01585 {
01586 return res->id;
01587 }
01588
01589 cp_http_result_status cp_httpclient_result_status(cp_httpclient_result *res)
01590 {
01591 return res->result;
01592 }
01593
01594 cp_http_response *cp_httpclient_result_get_response(cp_httpclient_result *res)
01595 {
01596 return res->response;
01597 }
01598
01599 cp_httpclient *cp_httpclient_result_get_client(cp_httpclient_result *res)
01600 {
01601 return res->client;
01602 }
01603
01604 static void
01605 cp_http_transfer_descriptor_callback(cp_http_transfer_descriptor *desc)
01606 {
01607 cp_httpclient_result *res;
01608 if (desc->callback)
01609 {
01610 res = cp_httpclient_result_new(desc,
01611 (cp_http_result_status) desc->status, desc->res);
01612 if (res)
01613 {
01614 desc->callback(res);
01615 cp_httpclient_result_destroy(res);
01616 }
01617 }
01618 }
01619
01620 static void prepare_async_stack_fds(cp_httpclient_ctl *ctl)
01621 {
01622 cp_hashlist_iterator *i;
01623 int j;
01624 int rc;
01625 cp_http_transfer_descriptor *desc;
01626
01627 i = cp_hashlist_create_iterator(ctl->stack, COLLECTION_LOCK_READ);
01628 j = 0;
01629 #ifndef CP_HAS_POLL
01630 FD_ZERO(&ctl->ufds_r);
01631 FD_ZERO(&ctl->ufds_w);
01632 ctl->fdmax = 0;
01633 #endif
01634 while ((desc = cp_hashlist_iterator_next_value(i)) != NULL)
01635 {
01636 if (desc->stage == TRANSFER_STAGE_CONNECT)
01637 {
01638 if (!desc->owner->connected)
01639 {
01640 rc = do_connect(desc->owner);
01641 if (rc)
01642 {
01643 cp_hashlist_iterator_remove(i);
01644 desc->stage = TRANSFER_STAGE_DONE;
01645 desc->status = CP_HTTP_RESULT_CONNECTION_FAILED;
01646 continue;
01647 }
01648 }
01649 desc->stage = TRANSFER_STAGE_WRITE;
01650 }
01651
01652 ctl->desc[j] = desc;
01653 #ifdef CP_HAS_POLL
01654 ctl->ufds[j].fd = desc->owner->socket->fd;
01655 ctl->ufds[j].events = 0;
01656 if (desc->stage == TRANSFER_STAGE_WRITE)
01657 ctl->ufds[j].events |= POLLOUT;
01658 else
01659 ctl->ufds[j].events |= POLLIN;
01660 #else
01661 if (desc->stage == TRANSFER_STAGE_WRITE)
01662 FD_SET(desc->owner->socket->fd, &ctl->ufds_w);
01663 else
01664 FD_SET(desc->owner->socket->fd, &ctl->ufds_r);
01665 if (desc->owner->socket->fd > ctl->fdmax)
01666 ctl->fdmax = desc->owner->socket->fd;
01667 #endif
01668 j++;
01669 }
01670 ctl->nfds = j;
01671 cp_hashlist_iterator_destroy(i);
01672 }
01673
01674 static void
01675 cp_http_transfer_descriptor_do_write(cp_http_transfer_descriptor *desc)
01676 {
01677 int rc;
01678 char *index = &desc->buf->data[desc->buf->len - desc->remaining];
01679 #ifdef CP_USE_SSL
01680 if (desc->owner->socket->use_ssl)
01681 rc = SSL_write(desc->owner->socket->ssl, index, desc->remaining);
01682 else
01683 #endif
01684 rc = send(desc->owner->socket->fd, index, desc->remaining, 0);
01685
01686 if (rc == -1)
01687 {
01688 int err = errno;
01689 if (!(err == EAGAIN || err == EINTR))
01690 {
01691 desc->stage = TRANSFER_STAGE_DONE;
01692 desc->status = CP_HTTP_RESULT_WRITE_ERROR;
01693 return;
01694 }
01695 }
01696 else
01697 desc->owner->socket->bytes_sent += rc;
01698
01699 desc->remaining -= rc;
01700 if (desc->remaining == 0)
01701 {
01702 desc->stage = TRANSFER_STAGE_READ;
01703 desc->buf->len = 0;
01704 }
01705 }
01706
01707 static void transfer_chunked(cp_http_transfer_descriptor *desc,
01708 char *tmpbuf,
01709 int index,
01710 int len)
01711 {
01712 char *curr = &tmpbuf[index];
01713 int read_len;
01714
01715 if (desc->remaining)
01716 {
01717 read_len = len - index;
01718 if (read_len <= desc->remaining)
01719 {
01720 cp_string_cat_bin(desc->res->content, curr, read_len);
01721 desc->remaining -= read_len;
01722 return;
01723 }
01724 else
01725 {
01726 cp_string_cat_bin(desc->res->content, curr, desc->remaining);
01727 curr = &tmpbuf[index + desc->remaining];
01728 desc->remaining = 0;
01729 }
01730 }
01731
01732 while (*curr == '\r' || *curr == '\n') curr++;
01733 if (*curr == 0) return;
01734
01735 read_len = xtoi(curr);
01736 if (read_len == 0)
01737 {
01738 desc->stage = TRANSFER_STAGE_DONE;
01739 desc->status = CP_HTTP_RESULT_SUCCESS;
01740 return;
01741 }
01742 else
01743 {
01744 char *str = strchr(curr, '\n');
01745 if (str == NULL)
01746 {
01747 desc->stage = TRANSFER_STAGE_DONE;
01748 desc->status = CP_HTTP_RESULT_READ_ERROR;
01749 }
01750 curr = str + 1;
01751 }
01752 desc->remaining = read_len;
01753
01754
01755
01756 if (curr - tmpbuf < len)
01757 transfer_chunked(desc, tmpbuf, (curr - tmpbuf), len);
01758 }
01759
01760 static void
01761 cp_http_transfer_descriptor_do_read(cp_http_transfer_descriptor *desc)
01762 {
01763 char tmpbuf[HTTP_PARSE_BUFSIZE + 1];
01764 int read_len;
01765 int rc;
01766
01767
01768 if (desc->remaining && desc->remaining < HTTP_PARSE_BUFSIZE)
01769 read_len = desc->remaining;
01770 else
01771 read_len = HTTP_PARSE_BUFSIZE;
01772
01773 rc = read_chunk(desc->owner, tmpbuf, read_len);
01774
01775
01776 if (rc < 0) goto READ_ERROR;
01777 if (rc == 0)
01778 {
01779 if (desc->transfer != TRANSFER_TYPE_READ_TO_EOF) goto READ_ERROR;
01780 desc->stage = TRANSFER_STAGE_DONE;
01781 desc->status = CP_HTTP_RESULT_SUCCESS;
01782 return;
01783 }
01784
01785
01786 if (desc->read_stage == RESPONSE_BODY)
01787 {
01788 switch (desc->transfer)
01789 {
01790 case TRANSFER_TYPE_CHUNKED:
01791 transfer_chunked(desc, tmpbuf, 0, rc);
01792 break;
01793
01794 case TRANSFER_TYPE_LENGTH:
01795 cp_string_cat_bin(desc->res->content, tmpbuf, rc);
01796 desc->remaining -= rc;
01797 if (desc->remaining == 0)
01798 {
01799 desc->stage = TRANSFER_STAGE_DONE;
01800 desc->status = CP_HTTP_RESULT_SUCCESS;
01801 }
01802 break;
01803
01804 case TRANSFER_TYPE_READ_TO_EOF:
01805 cp_string_cat_bin(desc->res->content, tmpbuf, rc);
01806 break;
01807 }
01808
01809 return;
01810 }
01811
01812
01813 cp_string_cat_bin(desc->buf, tmpbuf, rc);
01814 desc->buf->data[desc->buf->len] = '\0';
01815
01816 if (desc->read_stage == RESPONSE_INIT)
01817 {
01818 desc->read_ptr = desc->buf->data;
01819 desc->res = cp_http_client_response_create();
01820 if (read_status_line(desc->res, &desc->read_ptr, desc->buf->len))
01821 return;
01822
01823 desc->read_stage = RESPONSE_HEADERS;
01824 }
01825
01826
01827 if (desc->read_stage == RESPONSE_HEADERS)
01828 {
01829 int rv;
01830 char *prm;
01831
01832 rv = read_headers(desc->owner, desc->res, desc->uri, &desc->read_ptr,
01833 desc->buf->len + desc->skip -
01834 (desc->read_ptr - desc->buf->data));
01835 if (rv == -1)
01836 return;
01837 else if (rv == EAGAIN)
01838 {
01839 desc->skip += rc;
01840 return;
01841 }
01842
01843 desc->read_stage = RESPONSE_BODY;
01844 if ((prm = cp_hashtable_get(desc->res->header, "Transfer-Encoding"))
01845 && strcasecmp(prm, "chunked") == 0)
01846 {
01847 desc->transfer = TRANSFER_TYPE_CHUNKED;
01848
01849 desc->res->content = cp_string_create_empty(0x400);
01850 transfer_chunked(desc, tmpbuf,
01851 desc->read_ptr - desc->buf->data, rc);
01852 return;
01853 }
01854 else if ((prm = cp_hashtable_get(desc->res->header, "Content-Length")))
01855 {
01856 char *src = find_response_end(desc->read_ptr);
01857 if (src == NULL) goto READ_ERROR;
01858 desc->transfer = TRANSFER_TYPE_LENGTH;
01859
01860 desc->read_ptr = src;
01861 desc->res->len = atoi(prm);
01862 desc->res->content =
01863 cp_string_create(desc->read_ptr,
01864 rc + desc->skip -
01865 (desc->read_ptr - desc->buf->data));
01866 desc->remaining =
01867 desc->res->len - desc->res->content->len;
01868 if (desc->remaining == 0)
01869 {
01870 desc->stage = TRANSFER_STAGE_DONE;
01871 desc->status = CP_HTTP_RESULT_SUCCESS;
01872 return;
01873 }
01874 }
01875 else if (cp_hashtable_get(desc->res->header, "Content-Type"))
01876 {
01877 char *src = find_response_end(desc->read_ptr);
01878 if (src == NULL) goto READ_ERROR;
01879 desc->transfer = TRANSFER_TYPE_READ_TO_EOF;
01880
01881 desc->read_ptr = src;
01882 desc->res->content =
01883 cp_string_create(desc->read_ptr,
01884 rc + desc->skip - (desc->read_ptr - desc->buf->data));
01885 return;
01886 }
01887 }
01888
01889 return;
01890
01891 READ_ERROR:
01892 desc->stage = TRANSFER_STAGE_DONE;
01893 desc->status = CP_HTTP_RESULT_READ_ERROR;
01894 }
01895
01897 static void
01898 cp_http_transfer_descriptor_do_redirect(cp_http_transfer_descriptor *desc)
01899 {
01900 if (desc->owner->follow_redirects &&
01901 (desc->res->status == HTTP_302_FOUND ||
01902 desc->res->status == HTTP_301_MOVED_PERMANENTLY ||
01903 desc->res->status == HTTP_300_MULTIPLE_CHOICES) &&
01904 desc->redirects < desc->owner->max_redirects)
01905 {
01906 cp_url_descriptor *udesc;
01907 char *url = cp_hashtable_get(desc->res->header, "Location");
01908 if (url == NULL) return;
01909 udesc = cp_url_descriptor_parse(url);
01910 if (udesc == NULL) return;
01911
01912 if (udesc->host == NULL)
01913 {
01914 udesc->host = strdup(desc->owner->host);
01915 udesc->port = desc->owner->port;
01916 #ifdef CP_USE_SSL
01917 udesc->ssl = desc->owner->socket->use_ssl;
01918 #endif
01919 }
01920 else if (strcmp(udesc->host, desc->owner->host) ||
01921 udesc->port != desc->owner->port)
01922 {
01923 free(desc->owner->host);
01924 desc->owner->host = strdup(udesc->host);
01925 if (!desc->owner->proxy_host)
01926 cp_client_reopen(desc->owner->socket, udesc->host, udesc->port);
01927
01928 }
01929
01930 cp_string_destroy(desc->buf);
01931 desc->buf = cp_httpclient_format_request(desc->owner, udesc->uri);
01932 desc->remaining = desc->buf->len;
01933 desc->stage = TRANSFER_STAGE_WRITE;
01934 desc->read_stage = RESPONSE_INIT;
01935 cp_http_response_delete(desc->res);
01936 desc->res = NULL;
01937 desc->redirects++;
01938
01939 cp_url_descriptor_destroy(udesc);
01940 }
01941 }
01942
01943
01945 typedef struct _async_struct_entry
01946 {
01947 cp_httpclient_ctl *ctl;
01948 int index;
01949 } async_struct_entry;
01950
01951 static void *process_async_stack_entry(void *prm)
01952 {
01953 async_struct_entry *entry = prm;
01954 cp_http_transfer_descriptor *desc;
01955 cp_httpclient_ctl *ctl = entry->