source: squid-ssl/trunk/fuentes/src/client_side_request.cc @ 5502

Last change on this file since 5502 was 5502, checked in by Juanma, 4 years ago

Initial release

File size: 71.3 KB
Line 
1/*
2 * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9/* DEBUG: section 85    Client-side Request Routines */
10
11/*
12 * General logic of request processing:
13 *
14 * We run a series of tests to determine if access will be permitted, and to do
15 * any redirection. Then we call into the result clientStream to retrieve data.
16 * From that point on it's up to reply management.
17 */
18
19#include "squid.h"
20#include "acl/FilledChecklist.h"
21#include "acl/Gadgets.h"
22#include "anyp/PortCfg.h"
23#include "base/AsyncJobCalls.h"
24#include "client_side.h"
25#include "client_side_reply.h"
26#include "client_side_request.h"
27#include "ClientRequestContext.h"
28#include "clientStream.h"
29#include "comm/Connection.h"
30#include "comm/Write.h"
31#include "err_detail_type.h"
32#include "errorpage.h"
33#include "fd.h"
34#include "fde.h"
35#include "format/Token.h"
36#include "gopher.h"
37#include "helper.h"
38#include "helper/Reply.h"
39#include "http.h"
40#include "HttpHdrCc.h"
41#include "HttpReply.h"
42#include "HttpRequest.h"
43#include "ip/QosConfig.h"
44#include "ipcache.h"
45#include "log/access_log.h"
46#include "MemObject.h"
47#include "Parsing.h"
48#include "profiler/Profiler.h"
49#include "redirect.h"
50#include "SquidConfig.h"
51#include "SquidTime.h"
52#include "Store.h"
53#include "StrList.h"
54#include "tools.h"
55#include "URL.h"
56#include "wordlist.h"
57#if USE_AUTH
58#include "auth/UserRequest.h"
59#endif
60#if USE_ADAPTATION
61#include "adaptation/AccessCheck.h"
62#include "adaptation/Answer.h"
63#include "adaptation/Iterator.h"
64#include "adaptation/Service.h"
65#if ICAP_CLIENT
66#include "adaptation/icap/History.h"
67#endif
68#endif
69#if USE_OPENSSL
70#include "ssl/ServerBump.h"
71#include "ssl/support.h"
72#endif
73
74#if LINGERING_CLOSE
75#define comm_close comm_lingering_close
76#endif
77
78static const char *const crlf = "\r\n";
79
80#if FOLLOW_X_FORWARDED_FOR
81static void clientFollowXForwardedForCheck(allow_t answer, void *data);
82#endif /* FOLLOW_X_FORWARDED_FOR */
83
84ErrorState *clientBuildError(err_type, Http::StatusCode, char const *url, Ip::Address &, HttpRequest *);
85
86CBDATA_CLASS_INIT(ClientRequestContext);
87
88/* Local functions */
89/* other */
90static void clientAccessCheckDoneWrapper(allow_t, void *);
91#if USE_OPENSSL
92static void sslBumpAccessCheckDoneWrapper(allow_t, void *);
93#endif
94static int clientHierarchical(ClientHttpRequest * http);
95static void clientInterpretRequestHeaders(ClientHttpRequest * http);
96static HLPCB clientRedirectDoneWrapper;
97static HLPCB clientStoreIdDoneWrapper;
98static void checkNoCacheDoneWrapper(allow_t, void *);
99SQUIDCEXTERN CSR clientGetMoreData;
100SQUIDCEXTERN CSS clientReplyStatus;
101SQUIDCEXTERN CSD clientReplyDetach;
102static void checkFailureRatio(err_type, hier_code);
103
104ClientRequestContext::~ClientRequestContext()
105{
106    /*
107     * Release our "lock" on our parent, ClientHttpRequest, if we
108     * still have one
109     */
110
111    if (http)
112        cbdataReferenceDone(http);
113
114    delete error;
115    debugs(85,3, HERE << this << " ClientRequestContext destructed");
116}
117
118ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), store_id_state(REDIRECT_NONE),error(NULL), readNextRequest(false)
119{
120    http_access_done = false;
121    redirect_done = false;
122    redirect_fail_count = 0;
123    store_id_done = false;
124    store_id_fail_count = 0;
125    no_cache_done = false;
126    interpreted_req_hdrs = false;
127#if USE_OPENSSL
128    sslBumpCheckDone = false;
129#endif
130    debugs(85,3, HERE << this << " ClientRequestContext constructed");
131}
132
133CBDATA_CLASS_INIT(ClientHttpRequest);
134
135ClientHttpRequest::ClientHttpRequest(ConnStateData * aConn) :
136#if USE_ADAPTATION
137    AsyncJob("ClientHttpRequest"),
138#endif
139    loggingEntry_(NULL)
140{
141    setConn(aConn);
142    al = new AccessLogEntry;
143    al->cache.start_time = current_time;
144    if (aConn) {
145        al->tcpClient = clientConnection = aConn->clientConnection;
146        al->cache.port = aConn->port;
147        al->cache.caddr = aConn->log_addr;
148
149#if USE_OPENSSL
150        if (aConn->clientConnection != NULL && aConn->clientConnection->isOpen()) {
151            if (SSL *ssl = fd_table[aConn->clientConnection->fd].ssl)
152                al->cache.sslClientCert.reset(SSL_get_peer_certificate(ssl));
153        }
154#endif
155    }
156    dlinkAdd(this, &active, &ClientActiveRequests);
157#if USE_ADAPTATION
158    request_satisfaction_mode = false;
159#endif
160#if USE_OPENSSL
161    sslBumpNeed_ = Ssl::bumpEnd;
162#endif
163}
164
165/*
166 * returns true if client specified that the object must come from the cache
167 * without contacting origin server
168 */
169bool
170ClientHttpRequest::onlyIfCached()const
171{
172    assert(request);
173    return request->cache_control &&
174           request->cache_control->onlyIfCached();
175}
176
177/**
178 * This function is designed to serve a fairly specific purpose.
179 * Occasionally our vBNS-connected caches can talk to each other, but not
180 * the rest of the world.  Here we try to detect frequent failures which
181 * make the cache unusable (e.g. DNS lookup and connect() failures).  If
182 * the failure:success ratio goes above 1.0 then we go into "hit only"
183 * mode where we only return UDP_HIT or UDP_MISS_NOFETCH.  Neighbors
184 * will only fetch HITs from us if they are using the ICP protocol.  We
185 * stay in this mode for 5 minutes.
186 *
187 * Duane W., Sept 16, 1996
188 */
189static void
190checkFailureRatio(err_type etype, hier_code hcode)
191{
192    // Can be set at compile time with -D compiler flag
193#ifndef FAILURE_MODE_TIME
194#define FAILURE_MODE_TIME 300
195#endif
196
197    if (hcode == HIER_NONE)
198        return;
199
200    // don't bother when ICP is disabled.
201    if (Config.Port.icp <= 0)
202        return;
203
204    static double magic_factor = 100.0;
205    double n_good;
206    double n_bad;
207
208    n_good = magic_factor / (1.0 + request_failure_ratio);
209
210    n_bad = magic_factor - n_good;
211
212    switch (etype) {
213
214    case ERR_DNS_FAIL:
215
216    case ERR_CONNECT_FAIL:
217    case ERR_SECURE_CONNECT_FAIL:
218
219    case ERR_READ_ERROR:
220        ++n_bad;
221        break;
222
223    default:
224        ++n_good;
225    }
226
227    request_failure_ratio = n_bad / n_good;
228
229    if (hit_only_mode_until > squid_curtime)
230        return;
231
232    if (request_failure_ratio < 1.0)
233        return;
234
235    debugs(33, DBG_CRITICAL, "WARNING: Failure Ratio at "<< std::setw(4)<<
236           std::setprecision(3) << request_failure_ratio);
237
238    debugs(33, DBG_CRITICAL, "WARNING: ICP going into HIT-only mode for " <<
239           FAILURE_MODE_TIME / 60 << " minutes...");
240
241    hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME;
242
243    request_failure_ratio = 0.8;    /* reset to something less than 1.0 */
244}
245
246ClientHttpRequest::~ClientHttpRequest()
247{
248    debugs(33, 3, "httpRequestFree: " << uri);
249    PROF_start(httpRequestFree);
250
251    // Even though freeResources() below may destroy the request,
252    // we no longer set request->body_pipe to NULL here
253    // because we did not initiate that pipe (ConnStateData did)
254
255    /* the ICP check here was erroneous
256     * - StoreEntry::releaseRequest was always called if entry was valid
257     */
258    assert(logType < LOG_TYPE_MAX);
259
260    logRequest();
261
262    loggingEntry(NULL);
263
264    if (request)
265        checkFailureRatio(request->errType, al->hier.code);
266
267    freeResources();
268
269#if USE_ADAPTATION
270    announceInitiatorAbort(virginHeadSource);
271
272    if (adaptedBodySource != NULL)
273        stopConsumingFrom(adaptedBodySource);
274#endif
275
276    if (calloutContext)
277        delete calloutContext;
278
279    clientConnection = NULL;
280
281    if (conn_)
282        cbdataReferenceDone(conn_);
283
284    /* moving to the next connection is handled by the context free */
285    dlinkDelete(&active, &ClientActiveRequests);
286
287    PROF_stop(httpRequestFree);
288}
289
290/**
291 * Create a request and kick it off
292 *
293 * \retval 0     success
294 * \retval -1    failure
295 *
296 * TODO: Pass in the buffers to be used in the inital Read request, as they are
297 * determined by the user
298 */
299int
300clientBeginRequest(const HttpRequestMethod& method, char const *url, CSCB * streamcallback,
301                   CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
302                   char *tailbuf, size_t taillen)
303{
304    size_t url_sz;
305    ClientHttpRequest *http = new ClientHttpRequest(NULL);
306    HttpRequest *request;
307    StoreIOBuffer tempBuffer;
308    if (http->al != NULL)
309        http->al->cache.start_time = current_time;
310    /* this is only used to adjust the connection offset in client_side.c */
311    http->req_sz = 0;
312    tempBuffer.length = taillen;
313    tempBuffer.data = tailbuf;
314    /* client stream setup */
315    clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
316                     clientReplyStatus, new clientReplyContext(http), streamcallback,
317                     streamdetach, streamdata, tempBuffer);
318    /* make it visible in the 'current acctive requests list' */
319    /* Set flags */
320    /* internal requests only makes sense in an
321     * accelerator today. TODO: accept flags ? */
322    http->flags.accel = true;
323    /* allow size for url rewriting */
324    url_sz = strlen(url) + Config.appendDomainLen + 5;
325    http->uri = (char *)xcalloc(url_sz, 1);
326    strcpy(http->uri, url);
327
328    if ((request = HttpRequest::CreateFromUrlAndMethod(http->uri, method)) == NULL) {
329        debugs(85, 5, "Invalid URL: " << http->uri);
330        return -1;
331    }
332
333    /*
334     * now update the headers in request with our supplied headers. urlParse
335     * should return a blank header set, but we use Update to be sure of
336     * correctness.
337     */
338    if (header)
339        request->header.update(header, NULL);
340
341    http->log_uri = xstrdup(urlCanonicalClean(request));
342
343    /* http struct now ready */
344
345    /*
346     * build new header list *? TODO
347     */
348    request->flags.accelerated = http->flags.accel;
349
350    request->flags.internalClient = true;
351
352    /* this is an internally created
353     * request, not subject to acceleration
354     * target overrides */
355    /*
356     * FIXME? Do we want to detect and handle internal requests of internal
357     * objects ?
358     */
359
360    /* Internally created requests cannot have bodies today */
361    request->content_length = 0;
362
363    request->client_addr.setNoAddr();
364
365#if FOLLOW_X_FORWARDED_FOR
366    request->indirect_client_addr.setNoAddr();
367#endif /* FOLLOW_X_FORWARDED_FOR */
368
369    request->my_addr.setNoAddr();   /* undefined for internal requests */
370
371    request->my_addr.port(0);
372
373    /* Our version is HTTP/1.1 */
374    request->http_ver = Http::ProtocolVersion(1,1);
375
376    http->request = request;
377    HTTPMSGLOCK(http->request);
378
379    /* optional - skip the access check ? */
380    http->calloutContext = new ClientRequestContext(http);
381
382    http->calloutContext->http_access_done = false;
383
384    http->calloutContext->redirect_done = true;
385
386    http->calloutContext->no_cache_done = true;
387
388    http->doCallouts();
389
390    return 0;
391}
392
393bool
394ClientRequestContext::httpStateIsValid()
395{
396    ClientHttpRequest *http_ = http;
397
398    if (cbdataReferenceValid(http_))
399        return true;
400
401    http = NULL;
402
403    cbdataReferenceDone(http_);
404
405    return false;
406}
407
408#if FOLLOW_X_FORWARDED_FOR
409/**
410 * clientFollowXForwardedForCheck() checks the content of X-Forwarded-For:
411 * against the followXFF ACL, or cleans up and passes control to
412 * clientAccessCheck().
413 *
414 * The trust model here is a little ambiguous. So to clarify the logic:
415 * - we may always use the direct client address as the client IP.
416 * - these trust tests merey tell whether we trust given IP enough to believe the
417 *   IP string which it appended to the X-Forwarded-For: header.
418 * - if at any point we don't trust what an IP adds we stop looking.
419 * - at that point the current contents of indirect_client_addr are the value set
420 *   by the last previously trusted IP.
421 * ++ indirect_client_addr contains the remote direct client from the trusted peers viewpoint.
422 */
423static void
424clientFollowXForwardedForCheck(allow_t answer, void *data)
425{
426    ClientRequestContext *calloutContext = (ClientRequestContext *) data;
427
428    if (!calloutContext->httpStateIsValid())
429        return;
430
431    ClientHttpRequest *http = calloutContext->http;
432    HttpRequest *request = http->request;
433
434    /*
435     * answer should be be ACCESS_ALLOWED or ACCESS_DENIED if we are
436     * called as a result of ACL checks, or -1 if we are called when
437     * there's nothing left to do.
438     */
439    if (answer == ACCESS_ALLOWED &&
440            request->x_forwarded_for_iterator.size () != 0) {
441
442        /*
443         * Remove the last comma-delimited element from the
444         * x_forwarded_for_iterator and use it to repeat the cycle.
445         */
446        const char *p;
447        const char *asciiaddr;
448        int l;
449        Ip::Address addr;
450        p = request->x_forwarded_for_iterator.termedBuf();
451        l = request->x_forwarded_for_iterator.size();
452
453        /*
454        * XXX x_forwarded_for_iterator should really be a list of
455        * IP addresses, but it's a String instead.  We have to
456        * walk backwards through the String, biting off the last
457        * comma-delimited part each time.  As long as the data is in
458        * a String, we should probably implement and use a variant of
459        * strListGetItem() that walks backwards instead of forwards
460        * through a comma-separated list.  But we don't even do that;
461        * we just do the work in-line here.
462        */
463        /* skip trailing space and commas */
464        while (l > 0 && (p[l-1] == ',' || xisspace(p[l-1])))
465            --l;
466        request->x_forwarded_for_iterator.cut(l);
467        /* look for start of last item in list */
468        while (l > 0 && ! (p[l-1] == ',' || xisspace(p[l-1])))
469            --l;
470        asciiaddr = p+l;
471        if ((addr = asciiaddr)) {
472            request->indirect_client_addr = addr;
473            request->x_forwarded_for_iterator.cut(l);
474            calloutContext->acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
475            if (!Config.onoff.acl_uses_indirect_client) {
476                /* override the default src_addr tested if we have to go deeper than one level into XFF */
477                Filled(calloutContext->acl_checklist)->src_addr = request->indirect_client_addr;
478            }
479            calloutContext->acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, data);
480            return;
481        }
482    } /*if (answer == ACCESS_ALLOWED &&
483        request->x_forwarded_for_iterator.size () != 0)*/
484
485    /* clean up, and pass control to clientAccessCheck */
486    if (Config.onoff.log_uses_indirect_client) {
487        /*
488        * Ensure that the access log shows the indirect client
489        * instead of the direct client.
490        */
491        ConnStateData *conn = http->getConn();
492        conn->log_addr = request->indirect_client_addr;
493        http->al->cache.caddr = conn->log_addr;
494    }
495    request->x_forwarded_for_iterator.clean();
496    request->flags.done_follow_x_forwarded_for = true;
497
498    if (answer != ACCESS_ALLOWED && answer != ACCESS_DENIED) {
499        debugs(28, DBG_CRITICAL, "ERROR: Processing X-Forwarded-For. Stopping at IP address: " << request->indirect_client_addr );
500    }
501
502    /* process actual access ACL as normal. */
503    calloutContext->clientAccessCheck();
504}
505#endif /* FOLLOW_X_FORWARDED_FOR */
506
507static void
508hostHeaderIpVerifyWrapper(const ipcache_addrs* ia, const DnsLookupDetails &dns, void *data)
509{
510    ClientRequestContext *c = static_cast<ClientRequestContext*>(data);
511    c->hostHeaderIpVerify(ia, dns);
512}
513
514void
515ClientRequestContext::hostHeaderIpVerify(const ipcache_addrs* ia, const DnsLookupDetails &dns)
516{
517    Comm::ConnectionPointer clientConn = http->getConn()->clientConnection;
518
519    // note the DNS details for the transaction stats.
520    http->request->recordLookup(dns);
521
522    if (ia != NULL && ia->count > 0) {
523        // Is the NAT destination IP in DNS?
524        for (int i = 0; i < ia->count; ++i) {
525            if (clientConn->local.matchIPAddr(ia->in_addrs[i]) == 0) {
526                debugs(85, 3, HERE << "validate IP " << clientConn->local << " possible from Host:");
527                http->request->flags.hostVerified = true;
528                http->doCallouts();
529                return;
530            }
531            debugs(85, 3, HERE << "validate IP " << clientConn->local << " non-match from Host: IP " << ia->in_addrs[i]);
532        }
533    }
534    debugs(85, 3, HERE << "FAIL: validate IP " << clientConn->local << " possible from Host:");
535    hostHeaderVerifyFailed("local IP", "any domain IP");
536}
537
538void
539ClientRequestContext::hostHeaderVerifyFailed(const char *A, const char *B)
540{
541    // IP address validation for Host: failed. Admin wants to ignore them.
542    // NP: we do not yet handle CONNECT tunnels well, so ignore for them
543    if (!Config.onoff.hostStrictVerify && http->request->method != Http::METHOD_CONNECT) {
544        debugs(85, 3, "SECURITY ALERT: Host header forgery detected on " << http->getConn()->clientConnection <<
545               " (" << A << " does not match " << B << ") on URL: " << urlCanonical(http->request));
546
547        // NP: it is tempting to use 'flags.noCache' but that is all about READing cache data.
548        // The problems here are about WRITE for new cache content, which means flags.cachable
549        http->request->flags.cachable = false; // MUST NOT cache (for now)
550        // XXX: when we have updated the cache key to base on raw-IP + URI this cacheable limit can go.
551        http->request->flags.hierarchical = false; // MUST NOT pass to peers (for now)
552        // XXX: when we have sorted out the best way to relay requests properly to peers this hierarchical limit can go.
553        http->doCallouts();
554        return;
555    }
556
557    debugs(85, DBG_IMPORTANT, "SECURITY ALERT: Host header forgery detected on " <<
558           http->getConn()->clientConnection << " (" << A << " does not match " << B << ")");
559    debugs(85, DBG_IMPORTANT, "SECURITY ALERT: By user agent: " << http->request->header.getStr(HDR_USER_AGENT));
560    debugs(85, DBG_IMPORTANT, "SECURITY ALERT: on URL: " << urlCanonical(http->request));
561
562    // IP address validation for Host: failed. reject the connection.
563    clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
564    clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
565    assert (repContext);
566    repContext->setReplyToError(ERR_CONFLICT_HOST, Http::scConflict,
567                                http->request->method, NULL,
568                                http->getConn()->clientConnection->remote,
569                                http->request,
570                                NULL,
571#if USE_AUTH
572                                http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
573                                http->getConn()->getAuth() : http->request->auth_user_request);
574#else
575                                NULL);
576#endif
577    node = (clientStreamNode *)http->client_stream.tail->data;
578    clientStreamRead(node, http, node->readBuffer);
579}
580
581void
582ClientRequestContext::hostHeaderVerify()
583{
584    // Require a Host: header.
585    const char *host = http->request->header.getStr(HDR_HOST);
586
587    if (!host) {
588        // TODO: dump out the HTTP/1.1 error about missing host header.
589        // otherwise this is fine, can't forge a header value when its not even set.
590        debugs(85, 3, HERE << "validate skipped with no Host: header present.");
591        http->doCallouts();
592        return;
593    }
594
595    if (http->request->flags.internal) {
596        // TODO: kill this when URL handling allows partial URLs out of accel mode
597        //       and we no longer screw with the URL just to add our internal host there
598        debugs(85, 6, HERE << "validate skipped due to internal composite URL.");
599        http->doCallouts();
600        return;
601    }
602
603    // Locate if there is a port attached, strip ready for IP lookup
604    char *portStr = NULL;
605    char *hostB = xstrdup(host);
606    host = hostB;
607    if (host[0] == '[') {
608        // IPv6 literal.
609        portStr = strchr(hostB, ']');
610        if (portStr && *(++portStr) != ':') {
611            portStr = NULL;
612        }
613    } else {
614        // Domain or IPv4 literal with port
615        portStr = strrchr(hostB, ':');
616    }
617
618    uint16_t port = 0;
619    if (portStr) {
620        *portStr = '\0'; // strip the ':'
621        if (*(++portStr) != '\0') {
622            char *end = NULL;
623            int64_t ret = strtoll(portStr, &end, 10);
624            if (end == portStr || *end != '\0' || ret < 1 || ret > 0xFFFF) {
625                // invalid port details. Replace the ':'
626                *(--portStr) = ':';
627                portStr = NULL;
628            } else
629                port = (ret & 0xFFFF);
630        }
631    }
632
633    debugs(85, 3, HERE << "validate host=" << host << ", port=" << port << ", portStr=" << (portStr?portStr:"NULL"));
634    if (http->request->flags.intercepted || http->request->flags.interceptTproxy) {
635        // verify the Host: port (if any) matches the apparent destination
636        if (portStr && port != http->getConn()->clientConnection->local.port()) {
637            debugs(85, 3, HERE << "FAIL on validate port " << http->getConn()->clientConnection->local.port() <<
638                   " matches Host: port " << port << " (" << portStr << ")");
639            hostHeaderVerifyFailed("intercepted port", portStr);
640        } else {
641            // XXX: match the scheme default port against the apparent destination
642
643            // verify the destination DNS is one of the Host: headers IPs
644            ipcache_nbgethostbyname(host, hostHeaderIpVerifyWrapper, this);
645        }
646    } else if (!Config.onoff.hostStrictVerify) {
647        debugs(85, 3, HERE << "validate skipped.");
648        http->doCallouts();
649    } else if (strlen(host) != strlen(http->request->GetHost())) {
650        // Verify forward-proxy requested URL domain matches the Host: header
651        debugs(85, 3, HERE << "FAIL on validate URL domain length " << http->request->GetHost() << " matches Host: " << host);
652        hostHeaderVerifyFailed(host, http->request->GetHost());
653    } else if (matchDomainName(host, http->request->GetHost()) != 0) {
654        // Verify forward-proxy requested URL domain matches the Host: header
655        debugs(85, 3, HERE << "FAIL on validate URL domain " << http->request->GetHost() << " matches Host: " << host);
656        hostHeaderVerifyFailed(host, http->request->GetHost());
657    } else if (portStr && port != http->request->port) {
658        // Verify forward-proxy requested URL domain matches the Host: header
659        debugs(85, 3, HERE << "FAIL on validate URL port " << http->request->port << " matches Host: port " << portStr);
660        hostHeaderVerifyFailed("URL port", portStr);
661    } else if (!portStr && http->request->method != Http::METHOD_CONNECT && http->request->port != urlDefaultPort(http->request->url.getScheme())) {
662        // Verify forward-proxy requested URL domain matches the Host: header
663        // Special case: we don't have a default-port to check for CONNECT. Assume URL is correct.
664        debugs(85, 3, "FAIL on validate URL port " << http->request->port << " matches Host: default port " << urlDefaultPort(http->request->url.getScheme()));
665        hostHeaderVerifyFailed("URL port", "default port");
666    } else {
667        // Okay no problem.
668        debugs(85, 3, HERE << "validate passed.");
669        http->request->flags.hostVerified = true;
670        http->doCallouts();
671    }
672    safe_free(hostB);
673}
674
675/* This is the entry point for external users of the client_side routines */
676void
677ClientRequestContext::clientAccessCheck()
678{
679#if FOLLOW_X_FORWARDED_FOR
680    if (!http->request->flags.doneFollowXff() &&
681            Config.accessList.followXFF &&
682            http->request->header.has(HDR_X_FORWARDED_FOR)) {
683
684        /* we always trust the direct client address for actual use */
685        http->request->indirect_client_addr = http->request->client_addr;
686        http->request->indirect_client_addr.port(0);
687
688        /* setup the XFF iterator for processing */
689        http->request->x_forwarded_for_iterator = http->request->header.getList(HDR_X_FORWARDED_FOR);
690
691        /* begin by checking to see if we trust direct client enough to walk XFF */
692        acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
693        acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, this);
694        return;
695    }
696#endif
697
698    if (Config.accessList.http) {
699        acl_checklist = clientAclChecklistCreate(Config.accessList.http, http);
700        acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
701    } else {
702        debugs(0, DBG_CRITICAL, "No http_access configuration found. This will block ALL traffic");
703        clientAccessCheckDone(ACCESS_DENIED);
704    }
705}
706
707/**
708 * Identical in operation to clientAccessCheck() but performed later using different configured ACL list.
709 * The default here is to allow all. Since the earlier http_access should do a default deny all.
710 * This check is just for a last-minute denial based on adapted request headers.
711 */
712void
713ClientRequestContext::clientAccessCheck2()
714{
715    if (Config.accessList.adapted_http) {
716        acl_checklist = clientAclChecklistCreate(Config.accessList.adapted_http, http);
717        acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
718    } else {
719        debugs(85, 2, HERE << "No adapted_http_access configuration. default: ALLOW");
720        clientAccessCheckDone(ACCESS_ALLOWED);
721    }
722}
723
724void
725clientAccessCheckDoneWrapper(allow_t answer, void *data)
726{
727    ClientRequestContext *calloutContext = (ClientRequestContext *) data;
728
729    if (!calloutContext->httpStateIsValid())
730        return;
731
732    calloutContext->clientAccessCheckDone(answer);
733}
734
735void
736ClientRequestContext::clientAccessCheckDone(const allow_t &answer)
737{
738    acl_checklist = NULL;
739    err_type page_id;
740    Http::StatusCode status;
741    debugs(85, 2, "The request " << http->request->method << ' ' <<
742           http->uri << " is " << answer <<
743           "; last ACL checked: " << (AclMatchedName ? AclMatchedName : "[none]"));
744
745#if USE_AUTH
746    char const *proxy_auth_msg = "<null>";
747    if (http->getConn() != NULL && http->getConn()->getAuth() != NULL)
748        proxy_auth_msg = http->getConn()->getAuth()->denyMessage("<null>");
749    else if (http->request->auth_user_request != NULL)
750        proxy_auth_msg = http->request->auth_user_request->denyMessage("<null>");
751#endif
752
753    if (answer != ACCESS_ALLOWED) {
754        // auth has a grace period where credentials can be expired but okay not to challenge.
755
756        /* Send an auth challenge or error */
757        // XXX: do we still need aclIsProxyAuth() ?
758        bool auth_challenge = (answer == ACCESS_AUTH_REQUIRED || aclIsProxyAuth(AclMatchedName));
759        debugs(85, 5, "Access Denied: " << http->uri);
760        debugs(85, 5, "AclMatchedName = " << (AclMatchedName ? AclMatchedName : "<null>"));
761#if USE_AUTH
762        if (auth_challenge)
763            debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg ? proxy_auth_msg : "<null>"));
764#endif
765
766        /*
767         * NOTE: get page_id here, based on AclMatchedName because if
768         * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
769         * the clientCreateStoreEntry() call just below.  Pedro Ribeiro
770         * <pribeiro@isel.pt>
771         */
772        page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, answer != ACCESS_AUTH_REQUIRED);
773
774        http->logType = LOG_TCP_DENIED;
775
776        if (auth_challenge) {
777#if USE_AUTH
778            if (http->request->flags.sslBumped) {
779                /*SSL Bumped request, authentication is not possible*/
780                status = Http::scForbidden;
781            } else if (!http->flags.accel) {
782                /* Proxy authorisation needed */
783                status = Http::scProxyAuthenticationRequired;
784            } else {
785                /* WWW authorisation needed */
786                status = Http::scUnauthorized;
787            }
788#else
789            // need auth, but not possible to do.
790            status = Http::scForbidden;
791#endif
792            if (page_id == ERR_NONE)
793                page_id = ERR_CACHE_ACCESS_DENIED;
794        } else {
795            status = Http::scForbidden;
796
797            if (page_id == ERR_NONE)
798                page_id = ERR_ACCESS_DENIED;
799        }
800
801        Ip::Address tmpnoaddr;
802        tmpnoaddr.setNoAddr();
803        error = clientBuildError(page_id, status,
804                                 NULL,
805                                 http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmpnoaddr,
806                                 http->request
807                                );
808
809#if USE_AUTH
810        error->auth_user_request =
811            http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
812            http->getConn()->getAuth() : http->request->auth_user_request;
813#endif
814
815        readNextRequest = true;
816    }
817
818    /* ACCESS_ALLOWED continues here ... */
819    safe_free(http->uri);
820
821    http->uri = xstrdup(urlCanonical(http->request));
822
823    http->doCallouts();
824}
825
826#if USE_ADAPTATION
827void
828ClientHttpRequest::noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer g)
829{
830    debugs(93,3,HERE << this << " adaptationAclCheckDone called");
831
832#if ICAP_CLIENT
833    Adaptation::Icap::History::Pointer ih = request->icapHistory();
834    if (ih != NULL) {
835        if (getConn() != NULL && getConn()->clientConnection != NULL) {
836            ih->rfc931 = getConn()->clientConnection->rfc931;
837#if USE_OPENSSL
838            if (getConn()->clientConnection->isOpen()) {
839                ih->ssluser = sslGetUserEmail(fd_table[getConn()->clientConnection->fd].ssl);
840            }
841#endif
842        }
843        ih->log_uri = log_uri;
844        ih->req_sz = req_sz;
845    }
846#endif
847
848    if (!g) {
849        debugs(85,3, HERE << "no adaptation needed");
850        doCallouts();
851        return;
852    }
853
854    startAdaptation(g);
855}
856
857#endif
858
859static void
860clientRedirectAccessCheckDone(allow_t answer, void *data)
861{
862    ClientRequestContext *context = (ClientRequestContext *)data;
863    ClientHttpRequest *http = context->http;
864    context->acl_checklist = NULL;
865
866    if (answer == ACCESS_ALLOWED)
867        redirectStart(http, clientRedirectDoneWrapper, context);
868    else {
869        Helper::Reply nilReply;
870        nilReply.result = Helper::Error;
871        context->clientRedirectDone(nilReply);
872    }
873}
874
875void
876ClientRequestContext::clientRedirectStart()
877{
878    debugs(33, 5, HERE << "'" << http->uri << "'");
879    (void)SyncNotes(*http->al, *http->request);
880    if (Config.accessList.redirector) {
881        acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http);
882        acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this);
883    } else
884        redirectStart(http, clientRedirectDoneWrapper, this);
885}
886
887/**
888 * This methods handles Access checks result of StoreId access list.
889 * Will handle as "ERR" (no change) in a case Access is not allowed.
890 */
891static void
892clientStoreIdAccessCheckDone(allow_t answer, void *data)
893{
894    ClientRequestContext *context = static_cast<ClientRequestContext *>(data);
895    ClientHttpRequest *http = context->http;
896    context->acl_checklist = NULL;
897
898    if (answer == ACCESS_ALLOWED)
899        storeIdStart(http, clientStoreIdDoneWrapper, context);
900    else {
901        debugs(85, 3, "access denied expected ERR reply handling: " << answer);
902        Helper::Reply nilReply;
903        nilReply.result = Helper::Error;
904        context->clientStoreIdDone(nilReply);
905    }
906}
907
908/**
909 * Start locating an alternative storeage ID string (if any) from admin
910 * configured helper program. This is an asynchronous operation terminating in
911 * ClientRequestContext::clientStoreIdDone() when completed.
912 */
913void
914ClientRequestContext::clientStoreIdStart()
915{
916    debugs(33, 5,"'" << http->uri << "'");
917
918    if (Config.accessList.store_id) {
919        acl_checklist = clientAclChecklistCreate(Config.accessList.store_id, http);
920        acl_checklist->nonBlockingCheck(clientStoreIdAccessCheckDone, this);
921    } else
922        storeIdStart(http, clientStoreIdDoneWrapper, this);
923}
924
925static int
926clientHierarchical(ClientHttpRequest * http)
927{
928    HttpRequest *request = http->request;
929    HttpRequestMethod method = request->method;
930
931    // intercepted requests MUST NOT (yet) be sent to peers unless verified
932    if (!request->flags.hostVerified && (request->flags.intercepted || request->flags.interceptTproxy))
933        return 0;
934
935    /*
936     * IMS needs a private key, so we can use the hierarchy for IMS only if our
937     * neighbors support private keys
938     */
939
940    if (request->flags.ims && !neighbors_do_private_keys)
941        return 0;
942
943    /*
944     * This is incorrect: authenticating requests can be sent via a hierarchy
945     * (they can even be cached if the correct headers are set on the reply)
946     */
947    if (request->flags.auth)
948        return 0;
949
950    if (method == Http::METHOD_TRACE)
951        return 1;
952
953    if (method != Http::METHOD_GET)
954        return 0;
955
956    if (request->flags.loopDetected)
957        return 0;
958
959    if (request->url.getScheme() == AnyP::PROTO_HTTP)
960        return method.respMaybeCacheable();
961
962    if (request->url.getScheme() == AnyP::PROTO_GOPHER)
963        return gopherCachable(request);
964
965    if (request->url.getScheme() == AnyP::PROTO_CACHE_OBJECT)
966        return 0;
967
968    return 1;
969}
970
971static void
972clientCheckPinning(ClientHttpRequest * http)
973{
974    HttpRequest *request = http->request;
975    HttpHeader *req_hdr = &request->header;
976    ConnStateData *http_conn = http->getConn();
977
978    /* Internal requests such as those from ESI includes may be without
979     * a client connection
980     */
981    if (!http_conn)
982        return;
983
984    request->flags.connectionAuthDisabled = http_conn->port->connection_auth_disabled;
985    if (!request->flags.connectionAuthDisabled) {
986        if (Comm::IsConnOpen(http_conn->pinning.serverConnection)) {
987            if (http_conn->pinning.auth) {
988                request->flags.connectionAuth = true;
989                request->flags.auth = true;
990            } else {
991                request->flags.connectionProxyAuth = true;
992            }
993            // These should already be linked correctly.
994            assert(request->clientConnectionManager == http_conn);
995        }
996    }
997
998    /* check if connection auth is used, and flag as candidate for pinning
999     * in such case.
1000     * Note: we may need to set flags.connectionAuth even if the connection
1001     * is already pinned if it was pinned earlier due to proxy auth
1002     */
1003    if (!request->flags.connectionAuth) {
1004        if (req_hdr->has(HDR_AUTHORIZATION) || req_hdr->has(HDR_PROXY_AUTHORIZATION)) {
1005            HttpHeaderPos pos = HttpHeaderInitPos;
1006            HttpHeaderEntry *e;
1007            int may_pin = 0;
1008            while ((e = req_hdr->getEntry(&pos))) {
1009                if (e->id == HDR_AUTHORIZATION || e->id == HDR_PROXY_AUTHORIZATION) {
1010                    const char *value = e->value.rawBuf();
1011                    if (strncasecmp(value, "NTLM ", 5) == 0
1012                            ||
1013                            strncasecmp(value, "Negotiate ", 10) == 0
1014                            ||
1015                            strncasecmp(value, "Kerberos ", 9) == 0) {
1016                        if (e->id == HDR_AUTHORIZATION) {
1017                            request->flags.connectionAuth = true;
1018                            may_pin = 1;
1019                        } else {
1020                            request->flags.connectionProxyAuth = true;
1021                            may_pin = 1;
1022                        }
1023                    }
1024                }
1025            }
1026            if (may_pin && !request->pinnedConnection()) {
1027                // These should already be linked correctly. Just need the ServerConnection to pinn.
1028                assert(request->clientConnectionManager == http_conn);
1029            }
1030        }
1031    }
1032}
1033
1034static void
1035clientInterpretRequestHeaders(ClientHttpRequest * http)
1036{
1037    HttpRequest *request = http->request;
1038    HttpHeader *req_hdr = &request->header;
1039    bool no_cache = false;
1040    const char *str;
1041
1042    request->imslen = -1;
1043    request->ims = req_hdr->getTime(HDR_IF_MODIFIED_SINCE);
1044
1045    if (request->ims > 0)
1046        request->flags.ims = true;
1047
1048    if (!request->flags.ignoreCc) {
1049        if (request->cache_control) {
1050            if (request->cache_control->hasNoCache())
1051                no_cache=true;
1052
1053            // RFC 2616: treat Pragma:no-cache as if it was Cache-Control:no-cache when Cache-Control is missing
1054        } else if (req_hdr->has(HDR_PRAGMA))
1055            no_cache = req_hdr->hasListMember(HDR_PRAGMA,"no-cache",',');
1056
1057        /*
1058        * Work around for supporting the Reload button in IE browsers when Squid
1059        * is used as an accelerator or transparent proxy, by turning accelerated
1060        * IMS request to no-cache requests. Now knows about IE 5.5 fix (is
1061        * actually only fixed in SP1, but we can't tell whether we are talking to
1062        * SP1 or not so all 5.5 versions are treated 'normally').
1063        */
1064        if (Config.onoff.ie_refresh) {
1065            if (http->flags.accel && request->flags.ims) {
1066                if ((str = req_hdr->getStr(HDR_USER_AGENT))) {
1067                    if (strstr(str, "MSIE 5.01") != NULL)
1068                        no_cache=true;
1069                    else if (strstr(str, "MSIE 5.0") != NULL)
1070                        no_cache=true;
1071                    else if (strstr(str, "MSIE 4.") != NULL)
1072                        no_cache=true;
1073                    else if (strstr(str, "MSIE 3.") != NULL)
1074                        no_cache=true;
1075                }
1076            }
1077        }
1078    }
1079
1080    if (request->method == Http::METHOD_OTHER) {
1081        no_cache=true;
1082    }
1083
1084    if (no_cache) {
1085#if USE_HTTP_VIOLATIONS
1086
1087        if (Config.onoff.reload_into_ims)
1088            request->flags.nocacheHack = true;
1089        else if (refresh_nocache_hack)
1090            request->flags.nocacheHack = true;
1091        else
1092#endif
1093
1094            request->flags.noCache = true;
1095    }
1096
1097    /* ignore range header in non-GETs or non-HEADs */
1098    if (request->method == Http::METHOD_GET || request->method == Http::METHOD_HEAD) {
1099        // XXX: initialize if we got here without HttpRequest::parseHeader()
1100        if (!request->range)
1101            request->range = req_hdr->getRange();
1102
1103        if (request->range) {
1104            request->flags.isRanged = true;
1105            clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->data;
1106            /* XXX: This is suboptimal. We should give the stream the range set,
1107             * and thereby let the top of the stream set the offset when the
1108             * size becomes known. As it is, we will end up requesting from 0
1109             * for evey -X range specification.
1110             * RBC - this may be somewhat wrong. We should probably set the range
1111             * iter up at this point.
1112             */
1113            node->readBuffer.offset = request->range->lowestOffset(0);
1114            http->range_iter.pos = request->range->begin();
1115            http->range_iter.end = request->range->end();
1116            http->range_iter.valid = true;
1117        }
1118    }
1119
1120    /* Only HEAD and GET requests permit a Range or Request-Range header.
1121     * If these headers appear on any other type of request, delete them now.
1122     */
1123    else {
1124        req_hdr->delById(HDR_RANGE);
1125        req_hdr->delById(HDR_REQUEST_RANGE);
1126        request->ignoreRange("neither HEAD nor GET");
1127    }
1128
1129    if (req_hdr->has(HDR_AUTHORIZATION))
1130        request->flags.auth = true;
1131
1132    clientCheckPinning(http);
1133
1134    if (request->login[0] != '\0')
1135        request->flags.auth = true;
1136
1137    if (req_hdr->has(HDR_VIA)) {
1138        String s = req_hdr->getList(HDR_VIA);
1139        /*
1140         * ThisCache cannot be a member of Via header, "1.1 ThisCache" can.
1141         * Note ThisCache2 has a space prepended to the hostname so we don't
1142         * accidentally match super-domains.
1143         */
1144
1145        if (strListIsSubstr(&s, ThisCache2, ',')) {
1146            debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
1147                     request, (ObjPackMethod) & httpRequestPack);
1148            request->flags.loopDetected = true;
1149        }
1150
1151#if USE_FORW_VIA_DB
1152        fvdbCountVia(s.termedBuf());
1153
1154#endif
1155
1156        s.clean();
1157    }
1158
1159#if USE_FORW_VIA_DB
1160
1161    if (req_hdr->has(HDR_X_FORWARDED_FOR)) {
1162        String s = req_hdr->getList(HDR_X_FORWARDED_FOR);
1163        fvdbCountForw(s.termedBuf());
1164        s.clean();
1165    }
1166
1167#endif
1168
1169    request->flags.cachable = http->request->maybeCacheable();
1170
1171    if (clientHierarchical(http))
1172        request->flags.hierarchical = true;
1173
1174    debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
1175           (request->flags.noCache ? "SET" : "NOT SET"));
1176    debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
1177           (request->flags.cachable ? "SET" : "NOT SET"));
1178    debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
1179           (request->flags.hierarchical ? "SET" : "NOT SET"));
1180
1181}
1182
1183void
1184clientRedirectDoneWrapper(void *data, const Helper::Reply &result)
1185{
1186    ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1187
1188    if (!calloutContext->httpStateIsValid())
1189        return;
1190
1191    calloutContext->clientRedirectDone(result);
1192}
1193
1194void
1195clientStoreIdDoneWrapper(void *data, const Helper::Reply &result)
1196{
1197    ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1198
1199    if (!calloutContext->httpStateIsValid())
1200        return;
1201
1202    calloutContext->clientStoreIdDone(result);
1203}
1204
1205void
1206ClientRequestContext::clientRedirectDone(const Helper::Reply &reply)
1207{
1208    HttpRequest *old_request = http->request;
1209    debugs(85, 5, HERE << "'" << http->uri << "' result=" << reply);
1210    assert(redirect_state == REDIRECT_PENDING);
1211    redirect_state = REDIRECT_DONE;
1212
1213    // Put helper response Notes into the transaction state record (ALE) eventually
1214    // do it early to ensure that no matter what the outcome the notes are present.
1215    if (http->al != NULL)
1216        (void)SyncNotes(*http->al, *old_request);
1217
1218    UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1219
1220    switch (reply.result) {
1221    case Helper::Unknown:
1222    case Helper::TT:
1223        // Handler in redirect.cc should have already mapped Unknown
1224        // IF it contained valid entry for the old URL-rewrite helper protocol
1225        debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply);
1226        break;
1227
1228    case Helper::BrokenHelper:
1229        debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: " << reply << ", attempt #" << (redirect_fail_count+1) << " of 2");
1230        if (redirect_fail_count < 2) { // XXX: make this configurable ?
1231            ++redirect_fail_count;
1232            // reset state flag to try redirector again from scratch.
1233            redirect_done = false;
1234        }
1235        break;
1236
1237    case Helper::Error:
1238        // no change to be done.
1239        break;
1240
1241    case Helper::Okay: {
1242        // #1: redirect with a specific status code    OK status=NNN url="..."
1243        // #2: redirect with a default status code     OK url="..."
1244        // #3: re-write the URL                        OK rewrite-url="..."
1245
1246        const char *statusNote = reply.notes.findFirst("status");
1247        const char *urlNote = reply.notes.findFirst("url");
1248
1249        if (urlNote != NULL) {
1250            // HTTP protocol redirect to be done.
1251
1252            // TODO: change default redirect status for appropriate requests
1253            // Squid defaults to 302 status for now for better compatibility with old clients.
1254            // HTTP/1.0 client should get 302 (Http::scFound)
1255            // HTTP/1.1 client contacting reverse-proxy should get 307 (Http::scTemporaryRedirect)
1256            // HTTP/1.1 client being diverted by forward-proxy should get 303 (Http::scSeeOther)
1257            Http::StatusCode status = Http::scFound;
1258            if (statusNote != NULL) {
1259                const char * result = statusNote;
1260                status = static_cast<Http::StatusCode>(atoi(result));
1261            }
1262
1263            if (status == Http::scMovedPermanently
1264                    || status == Http::scFound
1265                    || status == Http::scSeeOther
1266                    || status == Http::scPermanentRedirect
1267                    || status == Http::scTemporaryRedirect) {
1268                http->redirect.status = status;
1269                http->redirect.location = xstrdup(urlNote);
1270                // TODO: validate the URL produced here is RFC 2616 compliant absolute URI
1271            } else {
1272                debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << urlNote);
1273            }
1274        } else {
1275            // URL-rewrite wanted. Ew.
1276            urlNote = reply.notes.findFirst("rewrite-url");
1277
1278            // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1279            if (urlNote != NULL && strcmp(urlNote, http->uri)) {
1280                // XXX: validate the URL properly *without* generating a whole new request object right here.
1281                // XXX: the clone() should be done only AFTER we know the new URL is valid.
1282                HttpRequest *new_request = old_request->clone();
1283                if (urlParse(old_request->method, const_cast<char*>(urlNote), new_request)) {
1284                    debugs(61,2, HERE << "URL-rewriter diverts URL from " << urlCanonical(old_request) << " to " << urlCanonical(new_request));
1285
1286                    // update the new request to flag the re-writing was done on it
1287                    new_request->flags.redirected = true;
1288
1289                    // unlink bodypipe from the old request. Not needed there any longer.
1290                    if (old_request->body_pipe != NULL) {
1291                        old_request->body_pipe = NULL;
1292                        debugs(61,2, HERE << "URL-rewriter diverts body_pipe " << new_request->body_pipe <<
1293                               " from request " << old_request << " to " << new_request);
1294                    }
1295
1296                    // update the current working ClientHttpRequest fields
1297                    safe_free(http->uri);
1298                    http->uri = xstrdup(urlCanonical(new_request));
1299                    HTTPMSGUNLOCK(old_request);
1300                    http->request = new_request;
1301                    HTTPMSGLOCK(http->request);
1302                } else {
1303                    debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid request: " <<
1304                           old_request->method << " " << urlNote << " " << old_request->http_ver);
1305                    delete new_request;
1306                }
1307            }
1308        }
1309    }
1310    break;
1311    }
1312
1313    /* FIXME PIPELINE: This is innacurate during pipelining */
1314
1315    if (http->getConn() != NULL && Comm::IsConnOpen(http->getConn()->clientConnection))
1316        fd_note(http->getConn()->clientConnection->fd, http->uri);
1317
1318    assert(http->uri);
1319
1320    http->doCallouts();
1321}
1322
1323/**
1324 * This method handles the different replies from StoreID helper.
1325 */
1326void
1327ClientRequestContext::clientStoreIdDone(const Helper::Reply &reply)
1328{
1329    HttpRequest *old_request = http->request;
1330    debugs(85, 5, "'" << http->uri << "' result=" << reply);
1331    assert(store_id_state == REDIRECT_PENDING);
1332    store_id_state = REDIRECT_DONE;
1333
1334    // Put helper response Notes into the transaction state record (ALE) eventually
1335    // do it early to ensure that no matter what the outcome the notes are present.
1336    if (http->al != NULL)
1337        (void)SyncNotes(*http->al, *old_request);
1338
1339    UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1340
1341    switch (reply.result) {
1342    case Helper::Unknown:
1343    case Helper::TT:
1344        // Handler in redirect.cc should have already mapped Unknown
1345        // IF it contained valid entry for the old helper protocol
1346        debugs(85, DBG_IMPORTANT, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply);
1347        break;
1348
1349    case Helper::BrokenHelper:
1350        debugs(85, DBG_IMPORTANT, "ERROR: storeID helper: " << reply << ", attempt #" << (store_id_fail_count+1) << " of 2");
1351        if (store_id_fail_count < 2) { // XXX: make this configurable ?
1352            ++store_id_fail_count;
1353            // reset state flag to try StoreID again from scratch.
1354            store_id_done = false;
1355        }
1356        break;
1357
1358    case Helper::Error:
1359        // no change to be done.
1360        break;
1361
1362    case Helper::Okay: {
1363        const char *urlNote = reply.notes.findFirst("store-id");
1364
1365        // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1366        if (urlNote != NULL && strcmp(urlNote, http->uri) ) {
1367            // Debug section required for some very specific cases.
1368            debugs(85, 9, "Setting storeID with: " << urlNote );
1369            http->request->store_id = urlNote;
1370            http->store_id = urlNote;
1371        }
1372    }
1373    break;
1374    }
1375
1376    http->doCallouts();
1377}
1378
1379/** Test cache allow/deny configuration
1380 *  Sets flags.cachable=1 if caching is not denied.
1381 */
1382void
1383ClientRequestContext::checkNoCache()
1384{
1385    if (Config.accessList.noCache) {
1386        acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http);
1387        acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this);
1388    } else {
1389        /* unless otherwise specified, we try to cache. */
1390        checkNoCacheDone(ACCESS_ALLOWED);
1391    }
1392}
1393
1394static void
1395checkNoCacheDoneWrapper(allow_t answer, void *data)
1396{
1397    ClientRequestContext *calloutContext = (ClientRequestContext *) data;
1398
1399    if (!calloutContext->httpStateIsValid())
1400        return;
1401
1402    calloutContext->checkNoCacheDone(answer);
1403}
1404
1405void
1406ClientRequestContext::checkNoCacheDone(const allow_t &answer)
1407{
1408    acl_checklist = NULL;
1409    http->request->flags.cachable = (answer == ACCESS_ALLOWED);
1410    http->doCallouts();
1411}
1412
1413#if USE_OPENSSL
1414bool
1415ClientRequestContext::sslBumpAccessCheck()
1416{
1417    // If SSL connection tunneling or bumping decision has been made, obey it.
1418    const Ssl::BumpMode bumpMode = http->getConn()->sslBumpMode;
1419    if (bumpMode != Ssl::bumpEnd) {
1420        debugs(85, 5, HERE << "SslBump already decided (" << bumpMode <<
1421               "), " << "ignoring ssl_bump for " << http->getConn());
1422        if (!http->getConn()->serverBump())
1423            http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed and not already bumped
1424        http->al->ssl.bumpMode = bumpMode; // inherited from bumped connection
1425        return false;
1426    }
1427
1428    // If we have not decided yet, decide whether to bump now.
1429
1430    // Bumping here can only start with a CONNECT request on a bumping port
1431    // (bumping of intercepted SSL conns is decided before we get 1st request).
1432    // We also do not bump redirected CONNECT requests.
1433    if (http->request->method != Http::METHOD_CONNECT || http->redirect.status ||
1434            !Config.accessList.ssl_bump ||
1435            !http->getConn()->port->flags.tunnelSslBumping) {
1436        http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1437        debugs(85, 5, HERE << "cannot SslBump this request");
1438        return false;
1439    }
1440
1441    // Do not bump during authentication: clients would not proxy-authenticate
1442    // if we delay a 407 response and respond with 200 OK to CONNECT.
1443    if (error && error->httpStatus == Http::scProxyAuthenticationRequired) {
1444        http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1445        debugs(85, 5, HERE << "no SslBump during proxy authentication");
1446        return false;
1447    }
1448
1449    debugs(85, 5, HERE << "SslBump possible, checking ACL");
1450
1451    ACLFilledChecklist *aclChecklist = clientAclChecklistCreate(Config.accessList.ssl_bump, http);
1452    aclChecklist->nonBlockingCheck(sslBumpAccessCheckDoneWrapper, this);
1453    return true;
1454}
1455
1456/**
1457 * A wrapper function to use the ClientRequestContext::sslBumpAccessCheckDone method
1458 * as ACLFilledChecklist callback
1459 */
1460static void
1461sslBumpAccessCheckDoneWrapper(allow_t answer, void *data)
1462{
1463    ClientRequestContext *calloutContext = static_cast<ClientRequestContext *>(data);
1464
1465    if (!calloutContext->httpStateIsValid())
1466        return;
1467    calloutContext->sslBumpAccessCheckDone(answer);
1468}
1469
1470void
1471ClientRequestContext::sslBumpAccessCheckDone(const allow_t &answer)
1472{
1473    if (!httpStateIsValid())
1474        return;
1475
1476    const Ssl::BumpMode bumpMode = answer == ACCESS_ALLOWED ?
1477                                   static_cast<Ssl::BumpMode>(answer.kind) : Ssl::bumpNone;
1478    http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed
1479    http->al->ssl.bumpMode = bumpMode; // for logging
1480
1481    http->doCallouts();
1482}
1483#endif
1484
1485/*
1486 * Identify requests that do not go through the store and client side stream
1487 * and forward them to the appropriate location. All other requests, request
1488 * them.
1489 */
1490void
1491ClientHttpRequest::processRequest()
1492{
1493    debugs(85, 4, request->method << ' ' << uri);
1494
1495    if (request->method == Http::METHOD_CONNECT && !redirect.status) {
1496#if USE_OPENSSL
1497        if (sslBumpNeeded()) {
1498            sslBumpStart();
1499            return;
1500        }
1501#endif
1502        getConn()->stopReading(); // tunnels read for themselves
1503        tunnelStart(this, &out.size, &al->http.code, al);
1504        return;
1505    }
1506
1507    httpStart();
1508}
1509
1510void
1511ClientHttpRequest::httpStart()
1512{
1513    PROF_start(httpStart);
1514    logType = LOG_TAG_NONE;
1515    debugs(85, 4, LogTags_str[logType] << " for '" << uri << "'");
1516
1517    /* no one should have touched this */
1518    assert(out.offset == 0);
1519    /* Use the Stream Luke */
1520    clientStreamNode *node = (clientStreamNode *)client_stream.tail->data;
1521    clientStreamRead(node, this, node->readBuffer);
1522    PROF_stop(httpStart);
1523}
1524
1525#if USE_OPENSSL
1526
1527void
1528ClientHttpRequest::sslBumpNeed(Ssl::BumpMode mode)
1529{
1530    debugs(83, 3, HERE << "sslBump required: "<< Ssl::bumpMode(mode));
1531    sslBumpNeed_ = mode;
1532}
1533
1534// called when comm_write has completed
1535static void
1536SslBumpEstablish(const Comm::ConnectionPointer &, char *, size_t, Comm::Flag errflag, int, void *data)
1537{
1538    ClientHttpRequest *r = static_cast<ClientHttpRequest*>(data);
1539    debugs(85, 5, HERE << "responded to CONNECT: " << r << " ? " << errflag);
1540
1541    assert(r && cbdataReferenceValid(r));
1542    r->sslBumpEstablish(errflag);
1543}
1544
1545void
1546ClientHttpRequest::sslBumpEstablish(Comm::Flag errflag)
1547{
1548    // Bail out quickly on Comm::ERR_CLOSING - close handlers will tidy up
1549    if (errflag == Comm::ERR_CLOSING)
1550        return;
1551
1552    if (errflag) {
1553        debugs(85, 3, HERE << "CONNECT response failure in SslBump: " << errflag);
1554        getConn()->clientConnection->close();
1555        return;
1556    }
1557
1558    // We lack HttpReply which logRequest() uses to log the status code.
1559    // TODO: Use HttpReply instead of the "200 Connection established" string.
1560    al->http.code = 200;
1561
1562#if USE_AUTH
1563    // Preserve authentication info for the ssl-bumped request
1564    if (request->auth_user_request != NULL)
1565        getConn()->setAuth(request->auth_user_request, "SSL-bumped CONNECT");
1566#endif
1567
1568    assert(sslBumpNeeded());
1569    getConn()->switchToHttps(request, sslBumpNeed_);
1570}
1571
1572void
1573ClientHttpRequest::sslBumpStart()
1574{
1575    debugs(85, 5, HERE << "Confirming " << Ssl::bumpMode(sslBumpNeed_) <<
1576           "-bumped CONNECT tunnel on FD " << getConn()->clientConnection);
1577    getConn()->sslBumpMode = sslBumpNeed_;
1578
1579    AsyncCall::Pointer bumpCall = commCbCall(85, 5, "ClientSocketContext::sslBumpEstablish",
1580                                  CommIoCbPtrFun(&SslBumpEstablish, this));
1581
1582    if (request->flags.interceptTproxy || request->flags.intercepted) {
1583        CommIoCbParams &params = GetCommParams<CommIoCbParams>(bumpCall);
1584        params.flag = Comm::OK;
1585        params.conn = getConn()->clientConnection;
1586        ScheduleCallHere(bumpCall);
1587        return;
1588    }
1589
1590    // send an HTTP 200 response to kick client SSL negotiation
1591    // TODO: Unify with tunnel.cc and add a Server(?) header
1592    static const char *const conn_established = "HTTP/1.1 200 Connection established\r\n\r\n";
1593    Comm::Write(getConn()->clientConnection, conn_established, strlen(conn_established), bumpCall, NULL);
1594}
1595
1596#endif
1597
1598bool
1599ClientHttpRequest::gotEnough() const
1600{
1601    /** TODO: should be querying the stream. */
1602    int64_t contentLength =
1603        memObject()->getReply()->bodySize(request->method);
1604    assert(contentLength >= 0);
1605
1606    if (out.offset < contentLength)
1607        return false;
1608
1609    return true;
1610}
1611
1612void
1613ClientHttpRequest::storeEntry(StoreEntry *newEntry)
1614{
1615    entry_ = newEntry;
1616}
1617
1618void
1619ClientHttpRequest::loggingEntry(StoreEntry *newEntry)
1620{
1621    if (loggingEntry_)
1622        loggingEntry_->unlock("ClientHttpRequest::loggingEntry");
1623
1624    loggingEntry_ = newEntry;
1625
1626    if (loggingEntry_)
1627        loggingEntry_->lock("ClientHttpRequest::loggingEntry");
1628}
1629
1630/*
1631 * doCallouts() - This function controls the order of "callout"
1632 * executions, including non-blocking access control checks, the
1633 * redirector, and ICAP.  Previously, these callouts were chained
1634 * together such that "clientAccessCheckDone()" would call
1635 * "clientRedirectStart()" and so on.
1636 *
1637 * The ClientRequestContext (aka calloutContext) class holds certain
1638 * state data for the callout/callback operations.  Previously
1639 * ClientHttpRequest would sort of hand off control to ClientRequestContext
1640 * for a short time.  ClientRequestContext would then delete itself
1641 * and pass control back to ClientHttpRequest when all callouts
1642 * were finished.
1643 *
1644 * This caused some problems for ICAP because we want to make the
1645 * ICAP callout after checking ACLs, but before checking the no_cache
1646 * list.  We can't stuff the ICAP state into the ClientRequestContext
1647 * class because we still need the ICAP state after ClientRequestContext
1648 * goes away.
1649 *
1650 * Note that ClientRequestContext is created before the first call
1651 * to doCallouts().
1652 *
1653 * If one of the callouts notices that ClientHttpRequest is no
1654 * longer valid, it should call cbdataReferenceDone() so that
1655 * ClientHttpRequest's reference count goes to zero and it will get
1656 * deleted.  ClientHttpRequest will then delete ClientRequestContext.
1657 *
1658 * Note that we set the _done flags here before actually starting
1659 * the callout.  This is strictly for convenience.
1660 */
1661
1662tos_t aclMapTOS (acl_tos * head, ACLChecklist * ch);
1663nfmark_t aclMapNfmark (acl_nfmark * head, ACLChecklist * ch);
1664
1665void
1666ClientHttpRequest::doCallouts()
1667{
1668    assert(calloutContext);
1669
1670    /*Save the original request for logging purposes*/
1671    if (!calloutContext->http->al->request) {
1672        calloutContext->http->al->request = request;
1673        HTTPMSGLOCK(calloutContext->http->al->request);
1674
1675        NotePairs &notes = SyncNotes(*calloutContext->http->al, *calloutContext->http->request);
1676        // Make the previously set client connection ID available as annotation.
1677        if (ConnStateData *csd = calloutContext->http->getConn()) {
1678            if (!csd->connectionTag().isEmpty())
1679                notes.add("clt_conn_tag", SBuf(csd->connectionTag()).c_str());
1680        }
1681    }
1682
1683    if (!calloutContext->error) {
1684        // CVE-2009-0801: verify the Host: header is consistent with other known details.
1685        if (!calloutContext->host_header_verify_done) {
1686            debugs(83, 3, HERE << "Doing calloutContext->hostHeaderVerify()");
1687            calloutContext->host_header_verify_done = true;
1688            calloutContext->hostHeaderVerify();
1689            return;
1690        }
1691
1692        if (!calloutContext->http_access_done) {
1693            debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck()");
1694            calloutContext->http_access_done = true;
1695            calloutContext->clientAccessCheck();
1696            return;
1697        }
1698
1699#if USE_ADAPTATION
1700        if (!calloutContext->adaptation_acl_check_done) {
1701            calloutContext->adaptation_acl_check_done = true;
1702            if (Adaptation::AccessCheck::Start(
1703                        Adaptation::methodReqmod, Adaptation::pointPreCache,
1704                        request, NULL, calloutContext->http->al, this))
1705                return; // will call callback
1706        }
1707#endif
1708
1709        if (!calloutContext->redirect_done) {
1710            calloutContext->redirect_done = true;
1711
1712            if (Config.Program.redirect) {
1713                debugs(83, 3, HERE << "Doing calloutContext->clientRedirectStart()");
1714                calloutContext->redirect_state = REDIRECT_PENDING;
1715                calloutContext->clientRedirectStart();
1716                return;
1717            }
1718        }
1719
1720        if (!calloutContext->adapted_http_access_done) {
1721            debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck2()");
1722            calloutContext->adapted_http_access_done = true;
1723            calloutContext->clientAccessCheck2();
1724            return;
1725        }
1726
1727        if (!calloutContext->store_id_done) {
1728            calloutContext->store_id_done = true;
1729
1730            if (Config.Program.store_id) {
1731                debugs(83, 3,"Doing calloutContext->clientStoreIdStart()");
1732                calloutContext->store_id_state = REDIRECT_PENDING;
1733                calloutContext->clientStoreIdStart();
1734                return;
1735            }
1736        }
1737
1738        if (!calloutContext->interpreted_req_hdrs) {
1739            debugs(83, 3, HERE << "Doing clientInterpretRequestHeaders()");
1740            calloutContext->interpreted_req_hdrs = 1;
1741            clientInterpretRequestHeaders(this);
1742        }
1743
1744        if (!calloutContext->no_cache_done) {
1745            calloutContext->no_cache_done = true;
1746
1747            if (Config.accessList.noCache && request->flags.cachable) {
1748                debugs(83, 3, HERE << "Doing calloutContext->checkNoCache()");
1749                calloutContext->checkNoCache();
1750                return;
1751            }
1752        }
1753    } //  if !calloutContext->error
1754
1755    if (!calloutContext->tosToClientDone) {
1756        calloutContext->tosToClientDone = true;
1757        if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1758            ACLFilledChecklist ch(NULL, request, NULL);
1759            ch.src_addr = request->client_addr;
1760            ch.my_addr = request->my_addr;
1761            tos_t tos = aclMapTOS(Ip::Qos::TheConfig.tosToClient, &ch);
1762            if (tos)
1763                Ip::Qos::setSockTos(getConn()->clientConnection, tos);
1764        }
1765    }
1766
1767    if (!calloutContext->nfmarkToClientDone) {
1768        calloutContext->nfmarkToClientDone = true;
1769        if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1770            ACLFilledChecklist ch(NULL, request, NULL);
1771            ch.src_addr = request->client_addr;
1772            ch.my_addr = request->my_addr;
1773            nfmark_t mark = aclMapNfmark(Ip::Qos::TheConfig.nfmarkToClient, &ch);
1774            if (mark)
1775                Ip::Qos::setSockNfmark(getConn()->clientConnection, mark);
1776        }
1777    }
1778
1779#if USE_OPENSSL
1780    // We need to check for SslBump even if the calloutContext->error is set
1781    // because bumping may require delaying the error until after CONNECT.
1782    if (!calloutContext->sslBumpCheckDone) {
1783        calloutContext->sslBumpCheckDone = true;
1784        if (calloutContext->sslBumpAccessCheck())
1785            return;
1786        /* else no ssl bump required*/
1787    }
1788#endif
1789
1790    if (calloutContext->error) {
1791        const char *storeUri = request->storeId();
1792        StoreEntry *e= storeCreateEntry(storeUri, storeUri, request->flags, request->method);
1793#if USE_OPENSSL
1794        if (sslBumpNeeded()) {
1795            // We have to serve an error, so bump the client first.
1796            sslBumpNeed(Ssl::bumpClientFirst);
1797            // set final error but delay sending until we bump
1798            Ssl::ServerBump *srvBump = new Ssl::ServerBump(request, e);
1799            errorAppendEntry(e, calloutContext->error);
1800            calloutContext->error = NULL;
1801            getConn()->setServerBump(srvBump);
1802            e->unlock("ClientHttpRequest::doCallouts+sslBumpNeeded");
1803        } else
1804#endif
1805        {
1806            // send the error to the client now
1807            clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1808            clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1809            assert (repContext);
1810            repContext->setReplyToStoreEntry(e, "immediate SslBump error");
1811            errorAppendEntry(e, calloutContext->error);
1812            calloutContext->error = NULL;
1813            if (calloutContext->readNextRequest)
1814                getConn()->flags.readMore = true; // resume any pipeline reads.
1815            node = (clientStreamNode *)client_stream.tail->data;
1816            clientStreamRead(node, this, node->readBuffer);
1817            e->unlock("ClientHttpRequest::doCallouts-sslBumpNeeded");
1818            return;
1819        }
1820    }
1821
1822    cbdataReferenceDone(calloutContext->http);
1823    delete calloutContext;
1824    calloutContext = NULL;
1825#if HEADERS_LOG
1826
1827    headersLog(0, 1, request->method, request);
1828#endif
1829
1830    debugs(83, 3, HERE << "calling processRequest()");
1831    processRequest();
1832
1833#if ICAP_CLIENT
1834    Adaptation::Icap::History::Pointer ih = request->icapHistory();
1835    if (ih != NULL)
1836        ih->logType = logType;
1837#endif
1838}
1839
1840#if !_USE_INLINE_
1841#include "client_side_request.cci"
1842#endif
1843
1844#if USE_ADAPTATION
1845/// Initiate an asynchronous adaptation transaction which will call us back.
1846void
1847ClientHttpRequest::startAdaptation(const Adaptation::ServiceGroupPointer &g)
1848{
1849    debugs(85, 3, HERE << "adaptation needed for " << this);
1850    assert(!virginHeadSource);
1851    assert(!adaptedBodySource);
1852    virginHeadSource = initiateAdaptation(
1853                           new Adaptation::Iterator(request, NULL, al, g));
1854
1855    // we could try to guess whether we can bypass this adaptation
1856    // initiation failure, but it should not really happen
1857    Must(initiated(virginHeadSource));
1858}
1859
1860void
1861ClientHttpRequest::noteAdaptationAnswer(const Adaptation::Answer &answer)
1862{
1863    assert(cbdataReferenceValid(this));     // indicates bug
1864    clearAdaptation(virginHeadSource);
1865    assert(!adaptedBodySource);
1866
1867    switch (answer.kind) {
1868    case Adaptation::Answer::akForward:
1869        handleAdaptedHeader(const_cast<HttpMsg*>(answer.message.getRaw()));
1870        break;
1871
1872    case Adaptation::Answer::akBlock:
1873        handleAdaptationBlock(answer);
1874        break;
1875
1876    case Adaptation::Answer::akError:
1877        handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_ABORT, !answer.final);
1878        break;
1879    }
1880}
1881
1882void
1883ClientHttpRequest::handleAdaptedHeader(HttpMsg *msg)
1884{
1885    assert(msg);
1886
1887    if (HttpRequest *new_req = dynamic_cast<HttpRequest*>(msg)) {
1888        /*
1889         * Replace the old request with the new request.
1890         */
1891        HTTPMSGUNLOCK(request);
1892        request = new_req;
1893        HTTPMSGLOCK(request);
1894
1895        // update the new message to flag whether URL re-writing was done on it
1896        if (strcmp(urlCanonical(request),uri) != 0)
1897            request->flags.redirected = 1;
1898
1899        /*
1900         * Store the new URI for logging
1901         */
1902        xfree(uri);
1903        uri = xstrdup(urlCanonical(request));
1904        setLogUri(this, urlCanonicalClean(request));
1905        assert(request->method.id());
1906    } else if (HttpReply *new_rep = dynamic_cast<HttpReply*>(msg)) {
1907        debugs(85,3,HERE << "REQMOD reply is HTTP reply");
1908
1909        // subscribe to receive reply body
1910        if (new_rep->body_pipe != NULL) {
1911            adaptedBodySource = new_rep->body_pipe;
1912            int consumer_ok = adaptedBodySource->setConsumerIfNotLate(this);
1913            assert(consumer_ok);
1914        }
1915
1916        clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1917        clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1918        assert(repContext);
1919        repContext->createStoreEntry(request->method, request->flags);
1920
1921        EBIT_CLR(storeEntry()->flags, ENTRY_FWD_HDR_WAIT);
1922        request_satisfaction_mode = true;
1923        request_satisfaction_offset = 0;
1924        storeEntry()->replaceHttpReply(new_rep);
1925        storeEntry()->timestampsSet();
1926
1927        if (!adaptedBodySource) // no body
1928            storeEntry()->complete();
1929        clientGetMoreData(node, this);
1930    }
1931
1932    // we are done with getting headers (but may be receiving body)
1933    clearAdaptation(virginHeadSource);
1934
1935    if (!request_satisfaction_mode)
1936        doCallouts();
1937}
1938
1939void
1940ClientHttpRequest::handleAdaptationBlock(const Adaptation::Answer &answer)
1941{
1942    request->detailError(ERR_ACCESS_DENIED, ERR_DETAIL_REQMOD_BLOCK);
1943    AclMatchedName = answer.ruleId.termedBuf();
1944    assert(calloutContext);
1945    calloutContext->clientAccessCheckDone(ACCESS_DENIED);
1946    AclMatchedName = NULL;
1947}
1948
1949void
1950ClientHttpRequest::resumeBodyStorage()
1951{
1952    if (!adaptedBodySource)
1953        return;
1954
1955    noteMoreBodyDataAvailable(adaptedBodySource);
1956}
1957
1958void
1959ClientHttpRequest::noteMoreBodyDataAvailable(BodyPipe::Pointer)
1960{
1961    assert(request_satisfaction_mode);
1962    assert(adaptedBodySource != NULL);
1963
1964    if (size_t contentSize = adaptedBodySource->buf().contentSize()) {
1965        const size_t spaceAvailable = storeEntry()->bytesWanted(Range<size_t>(0,contentSize));
1966
1967        if (spaceAvailable < contentSize ) {
1968            // No or partial body data consuming
1969            typedef NullaryMemFunT<ClientHttpRequest> Dialer;
1970            AsyncCall::Pointer call = asyncCall(93, 5, "ClientHttpRequest::resumeBodyStorage",
1971                                                Dialer(this, &ClientHttpRequest::resumeBodyStorage));
1972            storeEntry()->deferProducer(call);
1973        }
1974
1975        if (!spaceAvailable)
1976            return;
1977
1978        if (spaceAvailable < contentSize )
1979            contentSize = spaceAvailable;
1980
1981        BodyPipeCheckout bpc(*adaptedBodySource);
1982        const StoreIOBuffer ioBuf(&bpc.buf, request_satisfaction_offset, contentSize);
1983        storeEntry()->write(ioBuf);
1984        // assume StoreEntry::write() writes the entire ioBuf
1985        request_satisfaction_offset += ioBuf.length;
1986        bpc.buf.consume(contentSize);
1987        bpc.checkIn();
1988    }
1989
1990    if (adaptedBodySource->exhausted())
1991        endRequestSatisfaction();
1992    // else wait for more body data
1993}
1994
1995void
1996ClientHttpRequest::noteBodyProductionEnded(BodyPipe::Pointer)
1997{
1998    assert(!virginHeadSource);
1999    // should we end request satisfaction now?
2000    if (adaptedBodySource != NULL && adaptedBodySource->exhausted())
2001        endRequestSatisfaction();
2002}
2003
2004void
2005ClientHttpRequest::endRequestSatisfaction()
2006{
2007    debugs(85,4, HERE << this << " ends request satisfaction");
2008    assert(request_satisfaction_mode);
2009    stopConsumingFrom(adaptedBodySource);
2010
2011    // TODO: anything else needed to end store entry formation correctly?
2012    storeEntry()->complete();
2013}
2014
2015void
2016ClientHttpRequest::noteBodyProducerAborted(BodyPipe::Pointer)
2017{
2018    assert(!virginHeadSource);
2019    stopConsumingFrom(adaptedBodySource);
2020
2021    debugs(85,3, HERE << "REQMOD body production failed");
2022    if (request_satisfaction_mode) { // too late to recover or serve an error
2023        request->detailError(ERR_ICAP_FAILURE, ERR_DETAIL_CLT_REQMOD_RESP_BODY);
2024        const Comm::ConnectionPointer c = getConn()->clientConnection;
2025        Must(Comm::IsConnOpen(c));
2026        c->close(); // drastic, but we may be writing a response already
2027    } else {
2028        handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_REQ_BODY);
2029    }
2030}
2031
2032void
2033ClientHttpRequest::handleAdaptationFailure(int errDetail, bool bypassable)
2034{
2035    debugs(85,3, HERE << "handleAdaptationFailure(" << bypassable << ")");
2036
2037    const bool usedStore = storeEntry() && !storeEntry()->isEmpty();
2038    const bool usedPipe = request->body_pipe != NULL &&
2039                          request->body_pipe->consumedSize() > 0;
2040
2041    if (bypassable && !usedStore && !usedPipe) {
2042        debugs(85,3, HERE << "ICAP REQMOD callout failed, bypassing: " << calloutContext);
2043        if (calloutContext)
2044            doCallouts();
2045        return;
2046    }
2047
2048    debugs(85,3, HERE << "ICAP REQMOD callout failed, responding with error");
2049
2050    clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
2051    clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
2052    assert(repContext);
2053
2054    // The original author of the code also wanted to pass an errno to
2055    // setReplyToError, but it seems unlikely that the errno reflects the
2056    // true cause of the error at this point, so I did not pass it.
2057    if (calloutContext) {
2058        Ip::Address noAddr;
2059        noAddr.setNoAddr();
2060        ConnStateData * c = getConn();
2061        calloutContext->error = clientBuildError(ERR_ICAP_FAILURE, Http::scInternalServerError,
2062                                NULL,
2063                                c != NULL ? c->clientConnection->remote : noAddr,
2064                                request
2065                                                );
2066#if USE_AUTH
2067        calloutContext->error->auth_user_request =
2068            c != NULL && c->getAuth() != NULL ? c->getAuth() : request->auth_user_request;
2069#endif
2070        calloutContext->error->detailError(errDetail);
2071        calloutContext->readNextRequest = true;
2072        if (c != NULL)
2073            c->expectNoForwarding();
2074        doCallouts();
2075    }
2076    //else if(calloutContext == NULL) is it possible?
2077}
2078
2079#endif
2080
Note: See TracBrowser for help on using the repository browser.