source: squid-ssl/trunk/fuentes/src/client_side_request.cc @ 5496

Last change on this file since 5496 was 5496, checked in by Juanma, 21 months ago

Initial release

File size: 71.2 KB
Line 
1/*
2 * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
3 *
4 * Squid software is distributed under GPLv2+ license and includes
5 * contributions from numerous individuals and organizations.
6 * Please see the COPYING and CONTRIBUTORS files for details.
7 */
8
9/* DEBUG: section 85    Client-side Request Routines */
10
11/*
12 * General logic of request processing:
13 *
14 * We run a series of tests to determine if access will be permitted, and to do
15 * any redirection. Then we call into the result clientStream to retrieve data.
16 * From that point on it's up to reply management.
17 */
18
19#include "squid.h"
20#include "acl/FilledChecklist.h"
21#include "acl/Gadgets.h"
22#include "anyp/PortCfg.h"
23#include "base/AsyncJobCalls.h"
24#include "client_side.h"
25#include "client_side_reply.h"
26#include "client_side_request.h"
27#include "ClientRequestContext.h"
28#include "clientStream.h"
29#include "comm/Connection.h"
30#include "comm/Write.h"
31#include "err_detail_type.h"
32#include "errorpage.h"
33#include "fd.h"
34#include "fde.h"
35#include "format/Token.h"
36#include "gopher.h"
37#include "helper.h"
38#include "helper/Reply.h"
39#include "http.h"
40#include "HttpHdrCc.h"
41#include "HttpReply.h"
42#include "HttpRequest.h"
43#include "ip/QosConfig.h"
44#include "ipcache.h"
45#include "log/access_log.h"
46#include "MemObject.h"
47#include "Parsing.h"
48#include "profiler/Profiler.h"
49#include "redirect.h"
50#include "SquidConfig.h"
51#include "SquidTime.h"
52#include "Store.h"
53#include "StrList.h"
54#include "tools.h"
55#include "URL.h"
56#include "wordlist.h"
57#if USE_AUTH
58#include "auth/UserRequest.h"
59#endif
60#if USE_ADAPTATION
61#include "adaptation/AccessCheck.h"
62#include "adaptation/Answer.h"
63#include "adaptation/Iterator.h"
64#include "adaptation/Service.h"
65#if ICAP_CLIENT
66#include "adaptation/icap/History.h"
67#endif
68#endif
69#if USE_OPENSSL
70#include "ssl/ServerBump.h"
71#include "ssl/support.h"
72#endif
73
74#if LINGERING_CLOSE
75#define comm_close comm_lingering_close
76#endif
77
78static const char *const crlf = "\r\n";
79
80#if FOLLOW_X_FORWARDED_FOR
81static void clientFollowXForwardedForCheck(allow_t answer, void *data);
82#endif /* FOLLOW_X_FORWARDED_FOR */
83
84ErrorState *clientBuildError(err_type, Http::StatusCode, char const *url, Ip::Address &, HttpRequest *);
85
86CBDATA_CLASS_INIT(ClientRequestContext);
87
88/* Local functions */
89/* other */
90static void clientAccessCheckDoneWrapper(allow_t, void *);
91#if USE_OPENSSL
92static void sslBumpAccessCheckDoneWrapper(allow_t, void *);
93#endif
94static int clientHierarchical(ClientHttpRequest * http);
95static void clientInterpretRequestHeaders(ClientHttpRequest * http);
96static HLPCB clientRedirectDoneWrapper;
97static HLPCB clientStoreIdDoneWrapper;
98static void checkNoCacheDoneWrapper(allow_t, void *);
99SQUIDCEXTERN CSR clientGetMoreData;
100SQUIDCEXTERN CSS clientReplyStatus;
101SQUIDCEXTERN CSD clientReplyDetach;
102static void checkFailureRatio(err_type, hier_code);
103
104ClientRequestContext::~ClientRequestContext()
105{
106    /*
107     * Release our "lock" on our parent, ClientHttpRequest, if we
108     * still have one
109     */
110
111    if (http)
112        cbdataReferenceDone(http);
113
114    delete error;
115    debugs(85,3, HERE << this << " ClientRequestContext destructed");
116}
117
118ClientRequestContext::ClientRequestContext(ClientHttpRequest *anHttp) : http(cbdataReference(anHttp)), acl_checklist (NULL), redirect_state (REDIRECT_NONE), store_id_state(REDIRECT_NONE),error(NULL), readNextRequest(false)
119{
120    http_access_done = false;
121    redirect_done = false;
122    redirect_fail_count = 0;
123    store_id_done = false;
124    store_id_fail_count = 0;
125    no_cache_done = false;
126    interpreted_req_hdrs = false;
127#if USE_OPENSSL
128    sslBumpCheckDone = false;
129#endif
130    debugs(85,3, HERE << this << " ClientRequestContext constructed");
131}
132
133CBDATA_CLASS_INIT(ClientHttpRequest);
134
135ClientHttpRequest::ClientHttpRequest(ConnStateData * aConn) :
136#if USE_ADAPTATION
137    AsyncJob("ClientHttpRequest"),
138#endif
139    loggingEntry_(NULL)
140{
141    setConn(aConn);
142    al = new AccessLogEntry;
143    al->cache.start_time = current_time;
144    al->tcpClient = clientConnection = aConn->clientConnection;
145    al->cache.port = aConn->port;
146    al->cache.caddr = aConn->log_addr;
147
148#if USE_OPENSSL
149    if (aConn->clientConnection != NULL && aConn->clientConnection->isOpen()) {
150        if (SSL *ssl = fd_table[aConn->clientConnection->fd].ssl)
151            al->cache.sslClientCert.reset(SSL_get_peer_certificate(ssl));
152    }
153#endif
154    dlinkAdd(this, &active, &ClientActiveRequests);
155#if USE_ADAPTATION
156    request_satisfaction_mode = false;
157#endif
158#if USE_OPENSSL
159    sslBumpNeed_ = Ssl::bumpEnd;
160#endif
161}
162
163/*
164 * returns true if client specified that the object must come from the cache
165 * without contacting origin server
166 */
167bool
168ClientHttpRequest::onlyIfCached()const
169{
170    assert(request);
171    return request->cache_control &&
172           request->cache_control->onlyIfCached();
173}
174
175/**
176 * This function is designed to serve a fairly specific purpose.
177 * Occasionally our vBNS-connected caches can talk to each other, but not
178 * the rest of the world.  Here we try to detect frequent failures which
179 * make the cache unusable (e.g. DNS lookup and connect() failures).  If
180 * the failure:success ratio goes above 1.0 then we go into "hit only"
181 * mode where we only return UDP_HIT or UDP_MISS_NOFETCH.  Neighbors
182 * will only fetch HITs from us if they are using the ICP protocol.  We
183 * stay in this mode for 5 minutes.
184 *
185 * Duane W., Sept 16, 1996
186 */
187static void
188checkFailureRatio(err_type etype, hier_code hcode)
189{
190    // Can be set at compile time with -D compiler flag
191#ifndef FAILURE_MODE_TIME
192#define FAILURE_MODE_TIME 300
193#endif
194
195    if (hcode == HIER_NONE)
196        return;
197
198    // don't bother when ICP is disabled.
199    if (Config.Port.icp <= 0)
200        return;
201
202    static double magic_factor = 100.0;
203    double n_good;
204    double n_bad;
205
206    n_good = magic_factor / (1.0 + request_failure_ratio);
207
208    n_bad = magic_factor - n_good;
209
210    switch (etype) {
211
212    case ERR_DNS_FAIL:
213
214    case ERR_CONNECT_FAIL:
215    case ERR_SECURE_CONNECT_FAIL:
216
217    case ERR_READ_ERROR:
218        ++n_bad;
219        break;
220
221    default:
222        ++n_good;
223    }
224
225    request_failure_ratio = n_bad / n_good;
226
227    if (hit_only_mode_until > squid_curtime)
228        return;
229
230    if (request_failure_ratio < 1.0)
231        return;
232
233    debugs(33, DBG_CRITICAL, "WARNING: Failure Ratio at "<< std::setw(4)<<
234           std::setprecision(3) << request_failure_ratio);
235
236    debugs(33, DBG_CRITICAL, "WARNING: ICP going into HIT-only mode for " <<
237           FAILURE_MODE_TIME / 60 << " minutes...");
238
239    hit_only_mode_until = squid_curtime + FAILURE_MODE_TIME;
240
241    request_failure_ratio = 0.8;    /* reset to something less than 1.0 */
242}
243
244ClientHttpRequest::~ClientHttpRequest()
245{
246    debugs(33, 3, "httpRequestFree: " << uri);
247    PROF_start(httpRequestFree);
248
249    // Even though freeResources() below may destroy the request,
250    // we no longer set request->body_pipe to NULL here
251    // because we did not initiate that pipe (ConnStateData did)
252
253    /* the ICP check here was erroneous
254     * - StoreEntry::releaseRequest was always called if entry was valid
255     */
256    assert(logType < LOG_TYPE_MAX);
257
258    logRequest();
259
260    loggingEntry(NULL);
261
262    if (request)
263        checkFailureRatio(request->errType, al->hier.code);
264
265    freeResources();
266
267#if USE_ADAPTATION
268    announceInitiatorAbort(virginHeadSource);
269
270    if (adaptedBodySource != NULL)
271        stopConsumingFrom(adaptedBodySource);
272#endif
273
274    if (calloutContext)
275        delete calloutContext;
276
277    clientConnection = NULL;
278
279    if (conn_)
280        cbdataReferenceDone(conn_);
281
282    /* moving to the next connection is handled by the context free */
283    dlinkDelete(&active, &ClientActiveRequests);
284
285    PROF_stop(httpRequestFree);
286}
287
288/**
289 * Create a request and kick it off
290 *
291 * \retval 0     success
292 * \retval -1    failure
293 *
294 * TODO: Pass in the buffers to be used in the inital Read request, as they are
295 * determined by the user
296 */
297int
298clientBeginRequest(const HttpRequestMethod& method, char const *url, CSCB * streamcallback,
299                   CSD * streamdetach, ClientStreamData streamdata, HttpHeader const *header,
300                   char *tailbuf, size_t taillen)
301{
302    size_t url_sz;
303    ClientHttpRequest *http = new ClientHttpRequest(NULL);
304    HttpRequest *request;
305    StoreIOBuffer tempBuffer;
306    if (http->al != NULL)
307        http->al->cache.start_time = current_time;
308    /* this is only used to adjust the connection offset in client_side.c */
309    http->req_sz = 0;
310    tempBuffer.length = taillen;
311    tempBuffer.data = tailbuf;
312    /* client stream setup */
313    clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
314                     clientReplyStatus, new clientReplyContext(http), streamcallback,
315                     streamdetach, streamdata, tempBuffer);
316    /* make it visible in the 'current acctive requests list' */
317    /* Set flags */
318    /* internal requests only makes sense in an
319     * accelerator today. TODO: accept flags ? */
320    http->flags.accel = true;
321    /* allow size for url rewriting */
322    url_sz = strlen(url) + Config.appendDomainLen + 5;
323    http->uri = (char *)xcalloc(url_sz, 1);
324    strcpy(http->uri, url);
325
326    if ((request = HttpRequest::CreateFromUrlAndMethod(http->uri, method)) == NULL) {
327        debugs(85, 5, "Invalid URL: " << http->uri);
328        return -1;
329    }
330
331    /*
332     * now update the headers in request with our supplied headers. urlParse
333     * should return a blank header set, but we use Update to be sure of
334     * correctness.
335     */
336    if (header)
337        request->header.update(header, NULL);
338
339    http->log_uri = xstrdup(urlCanonicalClean(request));
340
341    /* http struct now ready */
342
343    /*
344     * build new header list *? TODO
345     */
346    request->flags.accelerated = http->flags.accel;
347
348    request->flags.internalClient = true;
349
350    /* this is an internally created
351     * request, not subject to acceleration
352     * target overrides */
353    /*
354     * FIXME? Do we want to detect and handle internal requests of internal
355     * objects ?
356     */
357
358    /* Internally created requests cannot have bodies today */
359    request->content_length = 0;
360
361    request->client_addr.setNoAddr();
362
363#if FOLLOW_X_FORWARDED_FOR
364    request->indirect_client_addr.setNoAddr();
365#endif /* FOLLOW_X_FORWARDED_FOR */
366
367    request->my_addr.setNoAddr();   /* undefined for internal requests */
368
369    request->my_addr.port(0);
370
371    /* Our version is HTTP/1.1 */
372    request->http_ver = Http::ProtocolVersion(1,1);
373
374    http->request = request;
375    HTTPMSGLOCK(http->request);
376
377    /* optional - skip the access check ? */
378    http->calloutContext = new ClientRequestContext(http);
379
380    http->calloutContext->http_access_done = false;
381
382    http->calloutContext->redirect_done = true;
383
384    http->calloutContext->no_cache_done = true;
385
386    http->doCallouts();
387
388    return 0;
389}
390
391bool
392ClientRequestContext::httpStateIsValid()
393{
394    ClientHttpRequest *http_ = http;
395
396    if (cbdataReferenceValid(http_))
397        return true;
398
399    http = NULL;
400
401    cbdataReferenceDone(http_);
402
403    return false;
404}
405
406#if FOLLOW_X_FORWARDED_FOR
407/**
408 * clientFollowXForwardedForCheck() checks the content of X-Forwarded-For:
409 * against the followXFF ACL, or cleans up and passes control to
410 * clientAccessCheck().
411 *
412 * The trust model here is a little ambiguous. So to clarify the logic:
413 * - we may always use the direct client address as the client IP.
414 * - these trust tests merey tell whether we trust given IP enough to believe the
415 *   IP string which it appended to the X-Forwarded-For: header.
416 * - if at any point we don't trust what an IP adds we stop looking.
417 * - at that point the current contents of indirect_client_addr are the value set
418 *   by the last previously trusted IP.
419 * ++ indirect_client_addr contains the remote direct client from the trusted peers viewpoint.
420 */
421static void
422clientFollowXForwardedForCheck(allow_t answer, void *data)
423{
424    ClientRequestContext *calloutContext = (ClientRequestContext *) data;
425
426    if (!calloutContext->httpStateIsValid())
427        return;
428
429    ClientHttpRequest *http = calloutContext->http;
430    HttpRequest *request = http->request;
431
432    /*
433     * answer should be be ACCESS_ALLOWED or ACCESS_DENIED if we are
434     * called as a result of ACL checks, or -1 if we are called when
435     * there's nothing left to do.
436     */
437    if (answer == ACCESS_ALLOWED &&
438            request->x_forwarded_for_iterator.size () != 0) {
439
440        /*
441         * Remove the last comma-delimited element from the
442         * x_forwarded_for_iterator and use it to repeat the cycle.
443         */
444        const char *p;
445        const char *asciiaddr;
446        int l;
447        Ip::Address addr;
448        p = request->x_forwarded_for_iterator.termedBuf();
449        l = request->x_forwarded_for_iterator.size();
450
451        /*
452        * XXX x_forwarded_for_iterator should really be a list of
453        * IP addresses, but it's a String instead.  We have to
454        * walk backwards through the String, biting off the last
455        * comma-delimited part each time.  As long as the data is in
456        * a String, we should probably implement and use a variant of
457        * strListGetItem() that walks backwards instead of forwards
458        * through a comma-separated list.  But we don't even do that;
459        * we just do the work in-line here.
460        */
461        /* skip trailing space and commas */
462        while (l > 0 && (p[l-1] == ',' || xisspace(p[l-1])))
463            --l;
464        request->x_forwarded_for_iterator.cut(l);
465        /* look for start of last item in list */
466        while (l > 0 && ! (p[l-1] == ',' || xisspace(p[l-1])))
467            --l;
468        asciiaddr = p+l;
469        if ((addr = asciiaddr)) {
470            request->indirect_client_addr = addr;
471            request->x_forwarded_for_iterator.cut(l);
472            calloutContext->acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
473            if (!Config.onoff.acl_uses_indirect_client) {
474                /* override the default src_addr tested if we have to go deeper than one level into XFF */
475                Filled(calloutContext->acl_checklist)->src_addr = request->indirect_client_addr;
476            }
477            calloutContext->acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, data);
478            return;
479        }
480    } /*if (answer == ACCESS_ALLOWED &&
481        request->x_forwarded_for_iterator.size () != 0)*/
482
483    /* clean up, and pass control to clientAccessCheck */
484    if (Config.onoff.log_uses_indirect_client) {
485        /*
486        * Ensure that the access log shows the indirect client
487        * instead of the direct client.
488        */
489        ConnStateData *conn = http->getConn();
490        conn->log_addr = request->indirect_client_addr;
491        http->al->cache.caddr = conn->log_addr;
492    }
493    request->x_forwarded_for_iterator.clean();
494    request->flags.done_follow_x_forwarded_for = true;
495
496    if (answer != ACCESS_ALLOWED && answer != ACCESS_DENIED) {
497        debugs(28, DBG_CRITICAL, "ERROR: Processing X-Forwarded-For. Stopping at IP address: " << request->indirect_client_addr );
498    }
499
500    /* process actual access ACL as normal. */
501    calloutContext->clientAccessCheck();
502}
503#endif /* FOLLOW_X_FORWARDED_FOR */
504
505static void
506hostHeaderIpVerifyWrapper(const ipcache_addrs* ia, const DnsLookupDetails &dns, void *data)
507{
508    ClientRequestContext *c = static_cast<ClientRequestContext*>(data);
509    c->hostHeaderIpVerify(ia, dns);
510}
511
512void
513ClientRequestContext::hostHeaderIpVerify(const ipcache_addrs* ia, const DnsLookupDetails &dns)
514{
515    Comm::ConnectionPointer clientConn = http->getConn()->clientConnection;
516
517    // note the DNS details for the transaction stats.
518    http->request->recordLookup(dns);
519
520    if (ia != NULL && ia->count > 0) {
521        // Is the NAT destination IP in DNS?
522        for (int i = 0; i < ia->count; ++i) {
523            if (clientConn->local.matchIPAddr(ia->in_addrs[i]) == 0) {
524                debugs(85, 3, HERE << "validate IP " << clientConn->local << " possible from Host:");
525                http->request->flags.hostVerified = true;
526                http->doCallouts();
527                return;
528            }
529            debugs(85, 3, HERE << "validate IP " << clientConn->local << " non-match from Host: IP " << ia->in_addrs[i]);
530        }
531    }
532    debugs(85, 3, HERE << "FAIL: validate IP " << clientConn->local << " possible from Host:");
533    hostHeaderVerifyFailed("local IP", "any domain IP");
534}
535
536void
537ClientRequestContext::hostHeaderVerifyFailed(const char *A, const char *B)
538{
539    // IP address validation for Host: failed. Admin wants to ignore them.
540    // NP: we do not yet handle CONNECT tunnels well, so ignore for them
541    if (!Config.onoff.hostStrictVerify && http->request->method != Http::METHOD_CONNECT) {
542        debugs(85, 3, "SECURITY ALERT: Host header forgery detected on " << http->getConn()->clientConnection <<
543               " (" << A << " does not match " << B << ") on URL: " << urlCanonical(http->request));
544
545        // NP: it is tempting to use 'flags.noCache' but that is all about READing cache data.
546        // The problems here are about WRITE for new cache content, which means flags.cachable
547        http->request->flags.cachable = false; // MUST NOT cache (for now)
548        // XXX: when we have updated the cache key to base on raw-IP + URI this cacheable limit can go.
549        http->request->flags.hierarchical = false; // MUST NOT pass to peers (for now)
550        // XXX: when we have sorted out the best way to relay requests properly to peers this hierarchical limit can go.
551        http->doCallouts();
552        return;
553    }
554
555    debugs(85, DBG_IMPORTANT, "SECURITY ALERT: Host header forgery detected on " <<
556           http->getConn()->clientConnection << " (" << A << " does not match " << B << ")");
557    debugs(85, DBG_IMPORTANT, "SECURITY ALERT: By user agent: " << http->request->header.getStr(HDR_USER_AGENT));
558    debugs(85, DBG_IMPORTANT, "SECURITY ALERT: on URL: " << urlCanonical(http->request));
559
560    // IP address validation for Host: failed. reject the connection.
561    clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->prev->data;
562    clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
563    assert (repContext);
564    repContext->setReplyToError(ERR_CONFLICT_HOST, Http::scConflict,
565                                http->request->method, NULL,
566                                http->getConn()->clientConnection->remote,
567                                http->request,
568                                NULL,
569#if USE_AUTH
570                                http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
571                                http->getConn()->getAuth() : http->request->auth_user_request);
572#else
573                                NULL);
574#endif
575    node = (clientStreamNode *)http->client_stream.tail->data;
576    clientStreamRead(node, http, node->readBuffer);
577}
578
579void
580ClientRequestContext::hostHeaderVerify()
581{
582    // Require a Host: header.
583    const char *host = http->request->header.getStr(HDR_HOST);
584
585    if (!host) {
586        // TODO: dump out the HTTP/1.1 error about missing host header.
587        // otherwise this is fine, can't forge a header value when its not even set.
588        debugs(85, 3, HERE << "validate skipped with no Host: header present.");
589        http->doCallouts();
590        return;
591    }
592
593    if (http->request->flags.internal) {
594        // TODO: kill this when URL handling allows partial URLs out of accel mode
595        //       and we no longer screw with the URL just to add our internal host there
596        debugs(85, 6, HERE << "validate skipped due to internal composite URL.");
597        http->doCallouts();
598        return;
599    }
600
601    // Locate if there is a port attached, strip ready for IP lookup
602    char *portStr = NULL;
603    char *hostB = xstrdup(host);
604    host = hostB;
605    if (host[0] == '[') {
606        // IPv6 literal.
607        portStr = strchr(hostB, ']');
608        if (portStr && *(++portStr) != ':') {
609            portStr = NULL;
610        }
611    } else {
612        // Domain or IPv4 literal with port
613        portStr = strrchr(hostB, ':');
614    }
615
616    uint16_t port = 0;
617    if (portStr) {
618        *portStr = '\0'; // strip the ':'
619        if (*(++portStr) != '\0') {
620            char *end = NULL;
621            int64_t ret = strtoll(portStr, &end, 10);
622            if (end == portStr || *end != '\0' || ret < 1 || ret > 0xFFFF) {
623                // invalid port details. Replace the ':'
624                *(--portStr) = ':';
625                portStr = NULL;
626            } else
627                port = (ret & 0xFFFF);
628        }
629    }
630
631    debugs(85, 3, HERE << "validate host=" << host << ", port=" << port << ", portStr=" << (portStr?portStr:"NULL"));
632    if (http->request->flags.intercepted || http->request->flags.interceptTproxy) {
633        // verify the Host: port (if any) matches the apparent destination
634        if (portStr && port != http->getConn()->clientConnection->local.port()) {
635            debugs(85, 3, HERE << "FAIL on validate port " << http->getConn()->clientConnection->local.port() <<
636                   " matches Host: port " << port << " (" << portStr << ")");
637            hostHeaderVerifyFailed("intercepted port", portStr);
638        } else {
639            // XXX: match the scheme default port against the apparent destination
640
641            // verify the destination DNS is one of the Host: headers IPs
642            ipcache_nbgethostbyname(host, hostHeaderIpVerifyWrapper, this);
643        }
644    } else if (!Config.onoff.hostStrictVerify) {
645        debugs(85, 3, HERE << "validate skipped.");
646        http->doCallouts();
647    } else if (strlen(host) != strlen(http->request->GetHost())) {
648        // Verify forward-proxy requested URL domain matches the Host: header
649        debugs(85, 3, HERE << "FAIL on validate URL domain length " << http->request->GetHost() << " matches Host: " << host);
650        hostHeaderVerifyFailed(host, http->request->GetHost());
651    } else if (matchDomainName(host, http->request->GetHost()) != 0) {
652        // Verify forward-proxy requested URL domain matches the Host: header
653        debugs(85, 3, HERE << "FAIL on validate URL domain " << http->request->GetHost() << " matches Host: " << host);
654        hostHeaderVerifyFailed(host, http->request->GetHost());
655    } else if (portStr && port != http->request->port) {
656        // Verify forward-proxy requested URL domain matches the Host: header
657        debugs(85, 3, HERE << "FAIL on validate URL port " << http->request->port << " matches Host: port " << portStr);
658        hostHeaderVerifyFailed("URL port", portStr);
659    } else if (!portStr && http->request->method != Http::METHOD_CONNECT && http->request->port != urlDefaultPort(http->request->url.getScheme())) {
660        // Verify forward-proxy requested URL domain matches the Host: header
661        // Special case: we don't have a default-port to check for CONNECT. Assume URL is correct.
662        debugs(85, 3, "FAIL on validate URL port " << http->request->port << " matches Host: default port " << urlDefaultPort(http->request->url.getScheme()));
663        hostHeaderVerifyFailed("URL port", "default port");
664    } else {
665        // Okay no problem.
666        debugs(85, 3, HERE << "validate passed.");
667        http->request->flags.hostVerified = true;
668        http->doCallouts();
669    }
670    safe_free(hostB);
671}
672
673/* This is the entry point for external users of the client_side routines */
674void
675ClientRequestContext::clientAccessCheck()
676{
677#if FOLLOW_X_FORWARDED_FOR
678    if (!http->request->flags.doneFollowXff() &&
679            Config.accessList.followXFF &&
680            http->request->header.has(HDR_X_FORWARDED_FOR)) {
681
682        /* we always trust the direct client address for actual use */
683        http->request->indirect_client_addr = http->request->client_addr;
684        http->request->indirect_client_addr.port(0);
685
686        /* setup the XFF iterator for processing */
687        http->request->x_forwarded_for_iterator = http->request->header.getList(HDR_X_FORWARDED_FOR);
688
689        /* begin by checking to see if we trust direct client enough to walk XFF */
690        acl_checklist = clientAclChecklistCreate(Config.accessList.followXFF, http);
691        acl_checklist->nonBlockingCheck(clientFollowXForwardedForCheck, this);
692        return;
693    }
694#endif
695
696    if (Config.accessList.http) {
697        acl_checklist = clientAclChecklistCreate(Config.accessList.http, http);
698        acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
699    } else {
700        debugs(0, DBG_CRITICAL, "No http_access configuration found. This will block ALL traffic");
701        clientAccessCheckDone(ACCESS_DENIED);
702    }
703}
704
705/**
706 * Identical in operation to clientAccessCheck() but performed later using different configured ACL list.
707 * The default here is to allow all. Since the earlier http_access should do a default deny all.
708 * This check is just for a last-minute denial based on adapted request headers.
709 */
710void
711ClientRequestContext::clientAccessCheck2()
712{
713    if (Config.accessList.adapted_http) {
714        acl_checklist = clientAclChecklistCreate(Config.accessList.adapted_http, http);
715        acl_checklist->nonBlockingCheck(clientAccessCheckDoneWrapper, this);
716    } else {
717        debugs(85, 2, HERE << "No adapted_http_access configuration. default: ALLOW");
718        clientAccessCheckDone(ACCESS_ALLOWED);
719    }
720}
721
722void
723clientAccessCheckDoneWrapper(allow_t answer, void *data)
724{
725    ClientRequestContext *calloutContext = (ClientRequestContext *) data;
726
727    if (!calloutContext->httpStateIsValid())
728        return;
729
730    calloutContext->clientAccessCheckDone(answer);
731}
732
733void
734ClientRequestContext::clientAccessCheckDone(const allow_t &answer)
735{
736    acl_checklist = NULL;
737    err_type page_id;
738    Http::StatusCode status;
739    debugs(85, 2, "The request " << http->request->method << ' ' <<
740           http->uri << " is " << answer <<
741           "; last ACL checked: " << (AclMatchedName ? AclMatchedName : "[none]"));
742
743#if USE_AUTH
744    char const *proxy_auth_msg = "<null>";
745    if (http->getConn() != NULL && http->getConn()->getAuth() != NULL)
746        proxy_auth_msg = http->getConn()->getAuth()->denyMessage("<null>");
747    else if (http->request->auth_user_request != NULL)
748        proxy_auth_msg = http->request->auth_user_request->denyMessage("<null>");
749#endif
750
751    if (answer != ACCESS_ALLOWED) {
752        // auth has a grace period where credentials can be expired but okay not to challenge.
753
754        /* Send an auth challenge or error */
755        // XXX: do we still need aclIsProxyAuth() ?
756        bool auth_challenge = (answer == ACCESS_AUTH_REQUIRED || aclIsProxyAuth(AclMatchedName));
757        debugs(85, 5, "Access Denied: " << http->uri);
758        debugs(85, 5, "AclMatchedName = " << (AclMatchedName ? AclMatchedName : "<null>"));
759#if USE_AUTH
760        if (auth_challenge)
761            debugs(33, 5, "Proxy Auth Message = " << (proxy_auth_msg ? proxy_auth_msg : "<null>"));
762#endif
763
764        /*
765         * NOTE: get page_id here, based on AclMatchedName because if
766         * USE_DELAY_POOLS is enabled, then AclMatchedName gets clobbered in
767         * the clientCreateStoreEntry() call just below.  Pedro Ribeiro
768         * <pribeiro@isel.pt>
769         */
770        page_id = aclGetDenyInfoPage(&Config.denyInfoList, AclMatchedName, answer != ACCESS_AUTH_REQUIRED);
771
772        http->logType = LOG_TCP_DENIED;
773
774        if (auth_challenge) {
775#if USE_AUTH
776            if (http->request->flags.sslBumped) {
777                /*SSL Bumped request, authentication is not possible*/
778                status = Http::scForbidden;
779            } else if (!http->flags.accel) {
780                /* Proxy authorisation needed */
781                status = Http::scProxyAuthenticationRequired;
782            } else {
783                /* WWW authorisation needed */
784                status = Http::scUnauthorized;
785            }
786#else
787            // need auth, but not possible to do.
788            status = Http::scForbidden;
789#endif
790            if (page_id == ERR_NONE)
791                page_id = ERR_CACHE_ACCESS_DENIED;
792        } else {
793            status = Http::scForbidden;
794
795            if (page_id == ERR_NONE)
796                page_id = ERR_ACCESS_DENIED;
797        }
798
799        Ip::Address tmpnoaddr;
800        tmpnoaddr.setNoAddr();
801        error = clientBuildError(page_id, status,
802                                 NULL,
803                                 http->getConn() != NULL ? http->getConn()->clientConnection->remote : tmpnoaddr,
804                                 http->request
805                                );
806
807#if USE_AUTH
808        error->auth_user_request =
809            http->getConn() != NULL && http->getConn()->getAuth() != NULL ?
810            http->getConn()->getAuth() : http->request->auth_user_request;
811#endif
812
813        readNextRequest = true;
814    }
815
816    /* ACCESS_ALLOWED continues here ... */
817    safe_free(http->uri);
818
819    http->uri = xstrdup(urlCanonical(http->request));
820
821    http->doCallouts();
822}
823
824#if USE_ADAPTATION
825void
826ClientHttpRequest::noteAdaptationAclCheckDone(Adaptation::ServiceGroupPointer g)
827{
828    debugs(93,3,HERE << this << " adaptationAclCheckDone called");
829
830#if ICAP_CLIENT
831    Adaptation::Icap::History::Pointer ih = request->icapHistory();
832    if (ih != NULL) {
833        if (getConn() != NULL && getConn()->clientConnection != NULL) {
834            ih->rfc931 = getConn()->clientConnection->rfc931;
835#if USE_OPENSSL
836            if (getConn()->clientConnection->isOpen()) {
837                ih->ssluser = sslGetUserEmail(fd_table[getConn()->clientConnection->fd].ssl);
838            }
839#endif
840        }
841        ih->log_uri = log_uri;
842        ih->req_sz = req_sz;
843    }
844#endif
845
846    if (!g) {
847        debugs(85,3, HERE << "no adaptation needed");
848        doCallouts();
849        return;
850    }
851
852    startAdaptation(g);
853}
854
855#endif
856
857static void
858clientRedirectAccessCheckDone(allow_t answer, void *data)
859{
860    ClientRequestContext *context = (ClientRequestContext *)data;
861    ClientHttpRequest *http = context->http;
862    context->acl_checklist = NULL;
863
864    if (answer == ACCESS_ALLOWED)
865        redirectStart(http, clientRedirectDoneWrapper, context);
866    else {
867        Helper::Reply nilReply;
868        nilReply.result = Helper::Error;
869        context->clientRedirectDone(nilReply);
870    }
871}
872
873void
874ClientRequestContext::clientRedirectStart()
875{
876    debugs(33, 5, HERE << "'" << http->uri << "'");
877    (void)SyncNotes(*http->al, *http->request);
878    if (Config.accessList.redirector) {
879        acl_checklist = clientAclChecklistCreate(Config.accessList.redirector, http);
880        acl_checklist->nonBlockingCheck(clientRedirectAccessCheckDone, this);
881    } else
882        redirectStart(http, clientRedirectDoneWrapper, this);
883}
884
885/**
886 * This methods handles Access checks result of StoreId access list.
887 * Will handle as "ERR" (no change) in a case Access is not allowed.
888 */
889static void
890clientStoreIdAccessCheckDone(allow_t answer, void *data)
891{
892    ClientRequestContext *context = static_cast<ClientRequestContext *>(data);
893    ClientHttpRequest *http = context->http;
894    context->acl_checklist = NULL;
895
896    if (answer == ACCESS_ALLOWED)
897        storeIdStart(http, clientStoreIdDoneWrapper, context);
898    else {
899        debugs(85, 3, "access denied expected ERR reply handling: " << answer);
900        Helper::Reply nilReply;
901        nilReply.result = Helper::Error;
902        context->clientStoreIdDone(nilReply);
903    }
904}
905
906/**
907 * Start locating an alternative storeage ID string (if any) from admin
908 * configured helper program. This is an asynchronous operation terminating in
909 * ClientRequestContext::clientStoreIdDone() when completed.
910 */
911void
912ClientRequestContext::clientStoreIdStart()
913{
914    debugs(33, 5,"'" << http->uri << "'");
915
916    if (Config.accessList.store_id) {
917        acl_checklist = clientAclChecklistCreate(Config.accessList.store_id, http);
918        acl_checklist->nonBlockingCheck(clientStoreIdAccessCheckDone, this);
919    } else
920        storeIdStart(http, clientStoreIdDoneWrapper, this);
921}
922
923static int
924clientHierarchical(ClientHttpRequest * http)
925{
926    HttpRequest *request = http->request;
927    HttpRequestMethod method = request->method;
928
929    // intercepted requests MUST NOT (yet) be sent to peers unless verified
930    if (!request->flags.hostVerified && (request->flags.intercepted || request->flags.interceptTproxy))
931        return 0;
932
933    /*
934     * IMS needs a private key, so we can use the hierarchy for IMS only if our
935     * neighbors support private keys
936     */
937
938    if (request->flags.ims && !neighbors_do_private_keys)
939        return 0;
940
941    /*
942     * This is incorrect: authenticating requests can be sent via a hierarchy
943     * (they can even be cached if the correct headers are set on the reply)
944     */
945    if (request->flags.auth)
946        return 0;
947
948    if (method == Http::METHOD_TRACE)
949        return 1;
950
951    if (method != Http::METHOD_GET)
952        return 0;
953
954    if (request->flags.loopDetected)
955        return 0;
956
957    if (request->url.getScheme() == AnyP::PROTO_HTTP)
958        return method.respMaybeCacheable();
959
960    if (request->url.getScheme() == AnyP::PROTO_GOPHER)
961        return gopherCachable(request);
962
963    if (request->url.getScheme() == AnyP::PROTO_CACHE_OBJECT)
964        return 0;
965
966    return 1;
967}
968
969static void
970clientCheckPinning(ClientHttpRequest * http)
971{
972    HttpRequest *request = http->request;
973    HttpHeader *req_hdr = &request->header;
974    ConnStateData *http_conn = http->getConn();
975
976    /* Internal requests such as those from ESI includes may be without
977     * a client connection
978     */
979    if (!http_conn)
980        return;
981
982    request->flags.connectionAuthDisabled = http_conn->port->connection_auth_disabled;
983    if (!request->flags.connectionAuthDisabled) {
984        if (Comm::IsConnOpen(http_conn->pinning.serverConnection)) {
985            if (http_conn->pinning.auth) {
986                request->flags.connectionAuth = true;
987                request->flags.auth = true;
988            } else {
989                request->flags.connectionProxyAuth = true;
990            }
991            // These should already be linked correctly.
992            assert(request->clientConnectionManager == http_conn);
993        }
994    }
995
996    /* check if connection auth is used, and flag as candidate for pinning
997     * in such case.
998     * Note: we may need to set flags.connectionAuth even if the connection
999     * is already pinned if it was pinned earlier due to proxy auth
1000     */
1001    if (!request->flags.connectionAuth) {
1002        if (req_hdr->has(HDR_AUTHORIZATION) || req_hdr->has(HDR_PROXY_AUTHORIZATION)) {
1003            HttpHeaderPos pos = HttpHeaderInitPos;
1004            HttpHeaderEntry *e;
1005            int may_pin = 0;
1006            while ((e = req_hdr->getEntry(&pos))) {
1007                if (e->id == HDR_AUTHORIZATION || e->id == HDR_PROXY_AUTHORIZATION) {
1008                    const char *value = e->value.rawBuf();
1009                    if (strncasecmp(value, "NTLM ", 5) == 0
1010                            ||
1011                            strncasecmp(value, "Negotiate ", 10) == 0
1012                            ||
1013                            strncasecmp(value, "Kerberos ", 9) == 0) {
1014                        if (e->id == HDR_AUTHORIZATION) {
1015                            request->flags.connectionAuth = true;
1016                            may_pin = 1;
1017                        } else {
1018                            request->flags.connectionProxyAuth = true;
1019                            may_pin = 1;
1020                        }
1021                    }
1022                }
1023            }
1024            if (may_pin && !request->pinnedConnection()) {
1025                // These should already be linked correctly. Just need the ServerConnection to pinn.
1026                assert(request->clientConnectionManager == http_conn);
1027            }
1028        }
1029    }
1030}
1031
1032static void
1033clientInterpretRequestHeaders(ClientHttpRequest * http)
1034{
1035    HttpRequest *request = http->request;
1036    HttpHeader *req_hdr = &request->header;
1037    bool no_cache = false;
1038    const char *str;
1039
1040    request->imslen = -1;
1041    request->ims = req_hdr->getTime(HDR_IF_MODIFIED_SINCE);
1042
1043    if (request->ims > 0)
1044        request->flags.ims = true;
1045
1046    if (!request->flags.ignoreCc) {
1047        if (request->cache_control) {
1048            if (request->cache_control->hasNoCache())
1049                no_cache=true;
1050
1051            // RFC 2616: treat Pragma:no-cache as if it was Cache-Control:no-cache when Cache-Control is missing
1052        } else if (req_hdr->has(HDR_PRAGMA))
1053            no_cache = req_hdr->hasListMember(HDR_PRAGMA,"no-cache",',');
1054
1055        /*
1056        * Work around for supporting the Reload button in IE browsers when Squid
1057        * is used as an accelerator or transparent proxy, by turning accelerated
1058        * IMS request to no-cache requests. Now knows about IE 5.5 fix (is
1059        * actually only fixed in SP1, but we can't tell whether we are talking to
1060        * SP1 or not so all 5.5 versions are treated 'normally').
1061        */
1062        if (Config.onoff.ie_refresh) {
1063            if (http->flags.accel && request->flags.ims) {
1064                if ((str = req_hdr->getStr(HDR_USER_AGENT))) {
1065                    if (strstr(str, "MSIE 5.01") != NULL)
1066                        no_cache=true;
1067                    else if (strstr(str, "MSIE 5.0") != NULL)
1068                        no_cache=true;
1069                    else if (strstr(str, "MSIE 4.") != NULL)
1070                        no_cache=true;
1071                    else if (strstr(str, "MSIE 3.") != NULL)
1072                        no_cache=true;
1073                }
1074            }
1075        }
1076    }
1077
1078    if (request->method == Http::METHOD_OTHER) {
1079        no_cache=true;
1080    }
1081
1082    if (no_cache) {
1083#if USE_HTTP_VIOLATIONS
1084
1085        if (Config.onoff.reload_into_ims)
1086            request->flags.nocacheHack = true;
1087        else if (refresh_nocache_hack)
1088            request->flags.nocacheHack = true;
1089        else
1090#endif
1091
1092            request->flags.noCache = true;
1093    }
1094
1095    /* ignore range header in non-GETs or non-HEADs */
1096    if (request->method == Http::METHOD_GET || request->method == Http::METHOD_HEAD) {
1097        // XXX: initialize if we got here without HttpRequest::parseHeader()
1098        if (!request->range)
1099            request->range = req_hdr->getRange();
1100
1101        if (request->range) {
1102            request->flags.isRanged = true;
1103            clientStreamNode *node = (clientStreamNode *)http->client_stream.tail->data;
1104            /* XXX: This is suboptimal. We should give the stream the range set,
1105             * and thereby let the top of the stream set the offset when the
1106             * size becomes known. As it is, we will end up requesting from 0
1107             * for evey -X range specification.
1108             * RBC - this may be somewhat wrong. We should probably set the range
1109             * iter up at this point.
1110             */
1111            node->readBuffer.offset = request->range->lowestOffset(0);
1112            http->range_iter.pos = request->range->begin();
1113            http->range_iter.end = request->range->end();
1114            http->range_iter.valid = true;
1115        }
1116    }
1117
1118    /* Only HEAD and GET requests permit a Range or Request-Range header.
1119     * If these headers appear on any other type of request, delete them now.
1120     */
1121    else {
1122        req_hdr->delById(HDR_RANGE);
1123        req_hdr->delById(HDR_REQUEST_RANGE);
1124        request->ignoreRange("neither HEAD nor GET");
1125    }
1126
1127    if (req_hdr->has(HDR_AUTHORIZATION))
1128        request->flags.auth = true;
1129
1130    clientCheckPinning(http);
1131
1132    if (request->login[0] != '\0')
1133        request->flags.auth = true;
1134
1135    if (req_hdr->has(HDR_VIA)) {
1136        String s = req_hdr->getList(HDR_VIA);
1137        /*
1138         * ThisCache cannot be a member of Via header, "1.1 ThisCache" can.
1139         * Note ThisCache2 has a space prepended to the hostname so we don't
1140         * accidentally match super-domains.
1141         */
1142
1143        if (strListIsSubstr(&s, ThisCache2, ',')) {
1144            debugObj(33, 1, "WARNING: Forwarding loop detected for:\n",
1145                     request, (ObjPackMethod) & httpRequestPack);
1146            request->flags.loopDetected = true;
1147        }
1148
1149#if USE_FORW_VIA_DB
1150        fvdbCountVia(s.termedBuf());
1151
1152#endif
1153
1154        s.clean();
1155    }
1156
1157#if USE_FORW_VIA_DB
1158
1159    if (req_hdr->has(HDR_X_FORWARDED_FOR)) {
1160        String s = req_hdr->getList(HDR_X_FORWARDED_FOR);
1161        fvdbCountForw(s.termedBuf());
1162        s.clean();
1163    }
1164
1165#endif
1166
1167    request->flags.cachable = http->request->maybeCacheable();
1168
1169    if (clientHierarchical(http))
1170        request->flags.hierarchical = true;
1171
1172    debugs(85, 5, "clientInterpretRequestHeaders: REQ_NOCACHE = " <<
1173           (request->flags.noCache ? "SET" : "NOT SET"));
1174    debugs(85, 5, "clientInterpretRequestHeaders: REQ_CACHABLE = " <<
1175           (request->flags.cachable ? "SET" : "NOT SET"));
1176    debugs(85, 5, "clientInterpretRequestHeaders: REQ_HIERARCHICAL = " <<
1177           (request->flags.hierarchical ? "SET" : "NOT SET"));
1178
1179}
1180
1181void
1182clientRedirectDoneWrapper(void *data, const Helper::Reply &result)
1183{
1184    ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1185
1186    if (!calloutContext->httpStateIsValid())
1187        return;
1188
1189    calloutContext->clientRedirectDone(result);
1190}
1191
1192void
1193clientStoreIdDoneWrapper(void *data, const Helper::Reply &result)
1194{
1195    ClientRequestContext *calloutContext = (ClientRequestContext *)data;
1196
1197    if (!calloutContext->httpStateIsValid())
1198        return;
1199
1200    calloutContext->clientStoreIdDone(result);
1201}
1202
1203void
1204ClientRequestContext::clientRedirectDone(const Helper::Reply &reply)
1205{
1206    HttpRequest *old_request = http->request;
1207    debugs(85, 5, HERE << "'" << http->uri << "' result=" << reply);
1208    assert(redirect_state == REDIRECT_PENDING);
1209    redirect_state = REDIRECT_DONE;
1210
1211    // Put helper response Notes into the transaction state record (ALE) eventually
1212    // do it early to ensure that no matter what the outcome the notes are present.
1213    if (http->al != NULL)
1214        (void)SyncNotes(*http->al, *old_request);
1215
1216    UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1217
1218    switch (reply.result) {
1219    case Helper::Unknown:
1220    case Helper::TT:
1221        // Handler in redirect.cc should have already mapped Unknown
1222        // IF it contained valid entry for the old URL-rewrite helper protocol
1223        debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper returned invalid result code. Wrong helper? " << reply);
1224        break;
1225
1226    case Helper::BrokenHelper:
1227        debugs(85, DBG_IMPORTANT, "ERROR: URL rewrite helper: " << reply << ", attempt #" << (redirect_fail_count+1) << " of 2");
1228        if (redirect_fail_count < 2) { // XXX: make this configurable ?
1229            ++redirect_fail_count;
1230            // reset state flag to try redirector again from scratch.
1231            redirect_done = false;
1232        }
1233        break;
1234
1235    case Helper::Error:
1236        // no change to be done.
1237        break;
1238
1239    case Helper::Okay: {
1240        // #1: redirect with a specific status code    OK status=NNN url="..."
1241        // #2: redirect with a default status code     OK url="..."
1242        // #3: re-write the URL                        OK rewrite-url="..."
1243
1244        const char *statusNote = reply.notes.findFirst("status");
1245        const char *urlNote = reply.notes.findFirst("url");
1246
1247        if (urlNote != NULL) {
1248            // HTTP protocol redirect to be done.
1249
1250            // TODO: change default redirect status for appropriate requests
1251            // Squid defaults to 302 status for now for better compatibility with old clients.
1252            // HTTP/1.0 client should get 302 (Http::scFound)
1253            // HTTP/1.1 client contacting reverse-proxy should get 307 (Http::scTemporaryRedirect)
1254            // HTTP/1.1 client being diverted by forward-proxy should get 303 (Http::scSeeOther)
1255            Http::StatusCode status = Http::scFound;
1256            if (statusNote != NULL) {
1257                const char * result = statusNote;
1258                status = static_cast<Http::StatusCode>(atoi(result));
1259            }
1260
1261            if (status == Http::scMovedPermanently
1262                    || status == Http::scFound
1263                    || status == Http::scSeeOther
1264                    || status == Http::scPermanentRedirect
1265                    || status == Http::scTemporaryRedirect) {
1266                http->redirect.status = status;
1267                http->redirect.location = xstrdup(urlNote);
1268                // TODO: validate the URL produced here is RFC 2616 compliant absolute URI
1269            } else {
1270                debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid " << status << " redirect Location: " << urlNote);
1271            }
1272        } else {
1273            // URL-rewrite wanted. Ew.
1274            urlNote = reply.notes.findFirst("rewrite-url");
1275
1276            // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1277            if (urlNote != NULL && strcmp(urlNote, http->uri)) {
1278                // XXX: validate the URL properly *without* generating a whole new request object right here.
1279                // XXX: the clone() should be done only AFTER we know the new URL is valid.
1280                HttpRequest *new_request = old_request->clone();
1281                if (urlParse(old_request->method, const_cast<char*>(urlNote), new_request)) {
1282                    debugs(61,2, HERE << "URL-rewriter diverts URL from " << urlCanonical(old_request) << " to " << urlCanonical(new_request));
1283
1284                    // update the new request to flag the re-writing was done on it
1285                    new_request->flags.redirected = true;
1286
1287                    // unlink bodypipe from the old request. Not needed there any longer.
1288                    if (old_request->body_pipe != NULL) {
1289                        old_request->body_pipe = NULL;
1290                        debugs(61,2, HERE << "URL-rewriter diverts body_pipe " << new_request->body_pipe <<
1291                               " from request " << old_request << " to " << new_request);
1292                    }
1293
1294                    // update the current working ClientHttpRequest fields
1295                    safe_free(http->uri);
1296                    http->uri = xstrdup(urlCanonical(new_request));
1297                    HTTPMSGUNLOCK(old_request);
1298                    http->request = new_request;
1299                    HTTPMSGLOCK(http->request);
1300                } else {
1301                    debugs(85, DBG_CRITICAL, "ERROR: URL-rewrite produces invalid request: " <<
1302                           old_request->method << " " << urlNote << " " << old_request->http_ver);
1303                    delete new_request;
1304                }
1305            }
1306        }
1307    }
1308    break;
1309    }
1310
1311    /* FIXME PIPELINE: This is innacurate during pipelining */
1312
1313    if (http->getConn() != NULL && Comm::IsConnOpen(http->getConn()->clientConnection))
1314        fd_note(http->getConn()->clientConnection->fd, http->uri);
1315
1316    assert(http->uri);
1317
1318    http->doCallouts();
1319}
1320
1321/**
1322 * This method handles the different replies from StoreID helper.
1323 */
1324void
1325ClientRequestContext::clientStoreIdDone(const Helper::Reply &reply)
1326{
1327    HttpRequest *old_request = http->request;
1328    debugs(85, 5, "'" << http->uri << "' result=" << reply);
1329    assert(store_id_state == REDIRECT_PENDING);
1330    store_id_state = REDIRECT_DONE;
1331
1332    // Put helper response Notes into the transaction state record (ALE) eventually
1333    // do it early to ensure that no matter what the outcome the notes are present.
1334    if (http->al != NULL)
1335        (void)SyncNotes(*http->al, *old_request);
1336
1337    UpdateRequestNotes(http->getConn(), *old_request, reply.notes);
1338
1339    switch (reply.result) {
1340    case Helper::Unknown:
1341    case Helper::TT:
1342        // Handler in redirect.cc should have already mapped Unknown
1343        // IF it contained valid entry for the old helper protocol
1344        debugs(85, DBG_IMPORTANT, "ERROR: storeID helper returned invalid result code. Wrong helper? " << reply);
1345        break;
1346
1347    case Helper::BrokenHelper:
1348        debugs(85, DBG_IMPORTANT, "ERROR: storeID helper: " << reply << ", attempt #" << (store_id_fail_count+1) << " of 2");
1349        if (store_id_fail_count < 2) { // XXX: make this configurable ?
1350            ++store_id_fail_count;
1351            // reset state flag to try StoreID again from scratch.
1352            store_id_done = false;
1353        }
1354        break;
1355
1356    case Helper::Error:
1357        // no change to be done.
1358        break;
1359
1360    case Helper::Okay: {
1361        const char *urlNote = reply.notes.findFirst("store-id");
1362
1363        // prevent broken helpers causing too much damage. If old URL == new URL skip the re-write.
1364        if (urlNote != NULL && strcmp(urlNote, http->uri) ) {
1365            // Debug section required for some very specific cases.
1366            debugs(85, 9, "Setting storeID with: " << urlNote );
1367            http->request->store_id = urlNote;
1368            http->store_id = urlNote;
1369        }
1370    }
1371    break;
1372    }
1373
1374    http->doCallouts();
1375}
1376
1377/** Test cache allow/deny configuration
1378 *  Sets flags.cachable=1 if caching is not denied.
1379 */
1380void
1381ClientRequestContext::checkNoCache()
1382{
1383    if (Config.accessList.noCache) {
1384        acl_checklist = clientAclChecklistCreate(Config.accessList.noCache, http);
1385        acl_checklist->nonBlockingCheck(checkNoCacheDoneWrapper, this);
1386    } else {
1387        /* unless otherwise specified, we try to cache. */
1388        checkNoCacheDone(ACCESS_ALLOWED);
1389    }
1390}
1391
1392static void
1393checkNoCacheDoneWrapper(allow_t answer, void *data)
1394{
1395    ClientRequestContext *calloutContext = (ClientRequestContext *) data;
1396
1397    if (!calloutContext->httpStateIsValid())
1398        return;
1399
1400    calloutContext->checkNoCacheDone(answer);
1401}
1402
1403void
1404ClientRequestContext::checkNoCacheDone(const allow_t &answer)
1405{
1406    acl_checklist = NULL;
1407    http->request->flags.cachable = (answer == ACCESS_ALLOWED);
1408    http->doCallouts();
1409}
1410
1411#if USE_OPENSSL
1412bool
1413ClientRequestContext::sslBumpAccessCheck()
1414{
1415    // If SSL connection tunneling or bumping decision has been made, obey it.
1416    const Ssl::BumpMode bumpMode = http->getConn()->sslBumpMode;
1417    if (bumpMode != Ssl::bumpEnd) {
1418        debugs(85, 5, HERE << "SslBump already decided (" << bumpMode <<
1419               "), " << "ignoring ssl_bump for " << http->getConn());
1420        if (!http->getConn()->serverBump())
1421            http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed and not already bumped
1422        http->al->ssl.bumpMode = bumpMode; // inherited from bumped connection
1423        return false;
1424    }
1425
1426    // If we have not decided yet, decide whether to bump now.
1427
1428    // Bumping here can only start with a CONNECT request on a bumping port
1429    // (bumping of intercepted SSL conns is decided before we get 1st request).
1430    // We also do not bump redirected CONNECT requests.
1431    if (http->request->method != Http::METHOD_CONNECT || http->redirect.status ||
1432            !Config.accessList.ssl_bump ||
1433            !http->getConn()->port->flags.tunnelSslBumping) {
1434        http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1435        debugs(85, 5, HERE << "cannot SslBump this request");
1436        return false;
1437    }
1438
1439    // Do not bump during authentication: clients would not proxy-authenticate
1440    // if we delay a 407 response and respond with 200 OK to CONNECT.
1441    if (error && error->httpStatus == Http::scProxyAuthenticationRequired) {
1442        http->al->ssl.bumpMode = Ssl::bumpEnd; // SslBump does not apply; log -
1443        debugs(85, 5, HERE << "no SslBump during proxy authentication");
1444        return false;
1445    }
1446
1447    debugs(85, 5, HERE << "SslBump possible, checking ACL");
1448
1449    ACLFilledChecklist *aclChecklist = clientAclChecklistCreate(Config.accessList.ssl_bump, http);
1450    aclChecklist->nonBlockingCheck(sslBumpAccessCheckDoneWrapper, this);
1451    return true;
1452}
1453
1454/**
1455 * A wrapper function to use the ClientRequestContext::sslBumpAccessCheckDone method
1456 * as ACLFilledChecklist callback
1457 */
1458static void
1459sslBumpAccessCheckDoneWrapper(allow_t answer, void *data)
1460{
1461    ClientRequestContext *calloutContext = static_cast<ClientRequestContext *>(data);
1462
1463    if (!calloutContext->httpStateIsValid())
1464        return;
1465    calloutContext->sslBumpAccessCheckDone(answer);
1466}
1467
1468void
1469ClientRequestContext::sslBumpAccessCheckDone(const allow_t &answer)
1470{
1471    if (!httpStateIsValid())
1472        return;
1473
1474    const Ssl::BumpMode bumpMode = answer == ACCESS_ALLOWED ?
1475                                   static_cast<Ssl::BumpMode>(answer.kind) : Ssl::bumpNone;
1476    http->sslBumpNeed(bumpMode); // for processRequest() to bump if needed
1477    http->al->ssl.bumpMode = bumpMode; // for logging
1478
1479    http->doCallouts();
1480}
1481#endif
1482
1483/*
1484 * Identify requests that do not go through the store and client side stream
1485 * and forward them to the appropriate location. All other requests, request
1486 * them.
1487 */
1488void
1489ClientHttpRequest::processRequest()
1490{
1491    debugs(85, 4, request->method << ' ' << uri);
1492
1493    if (request->method == Http::METHOD_CONNECT && !redirect.status) {
1494#if USE_OPENSSL
1495        if (sslBumpNeeded()) {
1496            sslBumpStart();
1497            return;
1498        }
1499#endif
1500        getConn()->stopReading(); // tunnels read for themselves
1501        tunnelStart(this, &out.size, &al->http.code, al);
1502        return;
1503    }
1504
1505    httpStart();
1506}
1507
1508void
1509ClientHttpRequest::httpStart()
1510{
1511    PROF_start(httpStart);
1512    logType = LOG_TAG_NONE;
1513    debugs(85, 4, LogTags_str[logType] << " for '" << uri << "'");
1514
1515    /* no one should have touched this */
1516    assert(out.offset == 0);
1517    /* Use the Stream Luke */
1518    clientStreamNode *node = (clientStreamNode *)client_stream.tail->data;
1519    clientStreamRead(node, this, node->readBuffer);
1520    PROF_stop(httpStart);
1521}
1522
1523#if USE_OPENSSL
1524
1525void
1526ClientHttpRequest::sslBumpNeed(Ssl::BumpMode mode)
1527{
1528    debugs(83, 3, HERE << "sslBump required: "<< Ssl::bumpMode(mode));
1529    sslBumpNeed_ = mode;
1530}
1531
1532// called when comm_write has completed
1533static void
1534SslBumpEstablish(const Comm::ConnectionPointer &, char *, size_t, Comm::Flag errflag, int, void *data)
1535{
1536    ClientHttpRequest *r = static_cast<ClientHttpRequest*>(data);
1537    debugs(85, 5, HERE << "responded to CONNECT: " << r << " ? " << errflag);
1538
1539    assert(r && cbdataReferenceValid(r));
1540    r->sslBumpEstablish(errflag);
1541}
1542
1543void
1544ClientHttpRequest::sslBumpEstablish(Comm::Flag errflag)
1545{
1546    // Bail out quickly on Comm::ERR_CLOSING - close handlers will tidy up
1547    if (errflag == Comm::ERR_CLOSING)
1548        return;
1549
1550    if (errflag) {
1551        debugs(85, 3, HERE << "CONNECT response failure in SslBump: " << errflag);
1552        getConn()->clientConnection->close();
1553        return;
1554    }
1555
1556    // We lack HttpReply which logRequest() uses to log the status code.
1557    // TODO: Use HttpReply instead of the "200 Connection established" string.
1558    al->http.code = 200;
1559
1560#if USE_AUTH
1561    // Preserve authentication info for the ssl-bumped request
1562    if (request->auth_user_request != NULL)
1563        getConn()->setAuth(request->auth_user_request, "SSL-bumped CONNECT");
1564#endif
1565
1566    assert(sslBumpNeeded());
1567    getConn()->switchToHttps(request, sslBumpNeed_);
1568}
1569
1570void
1571ClientHttpRequest::sslBumpStart()
1572{
1573    debugs(85, 5, HERE << "Confirming " << Ssl::bumpMode(sslBumpNeed_) <<
1574           "-bumped CONNECT tunnel on FD " << getConn()->clientConnection);
1575    getConn()->sslBumpMode = sslBumpNeed_;
1576
1577    AsyncCall::Pointer bumpCall = commCbCall(85, 5, "ClientSocketContext::sslBumpEstablish",
1578                                  CommIoCbPtrFun(&SslBumpEstablish, this));
1579
1580    if (request->flags.interceptTproxy || request->flags.intercepted) {
1581        CommIoCbParams &params = GetCommParams<CommIoCbParams>(bumpCall);
1582        params.flag = Comm::OK;
1583        params.conn = getConn()->clientConnection;
1584        ScheduleCallHere(bumpCall);
1585        return;
1586    }
1587
1588    // send an HTTP 200 response to kick client SSL negotiation
1589    // TODO: Unify with tunnel.cc and add a Server(?) header
1590    static const char *const conn_established = "HTTP/1.1 200 Connection established\r\n\r\n";
1591    Comm::Write(getConn()->clientConnection, conn_established, strlen(conn_established), bumpCall, NULL);
1592}
1593
1594#endif
1595
1596bool
1597ClientHttpRequest::gotEnough() const
1598{
1599    /** TODO: should be querying the stream. */
1600    int64_t contentLength =
1601        memObject()->getReply()->bodySize(request->method);
1602    assert(contentLength >= 0);
1603
1604    if (out.offset < contentLength)
1605        return false;
1606
1607    return true;
1608}
1609
1610void
1611ClientHttpRequest::storeEntry(StoreEntry *newEntry)
1612{
1613    entry_ = newEntry;
1614}
1615
1616void
1617ClientHttpRequest::loggingEntry(StoreEntry *newEntry)
1618{
1619    if (loggingEntry_)
1620        loggingEntry_->unlock("ClientHttpRequest::loggingEntry");
1621
1622    loggingEntry_ = newEntry;
1623
1624    if (loggingEntry_)
1625        loggingEntry_->lock("ClientHttpRequest::loggingEntry");
1626}
1627
1628/*
1629 * doCallouts() - This function controls the order of "callout"
1630 * executions, including non-blocking access control checks, the
1631 * redirector, and ICAP.  Previously, these callouts were chained
1632 * together such that "clientAccessCheckDone()" would call
1633 * "clientRedirectStart()" and so on.
1634 *
1635 * The ClientRequestContext (aka calloutContext) class holds certain
1636 * state data for the callout/callback operations.  Previously
1637 * ClientHttpRequest would sort of hand off control to ClientRequestContext
1638 * for a short time.  ClientRequestContext would then delete itself
1639 * and pass control back to ClientHttpRequest when all callouts
1640 * were finished.
1641 *
1642 * This caused some problems for ICAP because we want to make the
1643 * ICAP callout after checking ACLs, but before checking the no_cache
1644 * list.  We can't stuff the ICAP state into the ClientRequestContext
1645 * class because we still need the ICAP state after ClientRequestContext
1646 * goes away.
1647 *
1648 * Note that ClientRequestContext is created before the first call
1649 * to doCallouts().
1650 *
1651 * If one of the callouts notices that ClientHttpRequest is no
1652 * longer valid, it should call cbdataReferenceDone() so that
1653 * ClientHttpRequest's reference count goes to zero and it will get
1654 * deleted.  ClientHttpRequest will then delete ClientRequestContext.
1655 *
1656 * Note that we set the _done flags here before actually starting
1657 * the callout.  This is strictly for convenience.
1658 */
1659
1660tos_t aclMapTOS (acl_tos * head, ACLChecklist * ch);
1661nfmark_t aclMapNfmark (acl_nfmark * head, ACLChecklist * ch);
1662
1663void
1664ClientHttpRequest::doCallouts()
1665{
1666    assert(calloutContext);
1667
1668    /*Save the original request for logging purposes*/
1669    if (!calloutContext->http->al->request) {
1670        calloutContext->http->al->request = request;
1671        HTTPMSGLOCK(calloutContext->http->al->request);
1672
1673        NotePairs &notes = SyncNotes(*calloutContext->http->al, *calloutContext->http->request);
1674        // Make the previously set client connection ID available as annotation.
1675        if (ConnStateData *csd = calloutContext->http->getConn()) {
1676            if (!csd->connectionTag().isEmpty())
1677                notes.add("clt_conn_tag", SBuf(csd->connectionTag()).c_str());
1678        }
1679    }
1680
1681    if (!calloutContext->error) {
1682        // CVE-2009-0801: verify the Host: header is consistent with other known details.
1683        if (!calloutContext->host_header_verify_done) {
1684            debugs(83, 3, HERE << "Doing calloutContext->hostHeaderVerify()");
1685            calloutContext->host_header_verify_done = true;
1686            calloutContext->hostHeaderVerify();
1687            return;
1688        }
1689
1690        if (!calloutContext->http_access_done) {
1691            debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck()");
1692            calloutContext->http_access_done = true;
1693            calloutContext->clientAccessCheck();
1694            return;
1695        }
1696
1697#if USE_ADAPTATION
1698        if (!calloutContext->adaptation_acl_check_done) {
1699            calloutContext->adaptation_acl_check_done = true;
1700            if (Adaptation::AccessCheck::Start(
1701                        Adaptation::methodReqmod, Adaptation::pointPreCache,
1702                        request, NULL, calloutContext->http->al, this))
1703                return; // will call callback
1704        }
1705#endif
1706
1707        if (!calloutContext->redirect_done) {
1708            calloutContext->redirect_done = true;
1709
1710            if (Config.Program.redirect) {
1711                debugs(83, 3, HERE << "Doing calloutContext->clientRedirectStart()");
1712                calloutContext->redirect_state = REDIRECT_PENDING;
1713                calloutContext->clientRedirectStart();
1714                return;
1715            }
1716        }
1717
1718        if (!calloutContext->adapted_http_access_done) {
1719            debugs(83, 3, HERE << "Doing calloutContext->clientAccessCheck2()");
1720            calloutContext->adapted_http_access_done = true;
1721            calloutContext->clientAccessCheck2();
1722            return;
1723        }
1724
1725        if (!calloutContext->store_id_done) {
1726            calloutContext->store_id_done = true;
1727
1728            if (Config.Program.store_id) {
1729                debugs(83, 3,"Doing calloutContext->clientStoreIdStart()");
1730                calloutContext->store_id_state = REDIRECT_PENDING;
1731                calloutContext->clientStoreIdStart();
1732                return;
1733            }
1734        }
1735
1736        if (!calloutContext->interpreted_req_hdrs) {
1737            debugs(83, 3, HERE << "Doing clientInterpretRequestHeaders()");
1738            calloutContext->interpreted_req_hdrs = 1;
1739            clientInterpretRequestHeaders(this);
1740        }
1741
1742        if (!calloutContext->no_cache_done) {
1743            calloutContext->no_cache_done = true;
1744
1745            if (Config.accessList.noCache && request->flags.cachable) {
1746                debugs(83, 3, HERE << "Doing calloutContext->checkNoCache()");
1747                calloutContext->checkNoCache();
1748                return;
1749            }
1750        }
1751    } //  if !calloutContext->error
1752
1753    if (!calloutContext->tosToClientDone) {
1754        calloutContext->tosToClientDone = true;
1755        if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1756            ACLFilledChecklist ch(NULL, request, NULL);
1757            ch.src_addr = request->client_addr;
1758            ch.my_addr = request->my_addr;
1759            tos_t tos = aclMapTOS(Ip::Qos::TheConfig.tosToClient, &ch);
1760            if (tos)
1761                Ip::Qos::setSockTos(getConn()->clientConnection, tos);
1762        }
1763    }
1764
1765    if (!calloutContext->nfmarkToClientDone) {
1766        calloutContext->nfmarkToClientDone = true;
1767        if (getConn() != NULL && Comm::IsConnOpen(getConn()->clientConnection)) {
1768            ACLFilledChecklist ch(NULL, request, NULL);
1769            ch.src_addr = request->client_addr;
1770            ch.my_addr = request->my_addr;
1771            nfmark_t mark = aclMapNfmark(Ip::Qos::TheConfig.nfmarkToClient, &ch);
1772            if (mark)
1773                Ip::Qos::setSockNfmark(getConn()->clientConnection, mark);
1774        }
1775    }
1776
1777#if USE_OPENSSL
1778    // We need to check for SslBump even if the calloutContext->error is set
1779    // because bumping may require delaying the error until after CONNECT.
1780    if (!calloutContext->sslBumpCheckDone) {
1781        calloutContext->sslBumpCheckDone = true;
1782        if (calloutContext->sslBumpAccessCheck())
1783            return;
1784        /* else no ssl bump required*/
1785    }
1786#endif
1787
1788    if (calloutContext->error) {
1789        const char *storeUri = request->storeId();
1790        StoreEntry *e= storeCreateEntry(storeUri, storeUri, request->flags, request->method);
1791#if USE_OPENSSL
1792        if (sslBumpNeeded()) {
1793            // We have to serve an error, so bump the client first.
1794            sslBumpNeed(Ssl::bumpClientFirst);
1795            // set final error but delay sending until we bump
1796            Ssl::ServerBump *srvBump = new Ssl::ServerBump(request, e);
1797            errorAppendEntry(e, calloutContext->error);
1798            calloutContext->error = NULL;
1799            getConn()->setServerBump(srvBump);
1800            e->unlock("ClientHttpRequest::doCallouts+sslBumpNeeded");
1801        } else
1802#endif
1803        {
1804            // send the error to the client now
1805            clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1806            clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1807            assert (repContext);
1808            repContext->setReplyToStoreEntry(e, "immediate SslBump error");
1809            errorAppendEntry(e, calloutContext->error);
1810            calloutContext->error = NULL;
1811            if (calloutContext->readNextRequest)
1812                getConn()->flags.readMore = true; // resume any pipeline reads.
1813            node = (clientStreamNode *)client_stream.tail->data;
1814            clientStreamRead(node, this, node->readBuffer);
1815            e->unlock("ClientHttpRequest::doCallouts-sslBumpNeeded");
1816            return;
1817        }
1818    }
1819
1820    cbdataReferenceDone(calloutContext->http);
1821    delete calloutContext;
1822    calloutContext = NULL;
1823#if HEADERS_LOG
1824
1825    headersLog(0, 1, request->method, request);
1826#endif
1827
1828    debugs(83, 3, HERE << "calling processRequest()");
1829    processRequest();
1830
1831#if ICAP_CLIENT
1832    Adaptation::Icap::History::Pointer ih = request->icapHistory();
1833    if (ih != NULL)
1834        ih->logType = logType;
1835#endif
1836}
1837
1838#if !_USE_INLINE_
1839#include "client_side_request.cci"
1840#endif
1841
1842#if USE_ADAPTATION
1843/// Initiate an asynchronous adaptation transaction which will call us back.
1844void
1845ClientHttpRequest::startAdaptation(const Adaptation::ServiceGroupPointer &g)
1846{
1847    debugs(85, 3, HERE << "adaptation needed for " << this);
1848    assert(!virginHeadSource);
1849    assert(!adaptedBodySource);
1850    virginHeadSource = initiateAdaptation(
1851                           new Adaptation::Iterator(request, NULL, al, g));
1852
1853    // we could try to guess whether we can bypass this adaptation
1854    // initiation failure, but it should not really happen
1855    Must(initiated(virginHeadSource));
1856}
1857
1858void
1859ClientHttpRequest::noteAdaptationAnswer(const Adaptation::Answer &answer)
1860{
1861    assert(cbdataReferenceValid(this));     // indicates bug
1862    clearAdaptation(virginHeadSource);
1863    assert(!adaptedBodySource);
1864
1865    switch (answer.kind) {
1866    case Adaptation::Answer::akForward:
1867        handleAdaptedHeader(const_cast<HttpMsg*>(answer.message.getRaw()));
1868        break;
1869
1870    case Adaptation::Answer::akBlock:
1871        handleAdaptationBlock(answer);
1872        break;
1873
1874    case Adaptation::Answer::akError:
1875        handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_ABORT, !answer.final);
1876        break;
1877    }
1878}
1879
1880void
1881ClientHttpRequest::handleAdaptedHeader(HttpMsg *msg)
1882{
1883    assert(msg);
1884
1885    if (HttpRequest *new_req = dynamic_cast<HttpRequest*>(msg)) {
1886        /*
1887         * Replace the old request with the new request.
1888         */
1889        HTTPMSGUNLOCK(request);
1890        request = new_req;
1891        HTTPMSGLOCK(request);
1892
1893        // update the new message to flag whether URL re-writing was done on it
1894        if (strcmp(urlCanonical(request),uri) != 0)
1895            request->flags.redirected = 1;
1896
1897        /*
1898         * Store the new URI for logging
1899         */
1900        xfree(uri);
1901        uri = xstrdup(urlCanonical(request));
1902        setLogUri(this, urlCanonicalClean(request));
1903        assert(request->method.id());
1904    } else if (HttpReply *new_rep = dynamic_cast<HttpReply*>(msg)) {
1905        debugs(85,3,HERE << "REQMOD reply is HTTP reply");
1906
1907        // subscribe to receive reply body
1908        if (new_rep->body_pipe != NULL) {
1909            adaptedBodySource = new_rep->body_pipe;
1910            int consumer_ok = adaptedBodySource->setConsumerIfNotLate(this);
1911            assert(consumer_ok);
1912        }
1913
1914        clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
1915        clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
1916        assert(repContext);
1917        repContext->createStoreEntry(request->method, request->flags);
1918
1919        EBIT_CLR(storeEntry()->flags, ENTRY_FWD_HDR_WAIT);
1920        request_satisfaction_mode = true;
1921        request_satisfaction_offset = 0;
1922        storeEntry()->replaceHttpReply(new_rep);
1923        storeEntry()->timestampsSet();
1924
1925        if (!adaptedBodySource) // no body
1926            storeEntry()->complete();
1927        clientGetMoreData(node, this);
1928    }
1929
1930    // we are done with getting headers (but may be receiving body)
1931    clearAdaptation(virginHeadSource);
1932
1933    if (!request_satisfaction_mode)
1934        doCallouts();
1935}
1936
1937void
1938ClientHttpRequest::handleAdaptationBlock(const Adaptation::Answer &answer)
1939{
1940    request->detailError(ERR_ACCESS_DENIED, ERR_DETAIL_REQMOD_BLOCK);
1941    AclMatchedName = answer.ruleId.termedBuf();
1942    assert(calloutContext);
1943    calloutContext->clientAccessCheckDone(ACCESS_DENIED);
1944    AclMatchedName = NULL;
1945}
1946
1947void
1948ClientHttpRequest::resumeBodyStorage()
1949{
1950    if (!adaptedBodySource)
1951        return;
1952
1953    noteMoreBodyDataAvailable(adaptedBodySource);
1954}
1955
1956void
1957ClientHttpRequest::noteMoreBodyDataAvailable(BodyPipe::Pointer)
1958{
1959    assert(request_satisfaction_mode);
1960    assert(adaptedBodySource != NULL);
1961
1962    if (size_t contentSize = adaptedBodySource->buf().contentSize()) {
1963        const size_t spaceAvailable = storeEntry()->bytesWanted(Range<size_t>(0,contentSize));
1964
1965        if (spaceAvailable < contentSize ) {
1966            // No or partial body data consuming
1967            typedef NullaryMemFunT<ClientHttpRequest> Dialer;
1968            AsyncCall::Pointer call = asyncCall(93, 5, "ClientHttpRequest::resumeBodyStorage",
1969                                                Dialer(this, &ClientHttpRequest::resumeBodyStorage));
1970            storeEntry()->deferProducer(call);
1971        }
1972
1973        if (!spaceAvailable)
1974            return;
1975
1976        if (spaceAvailable < contentSize )
1977            contentSize = spaceAvailable;
1978
1979        BodyPipeCheckout bpc(*adaptedBodySource);
1980        const StoreIOBuffer ioBuf(&bpc.buf, request_satisfaction_offset, contentSize);
1981        storeEntry()->write(ioBuf);
1982        // assume StoreEntry::write() writes the entire ioBuf
1983        request_satisfaction_offset += ioBuf.length;
1984        bpc.buf.consume(contentSize);
1985        bpc.checkIn();
1986    }
1987
1988    if (adaptedBodySource->exhausted())
1989        endRequestSatisfaction();
1990    // else wait for more body data
1991}
1992
1993void
1994ClientHttpRequest::noteBodyProductionEnded(BodyPipe::Pointer)
1995{
1996    assert(!virginHeadSource);
1997    // should we end request satisfaction now?
1998    if (adaptedBodySource != NULL && adaptedBodySource->exhausted())
1999        endRequestSatisfaction();
2000}
2001
2002void
2003ClientHttpRequest::endRequestSatisfaction()
2004{
2005    debugs(85,4, HERE << this << " ends request satisfaction");
2006    assert(request_satisfaction_mode);
2007    stopConsumingFrom(adaptedBodySource);
2008
2009    // TODO: anything else needed to end store entry formation correctly?
2010    storeEntry()->complete();
2011}
2012
2013void
2014ClientHttpRequest::noteBodyProducerAborted(BodyPipe::Pointer)
2015{
2016    assert(!virginHeadSource);
2017    stopConsumingFrom(adaptedBodySource);
2018
2019    debugs(85,3, HERE << "REQMOD body production failed");
2020    if (request_satisfaction_mode) { // too late to recover or serve an error
2021        request->detailError(ERR_ICAP_FAILURE, ERR_DETAIL_CLT_REQMOD_RESP_BODY);
2022        const Comm::ConnectionPointer c = getConn()->clientConnection;
2023        Must(Comm::IsConnOpen(c));
2024        c->close(); // drastic, but we may be writing a response already
2025    } else {
2026        handleAdaptationFailure(ERR_DETAIL_CLT_REQMOD_REQ_BODY);
2027    }
2028}
2029
2030void
2031ClientHttpRequest::handleAdaptationFailure(int errDetail, bool bypassable)
2032{
2033    debugs(85,3, HERE << "handleAdaptationFailure(" << bypassable << ")");
2034
2035    const bool usedStore = storeEntry() && !storeEntry()->isEmpty();
2036    const bool usedPipe = request->body_pipe != NULL &&
2037                          request->body_pipe->consumedSize() > 0;
2038
2039    if (bypassable && !usedStore && !usedPipe) {
2040        debugs(85,3, HERE << "ICAP REQMOD callout failed, bypassing: " << calloutContext);
2041        if (calloutContext)
2042            doCallouts();
2043        return;
2044    }
2045
2046    debugs(85,3, HERE << "ICAP REQMOD callout failed, responding with error");
2047
2048    clientStreamNode *node = (clientStreamNode *)client_stream.tail->prev->data;
2049    clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
2050    assert(repContext);
2051
2052    // The original author of the code also wanted to pass an errno to
2053    // setReplyToError, but it seems unlikely that the errno reflects the
2054    // true cause of the error at this point, so I did not pass it.
2055    if (calloutContext) {
2056        Ip::Address noAddr;
2057        noAddr.setNoAddr();
2058        ConnStateData * c = getConn();
2059        calloutContext->error = clientBuildError(ERR_ICAP_FAILURE, Http::scInternalServerError,
2060                                NULL,
2061                                c != NULL ? c->clientConnection->remote : noAddr,
2062                                request
2063                                                );
2064#if USE_AUTH
2065        calloutContext->error->auth_user_request =
2066            c != NULL && c->getAuth() != NULL ? c->getAuth() : request->auth_user_request;
2067#endif
2068        calloutContext->error->detailError(errDetail);
2069        calloutContext->readNextRequest = true;
2070        if (c != NULL)
2071            c->expectNoForwarding();
2072        doCallouts();
2073    }
2074    //else if(calloutContext == NULL) is it possible?
2075}
2076
2077#endif
2078
Note: See TracBrowser for help on using the repository browser.