View Javadoc

1   //
2   //  ========================================================================
3   //  Copyright (c) 1995-2013 Mort Bay Consulting Pty. Ltd.
4   //  ------------------------------------------------------------------------
5   //  All rights reserved. This program and the accompanying materials
6   //  are made available under the terms of the Eclipse Public License v1.0
7   //  and Apache License v2.0 which accompanies this distribution.
8   //
9   //      The Eclipse Public License is available at
10  //      http://www.eclipse.org/legal/epl-v10.html
11  //
12  //      The Apache License v2.0 is available at
13  //      http://www.opensource.org/licenses/apache2.0.php
14  //
15  //  You may elect to redistribute this code under either of these licenses.
16  //  ========================================================================
17  //
18  
19  package org.eclipse.jetty.client;
20  
21  import java.io.IOException;
22  import java.net.ConnectException;
23  import java.net.CookieManager;
24  import java.net.CookiePolicy;
25  import java.net.CookieStore;
26  import java.net.SocketAddress;
27  import java.net.SocketException;
28  import java.net.URI;
29  import java.nio.channels.SelectionKey;
30  import java.nio.channels.SocketChannel;
31  import java.util.ArrayList;
32  import java.util.Arrays;
33  import java.util.Collection;
34  import java.util.HashSet;
35  import java.util.Iterator;
36  import java.util.List;
37  import java.util.Locale;
38  import java.util.Objects;
39  import java.util.Set;
40  import java.util.concurrent.ConcurrentHashMap;
41  import java.util.concurrent.ConcurrentMap;
42  import java.util.concurrent.ExecutionException;
43  import java.util.concurrent.Executor;
44  import java.util.concurrent.Future;
45  import java.util.concurrent.TimeoutException;
46  import javax.net.ssl.SSLEngine;
47  
48  import org.eclipse.jetty.client.api.AuthenticationStore;
49  import org.eclipse.jetty.client.api.Connection;
50  import org.eclipse.jetty.client.api.ContentResponse;
51  import org.eclipse.jetty.client.api.Destination;
52  import org.eclipse.jetty.client.api.ProxyConfiguration;
53  import org.eclipse.jetty.client.api.Request;
54  import org.eclipse.jetty.client.api.Response;
55  import org.eclipse.jetty.http.HttpField;
56  import org.eclipse.jetty.http.HttpHeader;
57  import org.eclipse.jetty.http.HttpMethod;
58  import org.eclipse.jetty.io.ByteBufferPool;
59  import org.eclipse.jetty.io.EndPoint;
60  import org.eclipse.jetty.io.MappedByteBufferPool;
61  import org.eclipse.jetty.io.SelectChannelEndPoint;
62  import org.eclipse.jetty.io.SelectorManager;
63  import org.eclipse.jetty.io.ssl.SslConnection;
64  import org.eclipse.jetty.util.FuturePromise;
65  import org.eclipse.jetty.util.Jetty;
66  import org.eclipse.jetty.util.Promise;
67  import org.eclipse.jetty.util.SocketAddressResolver;
68  import org.eclipse.jetty.util.component.ContainerLifeCycle;
69  import org.eclipse.jetty.util.log.Log;
70  import org.eclipse.jetty.util.log.Logger;
71  import org.eclipse.jetty.util.ssl.SslContextFactory;
72  import org.eclipse.jetty.util.thread.QueuedThreadPool;
73  import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
74  import org.eclipse.jetty.util.thread.Scheduler;
75  
76  /**
77   * <p>{@link HttpClient} provides an efficient, asynchronous, non-blocking implementation
78   * to perform HTTP requests to a server through a simple API that offers also blocking semantic.</p>
79   * <p>{@link HttpClient} provides easy-to-use methods such as {@link #GET(String)} that allow to perform HTTP
80   * requests in a one-liner, but also gives the ability to fine tune the configuration of requests via
81   * {@link HttpClient#newRequest(URI)}.</p>
82   * <p>{@link HttpClient} acts as a central configuration point for network parameters (such as idle timeouts)
83   * and HTTP parameters (such as whether to follow redirects).</p>
84   * <p>{@link HttpClient} transparently pools connections to servers, but allows direct control of connections
85   * for cases where this is needed.</p>
86   * <p>{@link HttpClient} also acts as a central configuration point for cookies, via {@link #getCookieStore()}.</p>
87   * <p>Typical usage:</p>
88   * <pre>
89   * HttpClient httpClient = new HttpClient();
90   * httpClient.start();
91   *
92   * // One liner:
93   * httpClient.GET("http://localhost:8080/").get().status();
94   *
95   * // Building a request with a timeout
96   * Response response = httpClient.newRequest("http://localhost:8080").send().get(5, TimeUnit.SECONDS);
97   * int status = response.status();
98   *
99   * // Asynchronously
100  * httpClient.newRequest("http://localhost:8080").send(new Response.CompleteListener()
101  * {
102  *     &#64;Override
103  *     public void onComplete(Result result)
104  *     {
105  *         ...
106  *     }
107  * });
108  * </pre>
109  */
110 public class HttpClient extends ContainerLifeCycle
111 {
112     private static final Logger LOG = Log.getLogger(HttpClient.class);
113 
114     private final ConcurrentMap<String, HttpDestination> destinations = new ConcurrentHashMap<>();
115     private final ConcurrentMap<Long, HttpConversation> conversations = new ConcurrentHashMap<>();
116     private final List<ProtocolHandler> handlers = new ArrayList<>();
117     private final List<Request.Listener> requestListeners = new ArrayList<>();
118     private final AuthenticationStore authenticationStore = new HttpAuthenticationStore();
119     private final Set<ContentDecoder.Factory> decoderFactories = new ContentDecoderFactorySet();
120     private final SslContextFactory sslContextFactory;
121     private volatile CookieManager cookieManager;
122     private volatile CookieStore cookieStore;
123     private volatile Executor executor;
124     private volatile ByteBufferPool byteBufferPool;
125     private volatile Scheduler scheduler;
126     private volatile SocketAddressResolver resolver;
127     private volatile SelectorManager selectorManager;
128     private volatile HttpField agentField = new HttpField(HttpHeader.USER_AGENT, "Jetty/" + Jetty.VERSION);
129     private volatile boolean followRedirects = true;
130     private volatile int maxConnectionsPerDestination = 64;
131     private volatile int maxRequestsQueuedPerDestination = 1024;
132     private volatile int requestBufferSize = 4096;
133     private volatile int responseBufferSize = 4096;
134     private volatile int maxRedirects = 8;
135     private volatile SocketAddress bindAddress;
136     private volatile long connectTimeout = 15000;
137     private volatile long addressResolutionTimeout = 15000;
138     private volatile long idleTimeout;
139     private volatile boolean tcpNoDelay = true;
140     private volatile boolean dispatchIO = true;
141     private volatile ProxyConfiguration proxyConfig;
142     private volatile HttpField encodingField;
143 
144     /**
145      * Creates a {@link HttpClient} instance that can perform requests to non-TLS destinations only
146      * (that is, requests with the "http" scheme only, and not "https").
147      *
148      * @see #HttpClient(SslContextFactory) to perform requests to TLS destinations.
149      */
150     public HttpClient()
151     {
152         this(null);
153     }
154 
155     /**
156      * Creates a {@link HttpClient} instance that can perform requests to non-TLS and TLS destinations
157      * (that is, both requests with the "http" scheme and with the "https" scheme).
158      *
159      * @param sslContextFactory the {@link SslContextFactory} that manages TLS encryption
160      * @see #getSslContextFactory()
161      */
162     public HttpClient(SslContextFactory sslContextFactory)
163     {
164         this.sslContextFactory = sslContextFactory;
165     }
166 
167     /**
168      * @return the {@link SslContextFactory} that manages TLS encryption
169      * @see #HttpClient(SslContextFactory)
170      */
171     public SslContextFactory getSslContextFactory()
172     {
173         return sslContextFactory;
174     }
175 
176     @Override
177     protected void doStart() throws Exception
178     {
179         if (sslContextFactory != null)
180         {
181             addBean(sslContextFactory);
182             // Avoid to double dispatch when using SSL
183             setDispatchIO(false);
184         }
185 
186         String name = HttpClient.class.getSimpleName() + "@" + hashCode();
187 
188         if (executor == null)
189         {
190             QueuedThreadPool threadPool = new QueuedThreadPool();
191             threadPool.setName(name);
192             executor = threadPool;
193         }
194         addBean(executor);
195 
196         if (byteBufferPool == null)
197             byteBufferPool = new MappedByteBufferPool();
198         addBean(byteBufferPool);
199 
200         if (scheduler == null)
201             scheduler = new ScheduledExecutorScheduler(name + "-scheduler", false);
202         addBean(scheduler);
203 
204         resolver = new SocketAddressResolver(executor, scheduler, getAddressResolutionTimeout());
205 
206         selectorManager = newSelectorManager();
207         selectorManager.setConnectTimeout(getConnectTimeout());
208         addBean(selectorManager);
209 
210         handlers.add(new ContinueProtocolHandler(this));
211         handlers.add(new RedirectProtocolHandler(this));
212         handlers.add(new AuthenticationProtocolHandler(this));
213 
214         decoderFactories.add(new GZIPContentDecoder.Factory());
215 
216         cookieManager = newCookieManager();
217         cookieStore = cookieManager.getCookieStore();
218 
219         super.doStart();
220 
221         LOG.info("Started {}", this);
222     }
223 
224     protected SelectorManager newSelectorManager()
225     {
226         return new ClientSelectorManager(getExecutor(), getScheduler());
227     }
228 
229     private CookieManager newCookieManager()
230     {
231         return new CookieManager(getCookieStore(), CookiePolicy.ACCEPT_ALL);
232     }
233 
234     @Override
235     protected void doStop() throws Exception
236     {
237         LOG.debug("Stopping {}", this);
238 
239         cookieStore.removeAll();
240         cookieStore = null;
241         decoderFactories.clear();
242         handlers.clear();
243 
244         for (HttpDestination destination : destinations.values())
245             destination.close();
246         destinations.clear();
247 
248         conversations.clear();
249         requestListeners.clear();
250         authenticationStore.clearAuthentications();
251         authenticationStore.clearAuthenticationResults();
252 
253         super.doStop();
254 
255         LOG.info("Stopped {}", this);
256     }
257 
258     /**
259      * Returns a <em>non</em> thread-safe list of {@link Request.Listener}s that can be modified before
260      * performing requests.
261      *
262      * @return a list of {@link Request.Listener} that can be used to add and remove listeners
263      */
264     public List<Request.Listener> getRequestListeners()
265     {
266         return requestListeners;
267     }
268 
269     /**
270      * @return the cookie store associated with this instance
271      */
272     public CookieStore getCookieStore()
273     {
274         return cookieStore;
275     }
276 
277     /**
278      * @param cookieStore the cookie store associated with this instance
279      */
280     public void setCookieStore(CookieStore cookieStore)
281     {
282         this.cookieStore = Objects.requireNonNull(cookieStore);
283         this.cookieManager = newCookieManager();
284     }
285 
286     /**
287      * Keep this method package-private because its interface is so ugly
288      * that we really don't want to expose it more than strictly needed.
289      *
290      * @return the cookie manager
291      */
292     CookieManager getCookieManager()
293     {
294         return cookieManager;
295     }
296 
297     /**
298      * @return the authentication store associated with this instance
299      */
300     public AuthenticationStore getAuthenticationStore()
301     {
302         return authenticationStore;
303     }
304 
305     /**
306      * Returns a <em>non</em> thread-safe set of {@link ContentDecoder.Factory}s that can be modified before
307      * performing requests.
308      *
309      * @return a set of {@link ContentDecoder.Factory} that can be used to add and remove content decoder factories
310      */
311     public Set<ContentDecoder.Factory> getContentDecoderFactories()
312     {
313         return decoderFactories;
314     }
315 
316     /**
317      * Performs a GET request to the specified URI.
318      *
319      * @param uri the URI to GET
320      * @return the {@link ContentResponse} for the request
321      * @see #GET(URI)
322      */
323     public ContentResponse GET(String uri) throws InterruptedException, ExecutionException, TimeoutException
324     {
325         return GET(URI.create(uri));
326     }
327 
328     /**
329      * Performs a GET request to the specified URI.
330      *
331      * @param uri the URI to GET
332      * @return the {@link ContentResponse} for the request
333      * @see #newRequest(URI)
334      */
335     public ContentResponse GET(URI uri) throws InterruptedException, ExecutionException, TimeoutException
336     {
337         return newRequest(uri).send();
338     }
339 
340     /**
341      * Creates a POST request to the specified URI.
342      *
343      * @param uri the URI to POST to
344      * @return the POST request
345      * @see #POST(URI)
346      */
347     public Request POST(String uri)
348     {
349         return POST(URI.create(uri));
350     }
351 
352     /**
353      * Creates a POST request to the specified URI.
354      *
355      * @param uri the URI to POST to
356      * @return the POST request
357      */
358     public Request POST(URI uri)
359     {
360         return newRequest(uri).method(HttpMethod.POST);
361     }
362 
363     /**
364      * Creates a new request with the "http" scheme and the specified host and port
365      *
366      * @param host the request host
367      * @param port the request port
368      * @return the request just created
369      */
370     public Request newRequest(String host, int port)
371     {
372         return newRequest(URI.create(address("http", host, port)));
373     }
374 
375     /**
376      * Creates a new request with the specified URI.
377      *
378      * @param uri the URI to request
379      * @return the request just created
380      */
381     public Request newRequest(String uri)
382     {
383         return newRequest(URI.create(uri));
384     }
385 
386     /**
387      * Creates a new request with the specified URI.
388      *
389      * @param uri the URI to request
390      * @return the request just created
391      */
392     public Request newRequest(URI uri)
393     {
394         return new HttpRequest(this, uri);
395     }
396 
397     protected Request copyRequest(Request oldRequest, URI newURI)
398     {
399         Request newRequest = new HttpRequest(this, oldRequest.getConversationID(), newURI);
400         newRequest.method(oldRequest.getMethod())
401                 .version(oldRequest.getVersion())
402                 .content(oldRequest.getContent());
403         for (HttpField header : oldRequest.getHeaders())
404         {
405             // We have a new URI, so skip the host header if present
406             if (HttpHeader.HOST == header.getHeader())
407                 continue;
408 
409             // Remove expectation headers
410             if (HttpHeader.EXPECT == header.getHeader())
411                 continue;
412 
413             // Remove cookies
414             if (HttpHeader.COOKIE == header.getHeader())
415                 continue;
416 
417             // Remove authorization headers
418             if (HttpHeader.AUTHORIZATION == header.getHeader() ||
419                     HttpHeader.PROXY_AUTHORIZATION == header.getHeader())
420                 continue;
421 
422             newRequest.header(header.getName(), header.getValue());
423         }
424         return newRequest;
425     }
426 
427     private String address(String scheme, String host, int port)
428     {
429         return scheme + "://" + host + ":" + port;
430     }
431 
432     /**
433      * Returns a {@link Destination} for the given scheme, host and port.
434      * Applications may use {@link Destination}s to create {@link Connection}s
435      * that will be outside {@link HttpClient}'s pooling mechanism, to explicitly
436      * control the connection lifecycle (in particular their termination with
437      * {@link Connection#close()}).
438      *
439      * @param scheme the destination scheme
440      * @param host the destination host
441      * @param port the destination port
442      * @return the destination
443      * @see #getDestinations()
444      */
445     public Destination getDestination(String scheme, String host, int port)
446     {
447         return destinationFor(scheme, host, port);
448     }
449 
450     protected HttpDestination destinationFor(String scheme, String host, int port)
451     {
452         port = normalizePort(scheme, port);
453 
454         String address = address(scheme, host, port);
455         HttpDestination destination = destinations.get(address);
456         if (destination == null)
457         {
458             destination = new HttpDestination(this, scheme, host, port);
459             if (isRunning())
460             {
461                 HttpDestination existing = destinations.putIfAbsent(address, destination);
462                 if (existing != null)
463                     destination = existing;
464                 else
465                     LOG.debug("Created {}", destination);
466                 if (!isRunning())
467                     destinations.remove(address);
468             }
469 
470         }
471         return destination;
472     }
473 
474     /**
475      * @return the list of destinations known to this {@link HttpClient}.
476      */
477     public List<Destination> getDestinations()
478     {
479         return new ArrayList<Destination>(destinations.values());
480     }
481 
482     protected void send(final Request request, List<Response.ResponseListener> listeners)
483     {
484         String scheme = request.getScheme().toLowerCase(Locale.ENGLISH);
485         if (!Arrays.asList("http", "https").contains(scheme))
486             throw new IllegalArgumentException("Invalid protocol " + scheme);
487 
488         HttpDestination destination = destinationFor(scheme, request.getHost(), request.getPort());
489         destination.send(request, listeners);
490     }
491 
492     protected void newConnection(final HttpDestination destination, final Promise<Connection> promise)
493     {
494         Destination.Address address = destination.getConnectAddress();
495         resolver.resolve(address.getHost(), address.getPort(), new Promise<SocketAddress>()
496         {
497             @Override
498             public void succeeded(SocketAddress socketAddress)
499             {
500                 SocketChannel channel = null;
501                 try
502                 {
503                     channel = SocketChannel.open();
504                     SocketAddress bindAddress = getBindAddress();
505                     if (bindAddress != null)
506                         channel.bind(bindAddress);
507                     configure(channel);
508                     channel.configureBlocking(false);
509                     channel.connect(socketAddress);
510 
511                     Future<Connection> futureConnection = new ConnectionCallback(destination, promise);
512                     selectorManager.connect(channel, futureConnection);
513                 }
514                 // Must catch all exceptions, since some like
515                 // UnresolvedAddressException are not IOExceptions.
516                 catch (Throwable x)
517                 {
518                     if (channel != null)
519                         close(channel);
520                     promise.failed(x);
521                 }
522             }
523 
524             @Override
525             public void failed(Throwable x)
526             {
527                 promise.failed(x);
528             }
529         });
530     }
531 
532     protected void configure(SocketChannel channel) throws SocketException
533     {
534         channel.socket().setTcpNoDelay(isTCPNoDelay());
535     }
536 
537     private void close(SocketChannel channel)
538     {
539         try
540         {
541             channel.close();
542         }
543         catch (IOException x)
544         {
545             LOG.ignore(x);
546         }
547     }
548 
549     protected HttpConversation getConversation(long id, boolean create)
550     {
551         HttpConversation conversation = conversations.get(id);
552         if (conversation == null && create)
553         {
554             conversation = new HttpConversation(this, id);
555             HttpConversation existing = conversations.putIfAbsent(id, conversation);
556             if (existing != null)
557                 conversation = existing;
558             else
559                 LOG.debug("{} created", conversation);
560         }
561         return conversation;
562     }
563 
564     protected void removeConversation(HttpConversation conversation)
565     {
566         conversations.remove(conversation.getID());
567         LOG.debug("{} removed", conversation);
568     }
569 
570     protected List<ProtocolHandler> getProtocolHandlers()
571     {
572         return handlers;
573     }
574 
575     protected ProtocolHandler findProtocolHandler(Request request, Response response)
576     {
577         // Optimized to avoid allocations of iterator instances
578         List<ProtocolHandler> protocolHandlers = getProtocolHandlers();
579         for (int i = 0; i < protocolHandlers.size(); ++i)
580         {
581             ProtocolHandler handler = protocolHandlers.get(i);
582             if (handler.accept(request, response))
583                 return handler;
584         }
585         return null;
586     }
587 
588     /**
589      * @return the {@link ByteBufferPool} of this {@link HttpClient}
590      */
591     public ByteBufferPool getByteBufferPool()
592     {
593         return byteBufferPool;
594     }
595 
596     /**
597      * @param byteBufferPool the {@link ByteBufferPool} of this {@link HttpClient}
598      */
599     public void setByteBufferPool(ByteBufferPool byteBufferPool)
600     {
601         this.byteBufferPool = byteBufferPool;
602     }
603 
604     /**
605      * @return the max time a connection can take to connect to destinations
606      */
607     public long getConnectTimeout()
608     {
609         return connectTimeout;
610     }
611 
612     /**
613      * @param connectTimeout the max time a connection can take to connect to destinations
614      * @see java.net.Socket#connect(SocketAddress, int)
615      */
616     public void setConnectTimeout(long connectTimeout)
617     {
618         this.connectTimeout = connectTimeout;
619     }
620 
621     /**
622      * @return the timeout, in milliseconds, for the DNS resolution of host addresses
623      */
624     public long getAddressResolutionTimeout()
625     {
626         return addressResolutionTimeout;
627     }
628 
629     /**
630      * @param addressResolutionTimeout the timeout, in milliseconds, for the DNS resolution of host addresses
631      */
632     public void setAddressResolutionTimeout(long addressResolutionTimeout)
633     {
634         this.addressResolutionTimeout = addressResolutionTimeout;
635     }
636 
637     /**
638      * @return the max time a connection can be idle (that is, without traffic of bytes in either direction)
639      */
640     public long getIdleTimeout()
641     {
642         return idleTimeout;
643     }
644 
645     /**
646      * @param idleTimeout the max time a connection can be idle (that is, without traffic of bytes in either direction)
647      */
648     public void setIdleTimeout(long idleTimeout)
649     {
650         this.idleTimeout = idleTimeout;
651     }
652 
653     /**
654      * @return the address to bind socket channels to
655      * @see #setBindAddress(SocketAddress)
656      */
657     public SocketAddress getBindAddress()
658     {
659         return bindAddress;
660     }
661 
662     /**
663      * @param bindAddress the address to bind socket channels to
664      * @see #getBindAddress()
665      * @see SocketChannel#bind(SocketAddress)
666      */
667     public void setBindAddress(SocketAddress bindAddress)
668     {
669         this.bindAddress = bindAddress;
670     }
671 
672     /**
673      * @return the "User-Agent" HTTP field of this {@link HttpClient}
674      */
675     public HttpField getUserAgentField()
676     {
677         return agentField;
678     }
679 
680     /**
681      * @param agent the "User-Agent" HTTP header string of this {@link HttpClient}
682      */
683     public void setUserAgentField(HttpField agent)
684     {
685         if (agent.getHeader() != HttpHeader.USER_AGENT)
686             throw new IllegalArgumentException();
687         this.agentField = agent;
688     }
689 
690     /**
691      * @return whether this {@link HttpClient} follows HTTP redirects
692      * @see Request#isFollowRedirects()
693      */
694     public boolean isFollowRedirects()
695     {
696         return followRedirects;
697     }
698 
699     /**
700      * @param follow whether this {@link HttpClient} follows HTTP redirects
701      * @see #setMaxRedirects(int)
702      */
703     public void setFollowRedirects(boolean follow)
704     {
705         this.followRedirects = follow;
706     }
707 
708     /**
709      * @return the {@link Executor} of this {@link HttpClient}
710      */
711     public Executor getExecutor()
712     {
713         return executor;
714     }
715 
716     /**
717      * @param executor the {@link Executor} of this {@link HttpClient}
718      */
719     public void setExecutor(Executor executor)
720     {
721         this.executor = executor;
722     }
723 
724     /**
725      * @return the {@link Scheduler} of this {@link HttpClient}
726      */
727     public Scheduler getScheduler()
728     {
729         return scheduler;
730     }
731 
732     /**
733      * @param scheduler the {@link Scheduler} of this {@link HttpClient}
734      */
735     public void setScheduler(Scheduler scheduler)
736     {
737         this.scheduler = scheduler;
738     }
739 
740     protected SelectorManager getSelectorManager()
741     {
742         return selectorManager;
743     }
744 
745     /**
746      * @return the max number of connections that this {@link HttpClient} opens to {@link Destination}s
747      */
748     public int getMaxConnectionsPerDestination()
749     {
750         return maxConnectionsPerDestination;
751     }
752 
753     /**
754      * Sets the max number of connections to open to each destinations.
755      * <p />
756      * RFC 2616 suggests that 2 connections should be opened per each destination,
757      * but browsers commonly open 6.
758      * If this {@link HttpClient} is used for load testing, it is common to have only one destination
759      * (the server to load test), and it is recommended to set this value to a high value (at least as
760      * much as the threads present in the {@link #getExecutor() executor}).
761      *
762      * @param maxConnectionsPerDestination the max number of connections that this {@link HttpClient} opens to {@link Destination}s
763      */
764     public void setMaxConnectionsPerDestination(int maxConnectionsPerDestination)
765     {
766         this.maxConnectionsPerDestination = maxConnectionsPerDestination;
767     }
768 
769     /**
770      * @return the max number of requests that may be queued to a {@link Destination}.
771      */
772     public int getMaxRequestsQueuedPerDestination()
773     {
774         return maxRequestsQueuedPerDestination;
775     }
776 
777     /**
778      * Sets the max number of requests that may be queued to a destination.
779      * <p />
780      * If this {@link HttpClient} performs a high rate of requests to a destination,
781      * and all the connections managed by that destination are busy with other requests,
782      * then new requests will be queued up in the destination.
783      * This parameter controls how many requests can be queued before starting to reject them.
784      * If this {@link HttpClient} is used for load testing, it is common to have this parameter
785      * set to a high value, although this may impact latency (requests sit in the queue for a long
786      * time before being sent).
787      *
788      * @param maxRequestsQueuedPerDestination the max number of requests that may be queued to a {@link Destination}.
789      */
790     public void setMaxRequestsQueuedPerDestination(int maxRequestsQueuedPerDestination)
791     {
792         this.maxRequestsQueuedPerDestination = maxRequestsQueuedPerDestination;
793     }
794 
795     /**
796      * @return the size of the buffer used to write requests
797      */
798     public int getRequestBufferSize()
799     {
800         return requestBufferSize;
801     }
802 
803     /**
804      * @param requestBufferSize the size of the buffer used to write requests
805      */
806     public void setRequestBufferSize(int requestBufferSize)
807     {
808         this.requestBufferSize = requestBufferSize;
809     }
810 
811     /**
812      * @return the size of the buffer used to read responses
813      */
814     public int getResponseBufferSize()
815     {
816         return responseBufferSize;
817     }
818 
819     /**
820      * @param responseBufferSize the size of the buffer used to read responses
821      */
822     public void setResponseBufferSize(int responseBufferSize)
823     {
824         this.responseBufferSize = responseBufferSize;
825     }
826 
827     /**
828      * @return the max number of HTTP redirects that are followed
829      * @see #setMaxRedirects(int)
830      */
831     public int getMaxRedirects()
832     {
833         return maxRedirects;
834     }
835 
836     /**
837      * @param maxRedirects the max number of HTTP redirects that are followed
838      * @see #setFollowRedirects(boolean)
839      */
840     public void setMaxRedirects(int maxRedirects)
841     {
842         this.maxRedirects = maxRedirects;
843     }
844 
845     /**
846      * @return whether TCP_NODELAY is enabled
847      */
848     public boolean isTCPNoDelay()
849     {
850         return tcpNoDelay;
851     }
852 
853     /**
854      * @param tcpNoDelay whether TCP_NODELAY is enabled
855      * @see java.net.Socket#setTcpNoDelay(boolean)
856      */
857     public void setTCPNoDelay(boolean tcpNoDelay)
858     {
859         this.tcpNoDelay = tcpNoDelay;
860     }
861 
862     /**
863      * @return true to dispatch I/O operations in a different thread, false to execute them in the selector thread
864      * @see #setDispatchIO(boolean)
865      */
866     public boolean isDispatchIO()
867     {
868         return dispatchIO;
869     }
870 
871     /**
872      * Whether to dispatch I/O operations from the selector thread to a different thread.
873      * <p />
874      * This implementation never blocks on I/O operation, but invokes application callbacks that may
875      * take time to execute or block on other I/O.
876      * If application callbacks are known to take time or block on I/O, then parameter {@code dispatchIO}
877      * should be set to true.
878      * If application callbacks are known to be quick and never block on I/O, then parameter {@code dispatchIO}
879      * may be set to false.
880      *
881      * @param dispatchIO true to dispatch I/O operations in a different thread,
882      *                   false to execute them in the selector thread
883      */
884     public void setDispatchIO(boolean dispatchIO)
885     {
886         this.dispatchIO = dispatchIO;
887     }
888 
889     /**
890      * @return the forward proxy configuration
891      */
892     public ProxyConfiguration getProxyConfiguration()
893     {
894         return proxyConfig;
895     }
896 
897     /**
898      * @param proxyConfig the forward proxy configuration
899      */
900     public void setProxyConfiguration(ProxyConfiguration proxyConfig)
901     {
902         this.proxyConfig = proxyConfig;
903     }
904 
905     protected HttpField getAcceptEncodingField()
906     {
907         return encodingField;
908     }
909 
910     protected int normalizePort(String scheme, int port)
911     {
912         return port > 0 ? port :
913                 "https".equalsIgnoreCase(scheme) ? 443 : 80;
914     }
915 
916     protected HttpConnection newHttpConnection(HttpClient httpClient, EndPoint endPoint, HttpDestination destination)
917     {
918         return new HttpConnection(httpClient, endPoint, destination);
919     }
920 
921     protected SslConnection newSslConnection(HttpClient httpClient, EndPoint endPoint, SSLEngine engine)
922     {
923         return new SslConnection(httpClient.getByteBufferPool(), httpClient.getExecutor(), endPoint, engine);
924     }
925 
926     @Override
927     public void dump(Appendable out, String indent) throws IOException
928     {
929         dumpThis(out);
930         dump(out, indent, getBeans(), destinations.values());
931     }
932 
933     protected class ClientSelectorManager extends SelectorManager
934     {
935         public ClientSelectorManager(Executor executor, Scheduler scheduler)
936         {
937             this(executor, scheduler, 1);
938         }
939 
940         public ClientSelectorManager(Executor executor, Scheduler scheduler, int selectors)
941         {
942             super(executor, scheduler, selectors);
943         }
944 
945         @Override
946         protected EndPoint newEndPoint(SocketChannel channel, ManagedSelector selector, SelectionKey key)
947         {
948             return new SelectChannelEndPoint(channel, selector, key, getScheduler(), getIdleTimeout());
949         }
950 
951         @Override
952         public org.eclipse.jetty.io.Connection newConnection(SocketChannel channel, EndPoint endPoint, Object attachment) throws IOException
953         {
954             ConnectionCallback callback = (ConnectionCallback)attachment;
955             HttpDestination destination = callback.destination;
956 
957             SslContextFactory sslContextFactory = getSslContextFactory();
958             if ("https".equals(destination.getScheme()))
959             {
960                 if (sslContextFactory == null)
961                 {
962                     IOException failure = new ConnectException("Missing " + SslContextFactory.class.getSimpleName() + " for " + destination.getScheme() + " requests");
963                     callback.failed(failure);
964                     throw failure;
965                 }
966                 else
967                 {
968                     SSLEngine engine = sslContextFactory.newSSLEngine(destination.getHost(), destination.getPort());
969                     engine.setUseClientMode(true);
970 
971                     SslConnection sslConnection = newSslConnection(HttpClient.this, endPoint, engine);
972                     EndPoint appEndPoint = sslConnection.getDecryptedEndPoint();
973                     HttpConnection connection = newHttpConnection(HttpClient.this, appEndPoint, destination);
974 
975                     appEndPoint.setConnection(connection);
976                     callback.promise.succeeded(connection);
977 
978                     return sslConnection;
979                 }
980             }
981             else
982             {
983                 HttpConnection connection = newHttpConnection(HttpClient.this, endPoint, destination);
984                 callback.promise.succeeded(connection);
985                 return connection;
986             }
987         }
988 
989         @Override
990         protected void connectionFailed(SocketChannel channel, Throwable ex, Object attachment)
991         {
992             ConnectionCallback callback = (ConnectionCallback)attachment;
993             callback.promise.failed(ex);
994         }
995     }
996 
997     private class ConnectionCallback extends FuturePromise<Connection>
998     {
999         private final HttpDestination destination;
1000         private final Promise<Connection> promise;
1001 
1002         private ConnectionCallback(HttpDestination destination, Promise<Connection> promise)
1003         {
1004             this.destination = destination;
1005             this.promise = promise;
1006         }
1007     }
1008 
1009     private class ContentDecoderFactorySet implements Set<ContentDecoder.Factory>
1010     {
1011         private final Set<ContentDecoder.Factory> set = new HashSet<>();
1012 
1013         @Override
1014         public boolean add(ContentDecoder.Factory e)
1015         {
1016             boolean result = set.add(e);
1017             invalidate();
1018             return result;
1019         }
1020 
1021         @Override
1022         public boolean addAll(Collection<? extends ContentDecoder.Factory> c)
1023         {
1024             boolean result = set.addAll(c);
1025             invalidate();
1026             return result;
1027         }
1028 
1029         @Override
1030         public boolean remove(Object o)
1031         {
1032             boolean result = set.remove(o);
1033             invalidate();
1034             return result;
1035         }
1036 
1037         @Override
1038         public boolean removeAll(Collection<?> c)
1039         {
1040             boolean result = set.removeAll(c);
1041             invalidate();
1042             return result;
1043         }
1044 
1045         @Override
1046         public boolean retainAll(Collection<?> c)
1047         {
1048             boolean result = set.retainAll(c);
1049             invalidate();
1050             return result;
1051         }
1052 
1053         @Override
1054         public void clear()
1055         {
1056             set.clear();
1057             invalidate();
1058         }
1059 
1060         @Override
1061         public int size()
1062         {
1063             return set.size();
1064         }
1065 
1066         @Override
1067         public boolean isEmpty()
1068         {
1069             return set.isEmpty();
1070         }
1071 
1072         @Override
1073         public boolean contains(Object o)
1074         {
1075             return set.contains(o);
1076         }
1077 
1078         @Override
1079         public boolean containsAll(Collection<?> c)
1080         {
1081             return set.containsAll(c);
1082         }
1083 
1084         @Override
1085         public Iterator<ContentDecoder.Factory> iterator()
1086         {
1087             return set.iterator();
1088         }
1089 
1090         @Override
1091         public Object[] toArray()
1092         {
1093             return set.toArray();
1094         }
1095 
1096         @Override
1097         public <T> T[] toArray(T[] a)
1098         {
1099             return set.toArray(a);
1100         }
1101 
1102         protected void invalidate()
1103         {
1104             if (set.isEmpty())
1105             {
1106                 encodingField = null;
1107             }
1108             else
1109             {
1110                 StringBuilder value = new StringBuilder();
1111                 for (Iterator<ContentDecoder.Factory> iterator = set.iterator(); iterator.hasNext();)
1112                 {
1113                     ContentDecoder.Factory decoderFactory = iterator.next();
1114                     value.append(decoderFactory.getEncoding());
1115                     if (iterator.hasNext())
1116                         value.append(",");
1117                 }
1118                 encodingField = new HttpField(HttpHeader.ACCEPT_ENCODING, value.toString());
1119             }
1120         }
1121     }
1122 }