View Javadoc

1   //
2   //  ========================================================================
3   //  Copyright (c) 1995-2013 Mort Bay Consulting Pty. Ltd.
4   //  ------------------------------------------------------------------------
5   //  All rights reserved. This program and the accompanying materials
6   //  are made available under the terms of the Eclipse Public License v1.0
7   //  and Apache License v2.0 which accompanies this distribution.
8   //
9   //      The Eclipse Public License is available at
10  //      http://www.eclipse.org/legal/epl-v10.html
11  //
12  //      The Apache License v2.0 is available at
13  //      http://www.opensource.org/licenses/apache2.0.php
14  //
15  //  You may elect to redistribute this code under either of these licenses.
16  //  ========================================================================
17  //
18  
19  package org.eclipse.jetty.server;
20  
21  import java.util.Arrays;
22  import java.util.Collection;
23  import java.util.Collections;
24  import java.util.concurrent.Executor;
25  import java.util.concurrent.TimeUnit;
26  import java.util.concurrent.atomic.AtomicBoolean;
27  
28  import org.eclipse.jetty.io.EndPoint;
29  import org.eclipse.jetty.util.annotation.ManagedAttribute;
30  import org.eclipse.jetty.util.annotation.ManagedObject;
31  import org.eclipse.jetty.util.annotation.Name;
32  import org.eclipse.jetty.util.component.AbstractLifeCycle;
33  import org.eclipse.jetty.util.log.Log;
34  import org.eclipse.jetty.util.log.Logger;
35  import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
36  import org.eclipse.jetty.util.thread.Scheduler;
37  import org.eclipse.jetty.util.thread.ThreadPool;
38  
39  
40  /* ------------------------------------------------------------ */
41  /** A monitor for low resources
42   * <p>An instance of this class will monitor all the connectors of a server (or a set of connectors
43   * configured with {@link #setMonitoredConnectors(Collection)}) for a low resources state.
44   * Low resources can be detected by:<ul>
45   * <li>{@link ThreadPool#isLowOnThreads()} if {@link Connector#getExecutor()} is
46   * an instance of {@link ThreadPool} and {@link #setMonitorThreads(boolean)} is true.<li>
47   * <li>If {@link #setMaxMemory(long)} is non zero then low resources is detected if the JVMs
48   * {@link Runtime} instance has {@link Runtime#totalMemory()} minus {@link Runtime#freeMemory()}
49   * greater than {@link #getMaxMemory()}</li>
50   * <li>If {@link #setMaxConnections(int)} is non zero then low resources is dected if the total number
51   * of connections exceeds {@link #getMaxConnections()}</li>
52   * </ul>
53   * </p>
54   * <p>Once low resources state is detected, the cause is logged and all existing connections returned
55   * by {@link Connector#getConnectedEndPoints()} have {@link EndPoint#setIdleTimeout(long)} set
56   * to {@link #getLowResourcesIdleTimeout()}.  New connections are not affected, however if the low
57   * resources state persists for more than {@link #getMaxLowResourcesTime()}, then the
58   * {@link #getLowResourcesIdleTimeout()} to all connections again.  Once the low resources state is
59   * cleared, the idle timeout is reset to the connector default given by {@link Connector#getIdleTimeout()}.
60   * </p>
61   */
62  @ManagedObject ("Monitor for low resource conditions and activate a low resource mode if detected")
63  public class LowResourceMonitor extends AbstractLifeCycle
64  {
65      private static final Logger LOG = Log.getLogger(LowResourceMonitor.class);
66      private final Server _server;
67      private Scheduler _scheduler;
68      private Connector[] _monitoredConnectors;
69      private int _period=1000;
70      private int _maxConnections;
71      private long _maxMemory;
72      private int _lowResourcesIdleTimeout=1000;
73      private int _maxLowResourcesTime=0;
74      private boolean _monitorThreads=true;
75      private final AtomicBoolean _low = new AtomicBoolean();
76      private String _cause;
77      private String _reasons;
78      private long _lowStarted;
79  
80  
81      private final Runnable _monitor = new Runnable()
82      {
83          @Override
84          public void run()
85          {
86              if (isRunning())
87              {
88                  monitor();
89                  _scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS);
90              }
91          }
92      };
93  
94      public LowResourceMonitor(@Name("server") Server server)
95      {
96          _server=server;
97      }
98  
99      @ManagedAttribute("Are the monitored connectors low on resources?")
100     public boolean isLowOnResources()
101     {
102         return _low.get();
103     }
104 
105     @ManagedAttribute("The reason(s) the monitored connectors are low on resources")
106     public String getLowResourcesReasons()
107     {
108         return _reasons;
109     }
110 
111     @ManagedAttribute("Get the timestamp in ms since epoch that low resources state started")
112     public long getLowResourcesStarted()
113     {
114         return _lowStarted;
115     }
116 
117     @ManagedAttribute("The monitored connectors. If null then all server connectors are monitored")
118     public Collection<Connector> getMonitoredConnectors()
119     {
120         if (_monitoredConnectors==null)
121             return Collections.emptyList();
122         return Arrays.asList(_monitoredConnectors);
123     }
124 
125     /**
126      * @param monitoredConnectors The collections of Connectors that should be monitored for low resources.
127      */
128     public void setMonitoredConnectors(Collection<Connector> monitoredConnectors)
129     {
130         if (monitoredConnectors==null || monitoredConnectors.size()==0)
131             _monitoredConnectors=null;
132         else
133             _monitoredConnectors = monitoredConnectors.toArray(new Connector[monitoredConnectors.size()]);
134     }
135 
136     @ManagedAttribute("The monitor period in ms")
137     public int getPeriod()
138     {
139         return _period;
140     }
141 
142     /**
143      * @param periodMS The period in ms to monitor for low resources
144      */
145     public void setPeriod(int periodMS)
146     {
147         _period = periodMS;
148     }
149 
150     @ManagedAttribute("True if low available threads status is monitored")
151     public boolean getMonitorThreads()
152     {
153         return _monitorThreads;
154     }
155 
156     /**
157      * @param monitorThreads If true, check connectors executors to see if they are
158      * {@link ThreadPool} instances that are low on threads.
159      */
160     public void setMonitorThreads(boolean monitorThreads)
161     {
162         _monitorThreads = monitorThreads;
163     }
164 
165     @ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated")
166     public int getMaxConnections()
167     {
168         return _maxConnections;
169     }
170 
171     /**
172      * @param maxConnections The maximum connections before low resources state is triggered
173      */
174     public void setMaxConnections(int maxConnections)
175     {
176         _maxConnections = maxConnections;
177     }
178 
179     @ManagedAttribute("The maximum memory (in bytes) that can be used before low resources is triggered.  Memory used is calculated as (totalMemory-freeMemory).")
180     public long getMaxMemory()
181     {
182         return _maxMemory;
183     }
184 
185     /**
186      * @param maxMemoryBytes The maximum memory in bytes in use before low resources is triggered.
187      */
188     public void setMaxMemory(long maxMemoryBytes)
189     {
190         _maxMemory = maxMemoryBytes;
191     }
192 
193     @ManagedAttribute("The idletimeout in ms to apply to all existing connections when low resources is detected")
194     public int getLowResourcesIdleTimeout()
195     {
196         return _lowResourcesIdleTimeout;
197     }
198 
199     /**
200      * @param lowResourcesIdleTimeoutMS The timeout in ms to apply to EndPoints when in the low resources state.
201      */
202     public void setLowResourcesIdleTimeout(int lowResourcesIdleTimeoutMS)
203     {
204         _lowResourcesIdleTimeout = lowResourcesIdleTimeoutMS;
205     }
206 
207     @ManagedAttribute("The maximum time in ms that low resources condition can persist before lowResourcesIdleTimeout is applied to new connections as well as existing connections")
208     public int getMaxLowResourcesTime()
209     {
210         return _maxLowResourcesTime;
211     }
212 
213     /**
214      * @param maxLowResourcesTimeMS The time in milliseconds that a low resource state can persist before the low resource idle timeout is reapplied to all connections
215      */
216     public void setMaxLowResourcesTime(int maxLowResourcesTimeMS)
217     {
218         _maxLowResourcesTime = maxLowResourcesTimeMS;
219     }
220 
221     @Override
222     protected void doStart() throws Exception
223     {
224         _scheduler = _server.getBean(Scheduler.class);
225 
226         if (_scheduler==null)
227         {
228             _scheduler=new LRMScheduler();
229             _scheduler.start();
230         }
231         super.doStart();
232 
233         _scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS);
234     }
235 
236     @Override
237     protected void doStop() throws Exception
238     {
239         if (_scheduler instanceof LRMScheduler)
240             _scheduler.stop();
241         super.doStop();
242     }
243 
244     protected Connector[] getMonitoredOrServerConnectors()
245     {
246         if (_monitoredConnectors!=null && _monitoredConnectors.length>0)
247             return _monitoredConnectors;
248         return _server.getConnectors();
249     }
250 
251     protected void monitor()
252     {
253         String reasons=null;
254         String cause="";
255         int connections=0;
256 
257         for(Connector connector : getMonitoredOrServerConnectors())
258         {
259             connections+=connector.getConnectedEndPoints().size();
260 
261             Executor executor = connector.getExecutor();
262             if (executor instanceof ThreadPool)
263             {
264                 ThreadPool threadpool=(ThreadPool) executor;
265                 if (_monitorThreads && threadpool.isLowOnThreads())
266                 {
267                     reasons=low(reasons,"Low on threads: "+threadpool);
268                     cause+="T";
269                 }
270             }
271         }
272 
273         if (_maxConnections>0 && connections>_maxConnections)
274         {
275             reasons=low(reasons,"Max Connections exceeded: "+connections+">"+_maxConnections);
276             cause+="C";
277         }
278 
279         long memory=Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory();
280         if (_maxMemory>0 && memory>_maxMemory)
281         {
282             reasons=low(reasons,"Max memory exceeded: "+memory+">"+_maxMemory);
283             cause+="M";
284         }
285 
286 
287         if (reasons!=null)
288         {
289             // Log the reasons if there is any change in the cause
290             if (!cause.equals(_cause))
291             {
292                 LOG.warn("Low Resources: {}",reasons);
293                 _cause=cause;
294             }
295 
296             // Enter low resources state?
297             if (_low.compareAndSet(false,true))
298             {
299                 _reasons=reasons;
300                 _lowStarted=System.currentTimeMillis();
301                 setLowResources();
302             }
303 
304             // Too long in low resources state?
305             if (_maxLowResourcesTime>0 && (System.currentTimeMillis()-_lowStarted)>_maxLowResourcesTime)
306                 setLowResources();
307         }
308         else
309         {
310             if (_low.compareAndSet(true,false))
311             {
312                 LOG.info("Low Resources cleared");
313                 _reasons=null;
314                 _lowStarted=0;
315                 _cause=null;
316                 clearLowResources();
317             }
318         }
319     }
320 
321     protected void setLowResources()
322     {
323         for(Connector connector : getMonitoredOrServerConnectors())
324         {
325             for (EndPoint endPoint : connector.getConnectedEndPoints())
326                 endPoint.setIdleTimeout(_lowResourcesIdleTimeout);
327         }
328     }
329 
330     protected void clearLowResources()
331     {
332         for(Connector connector : getMonitoredOrServerConnectors())
333         {
334             for (EndPoint endPoint : connector.getConnectedEndPoints())
335                 endPoint.setIdleTimeout(connector.getIdleTimeout());
336         }
337     }
338 
339     private String low(String reasons, String newReason)
340     {
341         if (reasons==null)
342             return newReason;
343         return reasons+", "+newReason;
344     }
345 
346 
347     private static class LRMScheduler extends ScheduledExecutorScheduler
348     {
349     }
350 }