1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.jetty.server;
20
21 import java.util.Arrays;
22 import java.util.Collection;
23 import java.util.Collections;
24 import java.util.concurrent.Executor;
25 import java.util.concurrent.TimeUnit;
26 import java.util.concurrent.atomic.AtomicBoolean;
27
28 import org.eclipse.jetty.io.EndPoint;
29 import org.eclipse.jetty.util.annotation.ManagedAttribute;
30 import org.eclipse.jetty.util.annotation.ManagedObject;
31 import org.eclipse.jetty.util.annotation.Name;
32 import org.eclipse.jetty.util.component.AbstractLifeCycle;
33 import org.eclipse.jetty.util.log.Log;
34 import org.eclipse.jetty.util.log.Logger;
35 import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
36 import org.eclipse.jetty.util.thread.Scheduler;
37 import org.eclipse.jetty.util.thread.ThreadPool;
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64 @ManagedObject ("Monitor for low resource conditions and activate a low resource mode if detected")
65 public class LowResourceMonitor extends AbstractLifeCycle
66 {
67 private static final Logger LOG = Log.getLogger(LowResourceMonitor.class);
68 private final Server _server;
69 private Scheduler _scheduler;
70 private Connector[] _monitoredConnectors;
71 private int _period=1000;
72 private int _maxConnections;
73 private long _maxMemory;
74 private int _lowResourcesIdleTimeout=1000;
75 private int _maxLowResourcesTime=0;
76 private boolean _monitorThreads=true;
77 private final AtomicBoolean _low = new AtomicBoolean();
78 private String _cause;
79 private String _reasons;
80 private long _lowStarted;
81
82
83 private final Runnable _monitor = new Runnable()
84 {
85 @Override
86 public void run()
87 {
88 if (isRunning())
89 {
90 monitor();
91 _scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS);
92 }
93 }
94 };
95
96 public LowResourceMonitor(@Name("server") Server server)
97 {
98 _server=server;
99 }
100
101 @ManagedAttribute("Are the monitored connectors low on resources?")
102 public boolean isLowOnResources()
103 {
104 return _low.get();
105 }
106
107 @ManagedAttribute("The reason(s) the monitored connectors are low on resources")
108 public String getLowResourcesReasons()
109 {
110 return _reasons;
111 }
112
113 @ManagedAttribute("Get the timestamp in ms since epoch that low resources state started")
114 public long getLowResourcesStarted()
115 {
116 return _lowStarted;
117 }
118
119 @ManagedAttribute("The monitored connectors. If null then all server connectors are monitored")
120 public Collection<Connector> getMonitoredConnectors()
121 {
122 if (_monitoredConnectors==null)
123 return Collections.emptyList();
124 return Arrays.asList(_monitoredConnectors);
125 }
126
127
128
129
130 public void setMonitoredConnectors(Collection<Connector> monitoredConnectors)
131 {
132 if (monitoredConnectors==null || monitoredConnectors.size()==0)
133 _monitoredConnectors=null;
134 else
135 _monitoredConnectors = monitoredConnectors.toArray(new Connector[monitoredConnectors.size()]);
136 }
137
138 @ManagedAttribute("The monitor period in ms")
139 public int getPeriod()
140 {
141 return _period;
142 }
143
144
145
146
147 public void setPeriod(int periodMS)
148 {
149 _period = periodMS;
150 }
151
152 @ManagedAttribute("True if low available threads status is monitored")
153 public boolean getMonitorThreads()
154 {
155 return _monitorThreads;
156 }
157
158
159
160
161
162 public void setMonitorThreads(boolean monitorThreads)
163 {
164 _monitorThreads = monitorThreads;
165 }
166
167 @ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated")
168 public int getMaxConnections()
169 {
170 return _maxConnections;
171 }
172
173
174
175
176 public void setMaxConnections(int maxConnections)
177 {
178 _maxConnections = maxConnections;
179 }
180
181 @ManagedAttribute("The maximum memory (in bytes) that can be used before low resources is triggered. Memory used is calculated as (totalMemory-freeMemory).")
182 public long getMaxMemory()
183 {
184 return _maxMemory;
185 }
186
187
188
189
190 public void setMaxMemory(long maxMemoryBytes)
191 {
192 _maxMemory = maxMemoryBytes;
193 }
194
195 @ManagedAttribute("The idletimeout in ms to apply to all existing connections when low resources is detected")
196 public int getLowResourcesIdleTimeout()
197 {
198 return _lowResourcesIdleTimeout;
199 }
200
201
202
203
204 public void setLowResourcesIdleTimeout(int lowResourcesIdleTimeoutMS)
205 {
206 _lowResourcesIdleTimeout = lowResourcesIdleTimeoutMS;
207 }
208
209 @ManagedAttribute("The maximum time in ms that low resources condition can persist before lowResourcesIdleTimeout is applied to new connections as well as existing connections")
210 public int getMaxLowResourcesTime()
211 {
212 return _maxLowResourcesTime;
213 }
214
215
216
217
218 public void setMaxLowResourcesTime(int maxLowResourcesTimeMS)
219 {
220 _maxLowResourcesTime = maxLowResourcesTimeMS;
221 }
222
223 @Override
224 protected void doStart() throws Exception
225 {
226 _scheduler = _server.getBean(Scheduler.class);
227
228 if (_scheduler==null)
229 {
230 _scheduler=new LRMScheduler();
231 _scheduler.start();
232 }
233 super.doStart();
234
235 _scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS);
236 }
237
238 @Override
239 protected void doStop() throws Exception
240 {
241 if (_scheduler instanceof LRMScheduler)
242 _scheduler.stop();
243 super.doStop();
244 }
245
246 protected Connector[] getMonitoredOrServerConnectors()
247 {
248 if (_monitoredConnectors!=null && _monitoredConnectors.length>0)
249 return _monitoredConnectors;
250 return _server.getConnectors();
251 }
252
253 protected void monitor()
254 {
255 String reasons=null;
256 String cause="";
257 int connections=0;
258
259 for(Connector connector : getMonitoredOrServerConnectors())
260 {
261 connections+=connector.getConnectedEndPoints().size();
262
263 Executor executor = connector.getExecutor();
264 if (executor instanceof ThreadPool)
265 {
266 ThreadPool threadpool=(ThreadPool) executor;
267 if (_monitorThreads && threadpool.isLowOnThreads())
268 {
269 reasons=low(reasons,"Low on threads: "+threadpool);
270 cause+="T";
271 }
272 }
273 }
274
275 if (_maxConnections>0 && connections>_maxConnections)
276 {
277 reasons=low(reasons,"Max Connections exceeded: "+connections+">"+_maxConnections);
278 cause+="C";
279 }
280
281 long memory=Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory();
282 if (_maxMemory>0 && memory>_maxMemory)
283 {
284 reasons=low(reasons,"Max memory exceeded: "+memory+">"+_maxMemory);
285 cause+="M";
286 }
287
288
289 if (reasons!=null)
290 {
291
292 if (!cause.equals(_cause))
293 {
294 LOG.warn("Low Resources: {}",reasons);
295 _cause=cause;
296 }
297
298
299 if (_low.compareAndSet(false,true))
300 {
301 _reasons=reasons;
302 _lowStarted=System.currentTimeMillis();
303 setLowResources();
304 }
305
306
307 if (_maxLowResourcesTime>0 && (System.currentTimeMillis()-_lowStarted)>_maxLowResourcesTime)
308 setLowResources();
309 }
310 else
311 {
312 if (_low.compareAndSet(true,false))
313 {
314 LOG.info("Low Resources cleared");
315 _reasons=null;
316 _lowStarted=0;
317 _cause=null;
318 clearLowResources();
319 }
320 }
321 }
322
323 protected void setLowResources()
324 {
325 for(Connector connector : getMonitoredOrServerConnectors())
326 {
327 for (EndPoint endPoint : connector.getConnectedEndPoints())
328 endPoint.setIdleTimeout(_lowResourcesIdleTimeout);
329 }
330 }
331
332 protected void clearLowResources()
333 {
334 for(Connector connector : getMonitoredOrServerConnectors())
335 {
336 for (EndPoint endPoint : connector.getConnectedEndPoints())
337 endPoint.setIdleTimeout(connector.getIdleTimeout());
338 }
339 }
340
341 private String low(String reasons, String newReason)
342 {
343 if (reasons==null)
344 return newReason;
345 return reasons+", "+newReason;
346 }
347
348
349 private static class LRMScheduler extends ScheduledExecutorScheduler
350 {
351 }
352 }