1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.eclipse.jetty.server;
20
21 import java.util.Arrays;
22 import java.util.Collection;
23 import java.util.Collections;
24 import java.util.concurrent.Executor;
25 import java.util.concurrent.TimeUnit;
26 import java.util.concurrent.atomic.AtomicBoolean;
27
28 import org.eclipse.jetty.io.EndPoint;
29 import org.eclipse.jetty.util.annotation.ManagedAttribute;
30 import org.eclipse.jetty.util.annotation.ManagedObject;
31 import org.eclipse.jetty.util.annotation.Name;
32 import org.eclipse.jetty.util.component.AbstractLifeCycle;
33 import org.eclipse.jetty.util.log.Log;
34 import org.eclipse.jetty.util.log.Logger;
35 import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
36 import org.eclipse.jetty.util.thread.Scheduler;
37 import org.eclipse.jetty.util.thread.ThreadPool;
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62 @ManagedObject ("Monitor for low resource conditions and activate a low resource mode if detected")
63 public class LowResourceMonitor extends AbstractLifeCycle
64 {
65 private static final Logger LOG = Log.getLogger(LowResourceMonitor.class);
66 private final Server _server;
67 private Scheduler _scheduler;
68 private Connector[] _monitoredConnectors;
69 private int _period=1000;
70 private int _maxConnections;
71 private long _maxMemory;
72 private int _lowResourcesIdleTimeout=1000;
73 private int _maxLowResourcesTime=0;
74 private boolean _monitorThreads=true;
75 private final AtomicBoolean _low = new AtomicBoolean();
76 private String _cause;
77 private String _reasons;
78 private long _lowStarted;
79
80
81 private final Runnable _monitor = new Runnable()
82 {
83 @Override
84 public void run()
85 {
86 if (isRunning())
87 {
88 monitor();
89 _scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS);
90 }
91 }
92 };
93
94 public LowResourceMonitor(@Name("server") Server server)
95 {
96 _server=server;
97 }
98
99 @ManagedAttribute("Are the monitored connectors low on resources?")
100 public boolean isLowOnResources()
101 {
102 return _low.get();
103 }
104
105 @ManagedAttribute("The reason(s) the monitored connectors are low on resources")
106 public String getLowResourcesReasons()
107 {
108 return _reasons;
109 }
110
111 @ManagedAttribute("Get the timestamp in ms since epoch that low resources state started")
112 public long getLowResourcesStarted()
113 {
114 return _lowStarted;
115 }
116
117 @ManagedAttribute("The monitored connectors. If null then all server connectors are monitored")
118 public Collection<Connector> getMonitoredConnectors()
119 {
120 if (_monitoredConnectors==null)
121 return Collections.emptyList();
122 return Arrays.asList(_monitoredConnectors);
123 }
124
125
126
127
128 public void setMonitoredConnectors(Collection<Connector> monitoredConnectors)
129 {
130 if (monitoredConnectors==null || monitoredConnectors.size()==0)
131 _monitoredConnectors=null;
132 else
133 _monitoredConnectors = monitoredConnectors.toArray(new Connector[monitoredConnectors.size()]);
134 }
135
136 @ManagedAttribute("The monitor period in ms")
137 public int getPeriod()
138 {
139 return _period;
140 }
141
142
143
144
145 public void setPeriod(int periodMS)
146 {
147 _period = periodMS;
148 }
149
150 @ManagedAttribute("True if low available threads status is monitored")
151 public boolean getMonitorThreads()
152 {
153 return _monitorThreads;
154 }
155
156
157
158
159
160 public void setMonitorThreads(boolean monitorThreads)
161 {
162 _monitorThreads = monitorThreads;
163 }
164
165 @ManagedAttribute("The maximum connections allowed for the monitored connectors before low resource handling is activated")
166 public int getMaxConnections()
167 {
168 return _maxConnections;
169 }
170
171
172
173
174 public void setMaxConnections(int maxConnections)
175 {
176 _maxConnections = maxConnections;
177 }
178
179 @ManagedAttribute("The maximum memory (in bytes) that can be used before low resources is triggered. Memory used is calculated as (totalMemory-freeMemory).")
180 public long getMaxMemory()
181 {
182 return _maxMemory;
183 }
184
185
186
187
188 public void setMaxMemory(long maxMemoryBytes)
189 {
190 _maxMemory = maxMemoryBytes;
191 }
192
193 @ManagedAttribute("The idletimeout in ms to apply to all existing connections when low resources is detected")
194 public int getLowResourcesIdleTimeout()
195 {
196 return _lowResourcesIdleTimeout;
197 }
198
199
200
201
202 public void setLowResourcesIdleTimeout(int lowResourcesIdleTimeoutMS)
203 {
204 _lowResourcesIdleTimeout = lowResourcesIdleTimeoutMS;
205 }
206
207 @ManagedAttribute("The maximum time in ms that low resources condition can persist before lowResourcesIdleTimeout is applied to new connections as well as existing connections")
208 public int getMaxLowResourcesTime()
209 {
210 return _maxLowResourcesTime;
211 }
212
213
214
215
216 public void setMaxLowResourcesTime(int maxLowResourcesTimeMS)
217 {
218 _maxLowResourcesTime = maxLowResourcesTimeMS;
219 }
220
221 @Override
222 protected void doStart() throws Exception
223 {
224 _scheduler = _server.getBean(Scheduler.class);
225
226 if (_scheduler==null)
227 {
228 _scheduler=new LRMScheduler();
229 _scheduler.start();
230 }
231 super.doStart();
232
233 _scheduler.schedule(_monitor,_period,TimeUnit.MILLISECONDS);
234 }
235
236 @Override
237 protected void doStop() throws Exception
238 {
239 if (_scheduler instanceof LRMScheduler)
240 _scheduler.stop();
241 super.doStop();
242 }
243
244 protected Connector[] getMonitoredOrServerConnectors()
245 {
246 if (_monitoredConnectors!=null && _monitoredConnectors.length>0)
247 return _monitoredConnectors;
248 return _server.getConnectors();
249 }
250
251 protected void monitor()
252 {
253 String reasons=null;
254 String cause="";
255 int connections=0;
256
257 for(Connector connector : getMonitoredOrServerConnectors())
258 {
259 connections+=connector.getConnectedEndPoints().size();
260
261 Executor executor = connector.getExecutor();
262 if (executor instanceof ThreadPool)
263 {
264 ThreadPool threadpool=(ThreadPool) executor;
265 if (_monitorThreads && threadpool.isLowOnThreads())
266 {
267 reasons=low(reasons,"Low on threads: "+threadpool);
268 cause+="T";
269 }
270 }
271 }
272
273 if (_maxConnections>0 && connections>_maxConnections)
274 {
275 reasons=low(reasons,"Max Connections exceeded: "+connections+">"+_maxConnections);
276 cause+="C";
277 }
278
279 long memory=Runtime.getRuntime().totalMemory()-Runtime.getRuntime().freeMemory();
280 if (_maxMemory>0 && memory>_maxMemory)
281 {
282 reasons=low(reasons,"Max memory exceeded: "+memory+">"+_maxMemory);
283 cause+="M";
284 }
285
286
287 if (reasons!=null)
288 {
289
290 if (!cause.equals(_cause))
291 {
292 LOG.warn("Low Resources: {}",reasons);
293 _cause=cause;
294 }
295
296
297 if (_low.compareAndSet(false,true))
298 {
299 _reasons=reasons;
300 _lowStarted=System.currentTimeMillis();
301 setLowResources();
302 }
303
304
305 if (_maxLowResourcesTime>0 && (System.currentTimeMillis()-_lowStarted)>_maxLowResourcesTime)
306 setLowResources();
307 }
308 else
309 {
310 if (_low.compareAndSet(true,false))
311 {
312 LOG.info("Low Resources cleared");
313 _reasons=null;
314 _lowStarted=0;
315 _cause=null;
316 clearLowResources();
317 }
318 }
319 }
320
321 protected void setLowResources()
322 {
323 for(Connector connector : getMonitoredOrServerConnectors())
324 {
325 for (EndPoint endPoint : connector.getConnectedEndPoints())
326 endPoint.setIdleTimeout(_lowResourcesIdleTimeout);
327 }
328 }
329
330 protected void clearLowResources()
331 {
332 for(Connector connector : getMonitoredOrServerConnectors())
333 {
334 for (EndPoint endPoint : connector.getConnectedEndPoints())
335 endPoint.setIdleTimeout(connector.getIdleTimeout());
336 }
337 }
338
339 private String low(String reasons, String newReason)
340 {
341 if (reasons==null)
342 return newReason;
343 return reasons+", "+newReason;
344 }
345
346
347 private static class LRMScheduler extends ScheduledExecutorScheduler
348 {
349 }
350 }