1
0

ServerStatsCollector.cs 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349
  1. /*
  2. * Copyright (c) Contributors, http://opensimulator.org/
  3. * See CONTRIBUTORS.TXT for a full list of copyright holders.
  4. *
  5. * Redistribution and use in source and binary forms, with or without
  6. * modification, are permitted provided that the following conditions are met:
  7. * * Redistributions of source code must retain the above copyright
  8. * notice, this list of conditions and the following disclaimer.
  9. * * Redistributions in binary form must reproduce the above copyright
  10. * notice, this list of conditions and the following disclaimer in the
  11. * documentation and/or other materials provided with the distribution.
  12. * * Neither the name of the OpenSimulator Project nor the
  13. * names of its contributors may be used to endorse or promote products
  14. * derived from this software without specific prior written permission.
  15. *
  16. * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
  17. * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19. * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
  20. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  21. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  22. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  23. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  25. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26. */
  27. using System;
  28. using System.Collections.Generic;
  29. using System.Diagnostics;
  30. using System.Linq;
  31. using System.Net.NetworkInformation;
  32. using System.Text;
  33. using System.Threading;
  34. using log4net;
  35. using Nini.Config;
  36. using OpenMetaverse.StructuredData;
  37. using OpenSim.Framework;
  38. namespace OpenSim.Framework.Monitoring
  39. {
  40. public class ServerStatsCollector
  41. {
  42. private readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
  43. private readonly string LogHeader = "[SERVER STATS]";
  44. public bool Enabled = false;
  45. private static Dictionary<string, Stat> RegisteredStats = new Dictionary<string, Stat>();
  46. public readonly string CategoryServer = "server";
  47. public readonly string ContainerThreadpool = "threadpool";
  48. public readonly string ContainerProcessor = "processor";
  49. public readonly string ContainerMemory = "memory";
  50. public readonly string ContainerNetwork = "network";
  51. public readonly string ContainerProcess = "process";
  52. public string NetworkInterfaceTypes = "Ethernet";
  53. readonly int performanceCounterSampleInterval = 500;
  54. // int lastperformanceCounterSampleTime = 0;
  55. private class PerfCounterControl
  56. {
  57. public PerformanceCounter perfCounter;
  58. public int lastFetch;
  59. public string name;
  60. public PerfCounterControl(PerformanceCounter pPc)
  61. : this(pPc, String.Empty)
  62. {
  63. }
  64. public PerfCounterControl(PerformanceCounter pPc, string pName)
  65. {
  66. perfCounter = pPc;
  67. lastFetch = 0;
  68. name = pName;
  69. }
  70. }
  71. PerfCounterControl processorPercentPerfCounter = null;
  72. // IRegionModuleBase.Initialize
  73. public void Initialise(IConfigSource source)
  74. {
  75. IConfig cfg = source.Configs["Monitoring"];
  76. if (cfg != null)
  77. Enabled = cfg.GetBoolean("ServerStatsEnabled", true);
  78. if (Enabled)
  79. {
  80. NetworkInterfaceTypes = cfg.GetString("NetworkInterfaceTypes", "Ethernet");
  81. }
  82. }
  83. public void Start()
  84. {
  85. if (RegisteredStats.Count == 0)
  86. RegisterServerStats();
  87. }
  88. public void Close()
  89. {
  90. if (RegisteredStats.Count > 0)
  91. {
  92. foreach (Stat stat in RegisteredStats.Values)
  93. {
  94. StatsManager.DeregisterStat(stat);
  95. stat.Dispose();
  96. }
  97. RegisteredStats.Clear();
  98. }
  99. }
  100. private void MakeStat(string pName, string pDesc, string pUnit, string pContainer, Action<Stat> act)
  101. {
  102. MakeStat(pName, pDesc, pUnit, pContainer, act, MeasuresOfInterest.None);
  103. }
  104. private void MakeStat(string pName, string pDesc, string pUnit, string pContainer, Action<Stat> act, MeasuresOfInterest moi)
  105. {
  106. string desc = pDesc;
  107. if (desc == null)
  108. desc = pName;
  109. Stat stat = new Stat(pName, pName, desc, pUnit, CategoryServer, pContainer, StatType.Pull, moi, act, StatVerbosity.Debug);
  110. StatsManager.RegisterStat(stat);
  111. RegisteredStats.Add(pName, stat);
  112. }
  113. public void RegisterServerStats()
  114. {
  115. // lastperformanceCounterSampleTime = Util.EnvironmentTickCount();
  116. PerformanceCounter tempPC;
  117. Stat tempStat;
  118. string tempName;
  119. try
  120. {
  121. tempName = "CPUPercent";
  122. tempPC = new PerformanceCounter("Processor", "% Processor Time", "_Total");
  123. processorPercentPerfCounter = new PerfCounterControl(tempPC);
  124. // A long time bug in mono is that CPU percent is reported as CPU percent idle. Windows reports CPU percent busy.
  125. tempStat = new Stat(tempName, tempName, "", "percent", CategoryServer, ContainerProcessor,
  126. StatType.Pull, (s) => { GetNextValue(s, processorPercentPerfCounter, Util.IsWindows() ? 1 : -1); },
  127. StatVerbosity.Info);
  128. StatsManager.RegisterStat(tempStat);
  129. RegisteredStats.Add(tempName, tempStat);
  130. MakeStat("TotalProcessorTime", null, "sec", ContainerProcessor,
  131. (s) => { s.Value = Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds; });
  132. MakeStat("UserProcessorTime", null, "sec", ContainerProcessor,
  133. (s) => { s.Value = Process.GetCurrentProcess().UserProcessorTime.TotalSeconds; });
  134. MakeStat("PrivilegedProcessorTime", null, "sec", ContainerProcessor,
  135. (s) => { s.Value = Process.GetCurrentProcess().PrivilegedProcessorTime.TotalSeconds; });
  136. MakeStat("Threads", null, "threads", ContainerProcessor,
  137. (s) => { s.Value = Process.GetCurrentProcess().Threads.Count; });
  138. }
  139. catch (Exception e)
  140. {
  141. m_log.ErrorFormat("{0} Exception creating 'Process': {1}", LogHeader, e);
  142. }
  143. MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerThreadpool,
  144. s =>
  145. {
  146. int workerThreads, iocpThreads;
  147. ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads);
  148. s.Value = workerThreads;
  149. });
  150. MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerThreadpool,
  151. s =>
  152. {
  153. int workerThreads, iocpThreads;
  154. ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads);
  155. s.Value = iocpThreads;
  156. });
  157. if (Util.FireAndForgetMethod == FireAndForgetMethod.SmartThreadPool && Util.GetSmartThreadPoolInfo() != null)
  158. {
  159. MakeStat("STPMaxThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxThreads);
  160. MakeStat("STPMinThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MinThreads);
  161. MakeStat("STPConcurrency", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxConcurrentWorkItems);
  162. MakeStat("STPActiveThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().ActiveThreads);
  163. MakeStat("STPInUseThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().InUseThreads);
  164. MakeStat("STPWorkItemsWaiting", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().WaitingCallbacks);
  165. }
  166. MakeStat(
  167. "HTTPRequestsMade",
  168. "Number of outbound HTTP requests made",
  169. "requests",
  170. ContainerNetwork,
  171. s => s.Value = WebUtil.RequestNumber,
  172. MeasuresOfInterest.AverageChangeOverTime);
  173. try
  174. {
  175. List<string> okInterfaceTypes = new List<string>(NetworkInterfaceTypes.Split(','));
  176. IEnumerable<NetworkInterface> nics = NetworkInterface.GetAllNetworkInterfaces();
  177. foreach (NetworkInterface nic in nics)
  178. {
  179. if (nic.OperationalStatus != OperationalStatus.Up)
  180. continue;
  181. string nicInterfaceType = nic.NetworkInterfaceType.ToString();
  182. if (!okInterfaceTypes.Contains(nicInterfaceType))
  183. {
  184. m_log.DebugFormat("{0} Not including stats for network interface '{1}' of type '{2}'.",
  185. LogHeader, nic.Name, nicInterfaceType);
  186. m_log.DebugFormat("{0} To include, add to comma separated list in [Monitoring]NetworkInterfaceTypes={1}",
  187. LogHeader, NetworkInterfaceTypes);
  188. continue;
  189. }
  190. if (nic.Supports(NetworkInterfaceComponent.IPv4))
  191. {
  192. IPv4InterfaceStatistics nicStats = nic.GetIPv4Statistics();
  193. if (nicStats != null)
  194. {
  195. MakeStat("BytesRcvd/" + nic.Name, nic.Name, "KB", ContainerNetwork,
  196. (s) => { LookupNic(s, (ns) => { return ns.BytesReceived; }, 1024.0); });
  197. MakeStat("BytesSent/" + nic.Name, nic.Name, "KB", ContainerNetwork,
  198. (s) => { LookupNic(s, (ns) => { return ns.BytesSent; }, 1024.0); });
  199. MakeStat("TotalBytes/" + nic.Name, nic.Name, "KB", ContainerNetwork,
  200. (s) => { LookupNic(s, (ns) => { return ns.BytesSent + ns.BytesReceived; }, 1024.0); });
  201. }
  202. }
  203. // TODO: add IPv6 (it may actually happen someday)
  204. }
  205. }
  206. catch (Exception e)
  207. {
  208. m_log.ErrorFormat("{0} Exception creating 'Network Interface': {1}", LogHeader, e);
  209. }
  210. MakeStat("ProcessMemory", null, "MB", ContainerMemory,
  211. (s) => { s.Value = Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d, 3); });
  212. MakeStat("HeapMemory", null, "MB", ContainerMemory,
  213. (s) => { s.Value = Math.Round(GC.GetTotalMemory(false) / 1024d / 1024d, 3); });
  214. MakeStat("LastHeapAllocationRate", null, "MB/sec", ContainerMemory,
  215. (s) => { s.Value = Math.Round(MemoryWatchdog.LastHeapAllocationRate * 1000d / 1024d / 1024d, 3); });
  216. MakeStat("AverageHeapAllocationRate", null, "MB/sec", ContainerMemory,
  217. (s) => { s.Value = Math.Round(MemoryWatchdog.AverageHeapAllocationRate * 1000d / 1024d / 1024d, 3); });
  218. }
  219. // Notes on performance counters:
  220. // "How To Read Performance Counters": http://blogs.msdn.com/b/bclteam/archive/2006/06/02/618156.aspx
  221. // "How to get the CPU Usage in C#": http://stackoverflow.com/questions/278071/how-to-get-the-cpu-usage-in-c
  222. // "Mono Performance Counters": http://www.mono-project.com/Mono_Performance_Counters
  223. private delegate double PerfCounterNextValue();
  224. private void GetNextValue(Stat stat, PerfCounterControl perfControl)
  225. {
  226. GetNextValue(stat, perfControl, 1.0);
  227. }
  228. private void GetNextValue(Stat stat, PerfCounterControl perfControl, double factor)
  229. {
  230. if (Util.EnvironmentTickCountSubtract(perfControl.lastFetch) > performanceCounterSampleInterval)
  231. {
  232. if (perfControl != null && perfControl.perfCounter != null)
  233. {
  234. try
  235. {
  236. // Kludge for factor to run double duty. If -1, subtract the value from one
  237. if (factor == -1)
  238. stat.Value = 1 - perfControl.perfCounter.NextValue();
  239. else
  240. stat.Value = perfControl.perfCounter.NextValue() / factor;
  241. }
  242. catch (Exception e)
  243. {
  244. m_log.ErrorFormat("{0} Exception on NextValue fetching {1}: {2}", LogHeader, stat.Name, e);
  245. }
  246. perfControl.lastFetch = Util.EnvironmentTickCount();
  247. }
  248. }
  249. }
  250. // Lookup the nic that goes with this stat and set the value by using a fetch action.
  251. // Not sure about closure with delegates inside delegates.
  252. private delegate double GetIPv4StatValue(IPv4InterfaceStatistics interfaceStat);
  253. private void LookupNic(Stat stat, GetIPv4StatValue getter, double factor)
  254. {
  255. // Get the one nic that has the name of this stat
  256. IEnumerable<NetworkInterface> nics = NetworkInterface.GetAllNetworkInterfaces().Where(
  257. (network) => network.Name == stat.Description);
  258. try
  259. {
  260. foreach (NetworkInterface nic in nics)
  261. {
  262. IPv4InterfaceStatistics intrStats = nic.GetIPv4Statistics();
  263. if (intrStats != null)
  264. {
  265. double newVal = Math.Round(getter(intrStats) / factor, 3);
  266. stat.Value = newVal;
  267. }
  268. break;
  269. }
  270. }
  271. catch
  272. {
  273. // There are times interfaces go away so we just won't update the stat for this
  274. m_log.ErrorFormat("{0} Exception fetching stat on interface '{1}'", LogHeader, stat.Description);
  275. }
  276. }
  277. }
  278. public class ServerStatsAggregator : Stat
  279. {
  280. public ServerStatsAggregator(
  281. string shortName,
  282. string name,
  283. string description,
  284. string unitName,
  285. string category,
  286. string container
  287. )
  288. : base(
  289. shortName,
  290. name,
  291. description,
  292. unitName,
  293. category,
  294. container,
  295. StatType.Push,
  296. MeasuresOfInterest.None,
  297. null,
  298. StatVerbosity.Info)
  299. {
  300. }
  301. public override string ToConsoleString()
  302. {
  303. StringBuilder sb = new StringBuilder();
  304. return sb.ToString();
  305. }
  306. public override OSDMap ToOSDMap()
  307. {
  308. OSDMap ret = new OSDMap();
  309. return ret;
  310. }
  311. }
  312. }