/* * Copyright (c) Contributors, http://opensimulator.org/ * See CONTRIBUTORS.TXT for a full list of copyright holders. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of the OpenSimulator Project nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Net.NetworkInformation; using System.Text; using System.Threading; using log4net; using Nini.Config; using OpenMetaverse.StructuredData; using OpenSim.Framework; namespace OpenSim.Framework.Monitoring { public class ServerStatsCollector { private readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); private readonly string LogHeader = "[SERVER STATS]"; public bool Enabled = false; private static Dictionary RegisteredStats = new Dictionary(); public readonly string CategoryServer = "server"; public readonly string ContainerThreadpool = "threadpool"; public readonly string ContainerProcessor = "processor"; public readonly string ContainerMemory = "memory"; public readonly string ContainerNetwork = "network"; public readonly string ContainerProcess = "process"; public string NetworkInterfaceTypes = "Ethernet"; readonly int performanceCounterSampleInterval = 500; // int lastperformanceCounterSampleTime = 0; private class PerfCounterControl { public PerformanceCounter perfCounter; public int lastFetch; public string name; public PerfCounterControl(PerformanceCounter pPc) : this(pPc, String.Empty) { } public PerfCounterControl(PerformanceCounter pPc, string pName) { perfCounter = pPc; lastFetch = 0; name = pName; } } PerfCounterControl processorPercentPerfCounter = null; // IRegionModuleBase.Initialize public void Initialise(IConfigSource source) { if (source == null) return; IConfig cfg = source.Configs["Monitoring"]; if (cfg != null) Enabled = cfg.GetBoolean("ServerStatsEnabled", false); if (Enabled) { NetworkInterfaceTypes = cfg.GetString("NetworkInterfaceTypes", "Ethernet"); } } public void Start() { if(!Enabled) return; if (RegisteredStats.Count == 0) RegisterServerStats(); } public void Close() { if(!Enabled) return; if (RegisteredStats.Count > 0) { foreach (Stat stat in RegisteredStats.Values) { StatsManager.DeregisterStat(stat); stat.Dispose(); } RegisteredStats.Clear(); } } private void MakeStat(string pName, string pDesc, string pUnit, string pContainer, Action act) { MakeStat(pName, pDesc, pUnit, pContainer, act, MeasuresOfInterest.None); } private void MakeStat(string pName, string pDesc, string pUnit, string pContainer, Action act, MeasuresOfInterest moi) { string desc = pDesc; if (desc == null) desc = pName; Stat stat = new Stat(pName, pName, desc, pUnit, CategoryServer, pContainer, StatType.Pull, moi, act, StatVerbosity.Debug); StatsManager.RegisterStat(stat); RegisteredStats.Add(pName, stat); } public void RegisterServerStats() { // lastperformanceCounterSampleTime = Util.EnvironmentTickCount(); PerformanceCounter tempPC; Stat tempStat; string tempName; try { tempName = "CPUPercent"; tempPC = new PerformanceCounter("Processor", "% Processor Time", "_Total"); processorPercentPerfCounter = new PerfCounterControl(tempPC); // A long time bug in mono is that CPU percent is reported as CPU percent idle. Windows reports CPU percent busy. tempStat = new Stat(tempName, tempName, "", "percent", CategoryServer, ContainerProcessor, StatType.Pull, (s) => { GetNextValue(s, processorPercentPerfCounter); }, StatVerbosity.Info); StatsManager.RegisterStat(tempStat); RegisteredStats.Add(tempName, tempStat); MakeStat("TotalProcessorTime", null, "sec", ContainerProcessor, (s) => { s.Value = Math.Round(Process.GetCurrentProcess().TotalProcessorTime.TotalSeconds, 3); }); MakeStat("UserProcessorTime", null, "sec", ContainerProcessor, (s) => { s.Value = Math.Round(Process.GetCurrentProcess().UserProcessorTime.TotalSeconds, 3); }); MakeStat("PrivilegedProcessorTime", null, "sec", ContainerProcessor, (s) => { s.Value = Math.Round(Process.GetCurrentProcess().PrivilegedProcessorTime.TotalSeconds, 3); }); MakeStat("Threads", null, "threads", ContainerProcessor, (s) => { s.Value = Process.GetCurrentProcess().Threads.Count; }); } catch (Exception e) { m_log.ErrorFormat("{0} Exception creating 'Process': {1}", LogHeader, e); } MakeStat("BuiltinThreadpoolWorkerThreadsAvailable", null, "threads", ContainerThreadpool, s => { int workerThreads, iocpThreads; ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); s.Value = workerThreads; }); MakeStat("BuiltinThreadpoolIOCPThreadsAvailable", null, "threads", ContainerThreadpool, s => { int workerThreads, iocpThreads; ThreadPool.GetAvailableThreads(out workerThreads, out iocpThreads); s.Value = iocpThreads; }); if (Util.FireAndForgetMethod == FireAndForgetMethod.SmartThreadPool && Util.GetSmartThreadPoolInfo() != null) { MakeStat("STPMaxThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxThreads); MakeStat("STPMinThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MinThreads); MakeStat("STPConcurrency", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().MaxConcurrentWorkItems); MakeStat("STPActiveThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().ActiveThreads); MakeStat("STPInUseThreads", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().InUseThreads); MakeStat("STPWorkItemsWaiting", null, "threads", ContainerThreadpool, s => s.Value = Util.GetSmartThreadPoolInfo().WaitingCallbacks); } MakeStat( "HTTPRequestsMade", "Number of outbound HTTP requests made", "requests", ContainerNetwork, s => s.Value = WebUtil.RequestNumber, MeasuresOfInterest.AverageChangeOverTime); try { List okInterfaceTypes = new List(NetworkInterfaceTypes.Split(',')); IEnumerable nics = NetworkInterface.GetAllNetworkInterfaces(); foreach (NetworkInterface nic in nics) { if (nic.OperationalStatus != OperationalStatus.Up) continue; string nicInterfaceType = nic.NetworkInterfaceType.ToString(); if (!okInterfaceTypes.Contains(nicInterfaceType)) { m_log.DebugFormat("{0} Not including stats for network interface '{1}' of type '{2}'.", LogHeader, nic.Name, nicInterfaceType); m_log.DebugFormat("{0} To include, add to comma separated list in [Monitoring]NetworkInterfaceTypes={1}", LogHeader, NetworkInterfaceTypes); continue; } if (nic.Supports(NetworkInterfaceComponent.IPv4)) { IPv4InterfaceStatistics nicStats = nic.GetIPv4Statistics(); if (nicStats != null) { MakeStat("BytesRcvd/" + nic.Name, nic.Name, "KB", ContainerNetwork, (s) => { LookupNic(s, (ns) => { return ns.BytesReceived; }, 1024.0); }); MakeStat("BytesSent/" + nic.Name, nic.Name, "KB", ContainerNetwork, (s) => { LookupNic(s, (ns) => { return ns.BytesSent; }, 1024.0); }); MakeStat("TotalBytes/" + nic.Name, nic.Name, "KB", ContainerNetwork, (s) => { LookupNic(s, (ns) => { return ns.BytesSent + ns.BytesReceived; }, 1024.0); }); } } // TODO: add IPv6 (it may actually happen someday) } } catch (Exception e) { m_log.ErrorFormat("{0} Exception creating 'Network Interface': {1}", LogHeader, e); } MakeStat("ProcessMemory", null, "MB", ContainerMemory, (s) => { s.Value = Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024d / 1024d, 3); }); MakeStat("HeapMemory", null, "MB", ContainerMemory, (s) => { s.Value = Math.Round(GC.GetTotalMemory(false) / 1024d / 1024d, 3); }); MakeStat("LastHeapAllocationRate", null, "MB/sec", ContainerMemory, (s) => { s.Value = Math.Round(MemoryWatchdog.LastHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); MakeStat("AverageHeapAllocationRate", null, "MB/sec", ContainerMemory, (s) => { s.Value = Math.Round(MemoryWatchdog.AverageHeapAllocationRate * 1000d / 1024d / 1024d, 3); }); MakeStat("ProcessResident", null, "MB", ContainerProcess, (s) => { Process myprocess = Process.GetCurrentProcess(); myprocess.Refresh(); s.Value = Math.Round(Process.GetCurrentProcess().WorkingSet64 / 1024.0 / 1024.0); }); MakeStat("ProcessPaged", null, "MB", ContainerProcess, (s) => { Process myprocess = Process.GetCurrentProcess(); myprocess.Refresh(); s.Value = Math.Round(Process.GetCurrentProcess().PagedMemorySize64 / 1024.0 / 1024.0); }); MakeStat("ProcessVirtual", null, "MB", ContainerProcess, (s) => { Process myprocess = Process.GetCurrentProcess(); myprocess.Refresh(); s.Value = Math.Round(Process.GetCurrentProcess().VirtualMemorySize64 / 1024.0 / 1024.0); }); MakeStat("PeakProcessResident", null, "MB", ContainerProcess, (s) => { Process myprocess = Process.GetCurrentProcess(); myprocess.Refresh(); s.Value = Math.Round(Process.GetCurrentProcess().PeakWorkingSet64 / 1024.0 / 1024.0); }); MakeStat("PeakProcessPaged", null, "MB", ContainerProcess, (s) => { Process myprocess = Process.GetCurrentProcess(); myprocess.Refresh(); s.Value = Math.Round(Process.GetCurrentProcess().PeakPagedMemorySize64 / 1024.0 / 1024.0); }); MakeStat("PeakProcessVirtual", null, "MB", ContainerProcess, (s) => { Process myprocess = Process.GetCurrentProcess(); myprocess.Refresh(); s.Value = Math.Round(Process.GetCurrentProcess().PeakVirtualMemorySize64 / 1024.0 / 1024.0); }); } // Notes on performance counters: // "How To Read Performance Counters": http://blogs.msdn.com/b/bclteam/archive/2006/06/02/618156.aspx // "How to get the CPU Usage in C#": http://stackoverflow.com/questions/278071/how-to-get-the-cpu-usage-in-c // "Mono Performance Counters": http://www.mono-project.com/Mono_Performance_Counters private delegate double PerfCounterNextValue(); private void GetNextValue(Stat stat, PerfCounterControl perfControl) { if (Util.EnvironmentTickCountSubtract(perfControl.lastFetch) > performanceCounterSampleInterval) { if (perfControl != null && perfControl.perfCounter != null) { try { stat.Value = Math.Round(perfControl.perfCounter.NextValue(), 3); } catch (Exception e) { m_log.ErrorFormat("{0} Exception on NextValue fetching {1}: {2}", LogHeader, stat.Name, e); } perfControl.lastFetch = Util.EnvironmentTickCount(); } } } // Lookup the nic that goes with this stat and set the value by using a fetch action. // Not sure about closure with delegates inside delegates. private delegate double GetIPv4StatValue(IPv4InterfaceStatistics interfaceStat); private void LookupNic(Stat stat, GetIPv4StatValue getter, double factor) { // Get the one nic that has the name of this stat IEnumerable nics = NetworkInterface.GetAllNetworkInterfaces().Where( (network) => network.Name == stat.Description); try { foreach (NetworkInterface nic in nics) { IPv4InterfaceStatistics intrStats = nic.GetIPv4Statistics(); if (intrStats != null) { double newVal = Math.Round(getter(intrStats) / factor, 3); stat.Value = newVal; } break; } } catch { // There are times interfaces go away so we just won't update the stat for this m_log.ErrorFormat("{0} Exception fetching stat on interface '{1}'", LogHeader, stat.Description); } } } public class ServerStatsAggregator : Stat { public ServerStatsAggregator( string shortName, string name, string description, string unitName, string category, string container ) : base( shortName, name, description, unitName, category, container, StatType.Push, MeasuresOfInterest.None, null, StatVerbosity.Info) { } public override string ToConsoleString() { StringBuilder sb = new StringBuilder(); return sb.ToString(); } public override OSDMap ToOSDMap() { OSDMap ret = new OSDMap(); return ret; } } }