123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294 |
- /*
- * Copyright (c) Contributors, http://opensimulator.org/
- * See CONTRIBUTORS.TXT for a full list of copyright holders.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * * Neither the name of the OpenSimulator Project nor the
- * names of its contributors may be used to endorse or promote products
- * derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
- * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
- using System;
- using System.Reflection;
- using System.Threading;
- using log4net;
- namespace OpenSim.Framework.Monitoring
- {
- /// <summary>
- /// Manages various work items in the simulator.
- /// </summary>
- /// <remarks>
- /// Currently, here work can be started
- /// * As a long-running and monitored thread.
- /// * In a thread that will never timeout but where the job is expected to eventually complete.
- /// * In a threadpool thread that will timeout if it takes a very long time to complete (> 10 mins).
- /// * As a job which will be run in a single-threaded job engine. Such jobs must not incorporate delays (sleeps,
- /// network waits, etc.).
- ///
- /// This is an evolving approach to better manage the work that OpenSimulator is asked to do from a very diverse
- /// range of sources (client actions, incoming network, outgoing network calls, etc.).
- ///
- /// Util.FireAndForget is still available to insert jobs in the threadpool, though this is equivalent to
- /// WorkManager.RunInThreadPool().
- /// </remarks>
- public static class WorkManager
- {
- private static readonly ILog m_log = LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType);
- public static JobEngine JobEngine { get; private set; }
- static WorkManager()
- {
- JobEngine = new JobEngine("Non-blocking non-critical job engine", "JOB ENGINE", 30000);
- StatsManager.RegisterStat(
- new Stat(
- "JobsWaiting",
- "Number of jobs waiting for processing.",
- "",
- "",
- "server",
- "jobengine",
- StatType.Pull,
- MeasuresOfInterest.None,
- stat => stat.Value = JobEngine.JobsWaiting,
- StatVerbosity.Debug));
- MainConsole.Instance.Commands.AddCommand(
- "Debug",
- false,
- "debug jobengine",
- "debug jobengine <start|stop|status|log>",
- "Start, stop, get status or set logging level of the job engine.",
- "If stopped then all outstanding jobs are processed immediately.",
- HandleControlCommand);
- }
- public static void Stop()
- {
- JobEngine.Stop();
- Watchdog.Stop();
- }
- /// <summary>
- /// Start a new long-lived thread.
- /// </summary>
- /// <param name="start">The method that will be executed in a new thread</param>
- /// <param name="name">A name to give to the new thread</param>
- /// <param name="priority">Priority to run the thread at</param>
- /// <param name="isBackground">True to run this thread as a background thread, otherwise false</param>
- /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
- /// <param name="log">If true then creation of thread is logged.</param>
- /// <returns>The newly created Thread object</returns>
- public static Thread StartThread(
- ThreadStart start, string name, ThreadPriority priority, bool isBackground, bool alarmIfTimeout, bool log = true)
- {
- return StartThread(start, name, priority, isBackground, alarmIfTimeout, null, Watchdog.DEFAULT_WATCHDOG_TIMEOUT_MS, log);
- }
- /// <summary>
- /// Start a new thread that is tracked by the watchdog
- /// </summary>
- /// <param name="start">The method that will be executed in a new thread</param>
- /// <param name="name">A name to give to the new thread</param>
- /// <param name="priority">Priority to run the thread at</param>
- /// <param name="isBackground">True to run this thread as a background
- /// thread, otherwise false</param>
- /// <param name="alarmIfTimeout">Trigger an alarm function is we have timed out</param>
- /// <param name="alarmMethod">
- /// Alarm method to call if alarmIfTimeout is true and there is a timeout.
- /// Normally, this will just return some useful debugging information.
- /// </param>
- /// <param name="timeout">Number of milliseconds to wait until we issue a warning about timeout.</param>
- /// <param name="log">If true then creation of thread is logged.</param>
- /// <returns>The newly created Thread object</returns>
- public static Thread StartThread(
- ThreadStart start, string name, ThreadPriority priority, bool isBackground,
- bool alarmIfTimeout, Func<string> alarmMethod, int timeout, bool log = true)
- {
- Thread thread = new Thread(start);
- thread.Priority = priority;
- thread.IsBackground = isBackground;
- thread.Name = name;
- Watchdog.ThreadWatchdogInfo twi
- = new Watchdog.ThreadWatchdogInfo(thread, timeout, name)
- { AlarmIfTimeout = alarmIfTimeout, AlarmMethod = alarmMethod };
- Watchdog.AddThread(twi, name, log:log);
- thread.Start();
- return thread;
- }
- /// <summary>
- /// Run the callback in a new thread immediately. If the thread exits with an exception log it but do
- /// not propogate it.
- /// </summary>
- /// <param name="callback">Code for the thread to execute.</param>
- /// <param name="obj">Object to pass to the thread.</param>
- /// <param name="name">Name of the thread</param>
- public static void RunInThread(WaitCallback callback, object obj, string name, bool log = false)
- {
- if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
- {
- Culture.SetCurrentCulture();
- callback(obj);
- return;
- }
- ThreadStart ts = new ThreadStart(delegate()
- {
- try
- {
- Culture.SetCurrentCulture();
- callback(obj);
- Watchdog.RemoveThread(log:false);
- }
- catch (Exception e)
- {
- m_log.Error(string.Format("[WATCHDOG]: Exception in thread {0}.", name), e);
- }
- });
- StartThread(ts, name, ThreadPriority.Normal, true, false, log:log);
- }
- /// <summary>
- /// Run the callback via a threadpool thread.
- /// </summary>
- /// <remarks>
- /// Such jobs may run after some delay but must always complete.
- /// </remarks>
- /// <param name="callback"></param>
- /// <param name="obj"></param>
- /// <param name="name">The name of the job. This is used in monitoring and debugging.</param>
- public static void RunInThreadPool(System.Threading.WaitCallback callback, object obj, string name, bool timeout = true)
- {
- Util.FireAndForget(callback, obj, name, timeout);
- }
- /// <summary>
- /// Run a job.
- /// </summary>
- /// <remarks>
- /// This differs from direct scheduling (e.g. Util.FireAndForget) in that a job can be run in the job
- /// engine if it is running, where all jobs are currently performed in sequence on a single thread. This is
- /// to prevent observed overload and server freeze problems when there are hundreds of connections which all attempt to
- /// perform work at once (e.g. in conference situations). With lower numbers of connections, the small
- /// delay in performing jobs in sequence rather than concurrently has not been notiecable in testing, though a future more
- /// sophisticated implementation could perform jobs concurrently when the server is under low load.
- ///
- /// However, be advised that some callers of this function rely on all jobs being performed in sequence if any
- /// jobs are performed in sequence (i.e. if jobengine is active or not). Therefore, expanding the jobengine
- /// beyond a single thread will require considerable thought.
- ///
- /// Also, any jobs submitted must be guaranteed to complete within a reasonable timeframe (e.g. they cannot
- /// incorporate a network delay with a long timeout). At the moment, work that could suffer such issues
- /// should still be run directly with RunInThread(), Util.FireAndForget(), etc. This is another area where
- /// the job engine could be improved and so CPU utilization improved by better management of concurrency within
- /// OpenSimulator.
- /// </remarks>
- /// <param name="jobType">General classification for the job (e.g. "RezAttachments").</param>
- /// <param name="callback">Callback for job.</param>
- /// <param name="obj">Object to pass to callback when run</param>
- /// <param name="name">Specific name of job (e.g. "RezAttachments for Joe Bloggs"</param>
- /// <param name="canRunInThisThread">If set to true then the job may be run in ths calling thread.</param>
- /// <param name="mustNotTimeout">If the true then the job must never timeout.</param>
- /// <param name="log">If set to true then extra logging is performed.</param>
- public static void RunJob(
- string jobType, WaitCallback callback, object obj, string name,
- bool canRunInThisThread = false, bool mustNotTimeout = false,
- bool log = false)
- {
- if (Util.FireAndForgetMethod == FireAndForgetMethod.RegressionTest)
- {
- Culture.SetCurrentCulture();
- callback(obj);
- return;
- }
- if (JobEngine.IsRunning)
- JobEngine.QueueJob(name, () => callback(obj));
- else if (canRunInThisThread)
- callback(obj);
- else
- Util.FireAndForget(callback, obj, name, !mustNotTimeout);
- }
- private static void HandleControlCommand(string module, string[] args)
- {
- // if (SceneManager.Instance.CurrentScene != null && SceneManager.Instance.CurrentScene != m_udpServer.Scene)
- // return;
- if (args.Length < 3)
- {
- MainConsole.Instance.Output("Usage: debug jobengine <stop|start|status|log>");
- return;
- }
- string subCommand = args[2];
- if (subCommand == "stop")
- {
- JobEngine.Stop();
- MainConsole.Instance.OutputFormat("Stopped job engine.");
- }
- else if (subCommand == "start")
- {
- JobEngine.Start();
- MainConsole.Instance.OutputFormat("Started job engine.");
- }
- else if (subCommand == "status")
- {
- MainConsole.Instance.OutputFormat("Job engine running: {0}", JobEngine.IsRunning);
- JobEngine.Job job = JobEngine.CurrentJob;
- MainConsole.Instance.OutputFormat("Current job {0}", job != null ? job.Name : "none");
- MainConsole.Instance.OutputFormat(
- "Jobs waiting: {0}", JobEngine.IsRunning ? JobEngine.JobsWaiting.ToString() : "n/a");
- MainConsole.Instance.OutputFormat("Log Level: {0}", JobEngine.LogLevel);
- }
- else if (subCommand == "log")
- {
- if (args.Length < 4)
- {
- MainConsole.Instance.Output("Usage: debug jobengine log <level>");
- return;
- }
- // int logLevel;
- int logLevel = int.Parse(args[3]);
- // if (ConsoleUtil.TryParseConsoleInt(MainConsole.Instance, args[4], out logLevel))
- // {
- JobEngine.LogLevel = logLevel;
- MainConsole.Instance.OutputFormat("Set debug log level to {0}", JobEngine.LogLevel);
- // }
- }
- else
- {
- MainConsole.Instance.OutputFormat("Unrecognized job engine subcommand {0}", subCommand);
- }
- }
- }
- }
|