AutoBackupModule.cs 39 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928
  1. /*
  2. * Copyright (c) Contributors, http://opensimulator.org/
  3. * See CONTRIBUTORS.TXT for a full list of copyright holders.
  4. *
  5. * Redistribution and use in source and binary forms, with or without
  6. * modification, are permitted provided that the following conditions are met:
  7. * * Redistributions of source code must retain the above copyright
  8. * notice, this list of conditions and the following disclaimer.
  9. * * Redistributions in binary form must reproduce the above copyright
  10. * notice, this list of conditions and the following disclaimer in the
  11. * documentation and/or other materials provided with the distribution.
  12. * * Neither the name of the OpenSimulator Project nor the
  13. * names of its contributors may be used to endorse or promote products
  14. * derived from this software without specific prior written permission.
  15. *
  16. * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
  17. * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19. * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
  20. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  21. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  22. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  23. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  25. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26. */
  27. using System;
  28. using System.Collections.Generic;
  29. using System.Diagnostics;
  30. using System.IO;
  31. using System.Reflection;
  32. using System.Timers;
  33. using System.Text.RegularExpressions;
  34. using log4net;
  35. using Mono.Addins;
  36. using Nini.Config;
  37. using OpenSim.Framework;
  38. using OpenSim.Region.Framework.Interfaces;
  39. using OpenSim.Region.Framework.Scenes;
  40. namespace OpenSim.Region.OptionalModules.World.AutoBackup
  41. {
  42. /// <summary>
  43. /// Choose between ways of naming the backup files that are generated.
  44. /// </summary>
  45. /// <remarks>Time: OARs are named by a timestamp.
  46. /// Sequential: OARs are named by counting (Region_1.oar, Region_2.oar, etc.)
  47. /// Overwrite: Only one file per region is created; it's overwritten each time a backup is made.</remarks>
  48. public enum NamingType
  49. {
  50. Time,
  51. Sequential,
  52. Overwrite
  53. }
  54. ///<summary>
  55. /// AutoBackupModule: save OAR region backups to disk periodically
  56. /// </summary>
  57. /// <remarks>
  58. /// Config Settings Documentation.
  59. /// Each configuration setting can be specified in two places: OpenSim.ini or Regions.ini.
  60. /// If specified in Regions.ini, the settings should be within the region's section name.
  61. /// If specified in OpenSim.ini, the settings should be within the [AutoBackupModule] section.
  62. /// Region-specific settings take precedence.
  63. ///
  64. /// AutoBackupModuleEnabled: True/False. Default: False. If True, use the auto backup module. This setting does not support per-region basis.
  65. /// All other settings under [AutoBackupModule] are ignored if AutoBackupModuleEnabled is false, even per-region settings!
  66. /// AutoBackup: True/False. Default: False. If True, activate auto backup functionality.
  67. /// This is the only required option for enabling auto-backup; the other options have sane defaults.
  68. /// If False for a particular region, the auto-backup module becomes a no-op for the region, and all other AutoBackup* settings are ignored.
  69. /// If False globally (the default), only regions that specifically override it in Regions.ini will get AutoBackup functionality.
  70. /// AutoBackupInterval: Double, non-negative value. Default: 720 (12 hours).
  71. /// The number of minutes between each backup attempt.
  72. /// If a negative or zero value is given, it is equivalent to setting AutoBackup = False.
  73. /// AutoBackupBusyCheck: True/False. Default: True.
  74. /// If True, we will only take an auto-backup if a set of conditions are met.
  75. /// These conditions are heuristics to try and avoid taking a backup when the sim is busy.
  76. /// AutoBackupScript: String. Default: not specified (disabled).
  77. /// File path to an executable script or binary to run when an automatic backup is taken.
  78. /// The file should really be (Windows) an .exe or .bat, or (Linux/Mac) a shell script or binary.
  79. /// Trying to "run" directories, or things with weird file associations on Win32, might cause unexpected results!
  80. /// argv[1] of the executed file/script will be the file name of the generated OAR.
  81. /// If the process can't be spawned for some reason (file not found, no execute permission, etc), write a warning to the console.
  82. /// AutoBackupNaming: string. Default: Time.
  83. /// One of three strings (case insensitive):
  84. /// "Time": Current timestamp is appended to file name. An existing file will never be overwritten.
  85. /// "Sequential": A number is appended to the file name. So if RegionName_x.oar exists, we'll save to RegionName_{x+1}.oar next. An existing file will never be overwritten.
  86. /// "Overwrite": Always save to file named "${AutoBackupDir}/RegionName.oar", even if we have to overwrite an existing file.
  87. /// AutoBackupDir: String. Default: "." (the current directory).
  88. /// A directory (absolute or relative) where backups should be saved.
  89. /// AutoBackupDilationThreshold: float. Default: 0.5. Lower bound on time dilation required for BusyCheck heuristics to pass.
  90. /// If the time dilation is below this value, don't take a backup right now.
  91. /// AutoBackupAgentThreshold: int. Default: 10. Upper bound on # of agents in region required for BusyCheck heuristics to pass.
  92. /// If the number of agents is greater than this value, don't take a backup right now
  93. /// Save memory by setting low initial capacities. Minimizes impact in common cases of all regions using same interval, and instances hosting 1 ~ 4 regions.
  94. /// Also helps if you don't want AutoBackup at all.
  95. /// </remarks>
  96. [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "AutoBackupModule")]
  97. public class AutoBackupModule : ISharedRegionModule
  98. {
  99. private static readonly ILog m_log =
  100. LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  101. private readonly Dictionary<Guid, IScene> m_pendingSaves = new Dictionary<Guid, IScene>(1);
  102. private readonly AutoBackupModuleState m_defaultState = new AutoBackupModuleState();
  103. private readonly Dictionary<IScene, AutoBackupModuleState> m_states =
  104. new Dictionary<IScene, AutoBackupModuleState>(1);
  105. private readonly Dictionary<Timer, List<IScene>> m_timerMap =
  106. new Dictionary<Timer, List<IScene>>(1);
  107. private readonly Dictionary<double, Timer> m_timers = new Dictionary<double, Timer>(1);
  108. private delegate T DefaultGetter<T>(string settingName, T defaultValue);
  109. private bool m_enabled;
  110. /// <summary>
  111. /// Whether the shared module should be enabled at all. NOT the same as m_Enabled in AutoBackupModuleState!
  112. /// </summary>
  113. private bool m_closed;
  114. private IConfigSource m_configSource;
  115. /// <summary>
  116. /// Required by framework.
  117. /// </summary>
  118. public bool IsSharedModule
  119. {
  120. get { return true; }
  121. }
  122. #region ISharedRegionModule Members
  123. /// <summary>
  124. /// Identifies the module to the system.
  125. /// </summary>
  126. string IRegionModuleBase.Name
  127. {
  128. get { return "AutoBackupModule"; }
  129. }
  130. /// <summary>
  131. /// We don't implement an interface, this is a single-use module.
  132. /// </summary>
  133. Type IRegionModuleBase.ReplaceableInterface
  134. {
  135. get { return null; }
  136. }
  137. /// <summary>
  138. /// Called once in the lifetime of the module at startup.
  139. /// </summary>
  140. /// <param name="source">The input config source for OpenSim.ini.</param>
  141. void IRegionModuleBase.Initialise(IConfigSource source)
  142. {
  143. // Determine if we have been enabled at all in OpenSim.ini -- this is part and parcel of being an optional module
  144. this.m_configSource = source;
  145. IConfig moduleConfig = source.Configs["AutoBackupModule"];
  146. if (moduleConfig == null)
  147. {
  148. this.m_enabled = false;
  149. return;
  150. }
  151. else
  152. {
  153. this.m_enabled = moduleConfig.GetBoolean("AutoBackupModuleEnabled", false);
  154. if (this.m_enabled)
  155. {
  156. m_log.Info("[AUTO BACKUP]: AutoBackupModule enabled");
  157. }
  158. else
  159. {
  160. return;
  161. }
  162. }
  163. Timer defTimer = new Timer(43200000);
  164. this.m_defaultState.Timer = defTimer;
  165. this.m_timers.Add(43200000, defTimer);
  166. defTimer.Elapsed += this.HandleElapsed;
  167. defTimer.AutoReset = true;
  168. defTimer.Start();
  169. AutoBackupModuleState abms = this.ParseConfig(null, true);
  170. m_log.Debug("[AUTO BACKUP]: Here is the default config:");
  171. m_log.Debug(abms.ToString());
  172. }
  173. /// <summary>
  174. /// Called once at de-init (sim shutting down).
  175. /// </summary>
  176. void IRegionModuleBase.Close()
  177. {
  178. if (!this.m_enabled)
  179. {
  180. return;
  181. }
  182. // We don't want any timers firing while the sim's coming down; strange things may happen.
  183. this.StopAllTimers();
  184. }
  185. /// <summary>
  186. /// Currently a no-op for AutoBackup because we have to wait for region to be fully loaded.
  187. /// </summary>
  188. /// <param name="scene"></param>
  189. void IRegionModuleBase.AddRegion(Scene scene)
  190. {
  191. }
  192. /// <summary>
  193. /// Here we just clean up some resources and stop the OAR backup (if any) for the given scene.
  194. /// </summary>
  195. /// <param name="scene">The scene (region) to stop performing AutoBackup on.</param>
  196. void IRegionModuleBase.RemoveRegion(Scene scene)
  197. {
  198. if (!this.m_enabled)
  199. {
  200. return;
  201. }
  202. if (this.m_states.ContainsKey(scene))
  203. {
  204. AutoBackupModuleState abms = this.m_states[scene];
  205. // Remove this scene out of the timer map list
  206. Timer timer = abms.Timer;
  207. List<IScene> list = this.m_timerMap[timer];
  208. list.Remove(scene);
  209. // Shut down the timer if this was the last scene for the timer
  210. if (list.Count == 0)
  211. {
  212. this.m_timerMap.Remove(timer);
  213. this.m_timers.Remove(timer.Interval);
  214. timer.Close();
  215. }
  216. this.m_states.Remove(scene);
  217. }
  218. }
  219. /// <summary>
  220. /// Most interesting/complex code paths in AutoBackup begin here.
  221. /// We read lots of Nini config, maybe set a timer, add members to state tracking Dictionaries, etc.
  222. /// </summary>
  223. /// <param name="scene">The scene to (possibly) perform AutoBackup on.</param>
  224. void IRegionModuleBase.RegionLoaded(Scene scene)
  225. {
  226. if (!this.m_enabled)
  227. {
  228. return;
  229. }
  230. // This really ought not to happen, but just in case, let's pretend it didn't...
  231. if (scene == null)
  232. {
  233. return;
  234. }
  235. AutoBackupModuleState abms = this.ParseConfig(scene, false);
  236. m_log.Debug("[AUTO BACKUP]: Config for " + scene.RegionInfo.RegionName);
  237. m_log.Debug((abms == null ? "DEFAULT" : abms.ToString()));
  238. }
  239. /// <summary>
  240. /// Currently a no-op.
  241. /// </summary>
  242. void ISharedRegionModule.PostInitialise()
  243. {
  244. }
  245. #endregion
  246. /// <summary>
  247. /// Set up internal state for a given scene. Fairly complex code.
  248. /// When this method returns, we've started auto-backup timers, put members in Dictionaries, and created a State object for this scene.
  249. /// </summary>
  250. /// <param name="scene">The scene to look at.</param>
  251. /// <param name="parseDefault">Whether this call is intended to figure out what we consider the "default" config (applied to all regions unless overridden by per-region settings).</param>
  252. /// <returns>An AutoBackupModuleState contains most information you should need to know relevant to auto-backup, as applicable to a single region.</returns>
  253. private AutoBackupModuleState ParseConfig(IScene scene, bool parseDefault)
  254. {
  255. string sRegionName;
  256. string sRegionLabel;
  257. // string prepend;
  258. AutoBackupModuleState state;
  259. if (parseDefault)
  260. {
  261. sRegionName = null;
  262. sRegionLabel = "DEFAULT";
  263. // prepend = "";
  264. state = this.m_defaultState;
  265. }
  266. else
  267. {
  268. sRegionName = scene.RegionInfo.RegionName;
  269. sRegionLabel = sRegionName;
  270. // prepend = sRegionName + ".";
  271. state = null;
  272. }
  273. // Read the config settings and set variables.
  274. IConfig regionConfig = (scene != null ? scene.Config.Configs[sRegionName] : null);
  275. IConfig config = this.m_configSource.Configs["AutoBackupModule"];
  276. if (config == null)
  277. {
  278. // defaultState would be disabled too if the section doesn't exist.
  279. state = this.m_defaultState;
  280. return state;
  281. }
  282. bool tmpEnabled = ResolveBoolean("AutoBackup", this.m_defaultState.Enabled, config, regionConfig);
  283. if (state == null && tmpEnabled != this.m_defaultState.Enabled)
  284. //Varies from default state
  285. {
  286. state = new AutoBackupModuleState();
  287. }
  288. if (state != null)
  289. {
  290. state.Enabled = tmpEnabled;
  291. }
  292. // If you don't want AutoBackup, we stop.
  293. if ((state == null && !this.m_defaultState.Enabled) || (state != null && !state.Enabled))
  294. {
  295. return state;
  296. }
  297. else
  298. {
  299. m_log.Info("[AUTO BACKUP]: Region " + sRegionLabel + " is AutoBackup ENABLED.");
  300. }
  301. // Borrow an existing timer if one exists for the same interval; otherwise, make a new one.
  302. double interval =
  303. this.ResolveDouble("AutoBackupInterval", this.m_defaultState.IntervalMinutes,
  304. config, regionConfig) * 60000.0;
  305. if (state == null && interval != this.m_defaultState.IntervalMinutes*60000.0)
  306. {
  307. state = new AutoBackupModuleState();
  308. }
  309. if (this.m_timers.ContainsKey(interval))
  310. {
  311. if (state != null)
  312. {
  313. state.Timer = this.m_timers[interval];
  314. }
  315. m_log.Debug("[AUTO BACKUP]: Reusing timer for " + interval + " msec for region " +
  316. sRegionLabel);
  317. }
  318. else
  319. {
  320. // 0 or negative interval == do nothing.
  321. if (interval <= 0.0 && state != null)
  322. {
  323. state.Enabled = false;
  324. return state;
  325. }
  326. if (state == null)
  327. {
  328. state = new AutoBackupModuleState();
  329. }
  330. Timer tim = new Timer(interval);
  331. state.Timer = tim;
  332. //Milliseconds -> minutes
  333. this.m_timers.Add(interval, tim);
  334. tim.Elapsed += this.HandleElapsed;
  335. tim.AutoReset = true;
  336. tim.Start();
  337. }
  338. // Add the current region to the list of regions tied to this timer.
  339. if (scene != null)
  340. {
  341. if (state != null)
  342. {
  343. if (this.m_timerMap.ContainsKey(state.Timer))
  344. {
  345. this.m_timerMap[state.Timer].Add(scene);
  346. }
  347. else
  348. {
  349. List<IScene> scns = new List<IScene>(1);
  350. scns.Add(scene);
  351. this.m_timerMap.Add(state.Timer, scns);
  352. }
  353. }
  354. else
  355. {
  356. if (this.m_timerMap.ContainsKey(this.m_defaultState.Timer))
  357. {
  358. this.m_timerMap[this.m_defaultState.Timer].Add(scene);
  359. }
  360. else
  361. {
  362. List<IScene> scns = new List<IScene>(1);
  363. scns.Add(scene);
  364. this.m_timerMap.Add(this.m_defaultState.Timer, scns);
  365. }
  366. }
  367. }
  368. bool tmpBusyCheck = ResolveBoolean("AutoBackupBusyCheck",
  369. this.m_defaultState.BusyCheck, config, regionConfig);
  370. if (state == null && tmpBusyCheck != this.m_defaultState.BusyCheck)
  371. {
  372. state = new AutoBackupModuleState();
  373. }
  374. if (state != null)
  375. {
  376. state.BusyCheck = tmpBusyCheck;
  377. }
  378. // Set file naming algorithm
  379. string stmpNamingType = ResolveString("AutoBackupNaming",
  380. this.m_defaultState.NamingType.ToString(), config, regionConfig);
  381. NamingType tmpNamingType;
  382. if (stmpNamingType.Equals("Time", StringComparison.CurrentCultureIgnoreCase))
  383. {
  384. tmpNamingType = NamingType.Time;
  385. }
  386. else if (stmpNamingType.Equals("Sequential", StringComparison.CurrentCultureIgnoreCase))
  387. {
  388. tmpNamingType = NamingType.Sequential;
  389. }
  390. else if (stmpNamingType.Equals("Overwrite", StringComparison.CurrentCultureIgnoreCase))
  391. {
  392. tmpNamingType = NamingType.Overwrite;
  393. }
  394. else
  395. {
  396. m_log.Warn("Unknown naming type specified for region " + sRegionLabel + ": " +
  397. stmpNamingType);
  398. tmpNamingType = NamingType.Time;
  399. }
  400. if (state == null && tmpNamingType != this.m_defaultState.NamingType)
  401. {
  402. state = new AutoBackupModuleState();
  403. }
  404. if (state != null)
  405. {
  406. state.NamingType = tmpNamingType;
  407. }
  408. string tmpScript = ResolveString("AutoBackupScript",
  409. this.m_defaultState.Script, config, regionConfig);
  410. if (state == null && tmpScript != this.m_defaultState.Script)
  411. {
  412. state = new AutoBackupModuleState();
  413. }
  414. if (state != null)
  415. {
  416. state.Script = tmpScript;
  417. }
  418. string tmpBackupDir = ResolveString("AutoBackupDir", ".", config, regionConfig);
  419. if (state == null && tmpBackupDir != this.m_defaultState.BackupDir)
  420. {
  421. state = new AutoBackupModuleState();
  422. }
  423. if (state != null)
  424. {
  425. state.BackupDir = tmpBackupDir;
  426. // Let's give the user some convenience and auto-mkdir
  427. if (state.BackupDir != ".")
  428. {
  429. try
  430. {
  431. DirectoryInfo dirinfo = new DirectoryInfo(state.BackupDir);
  432. if (!dirinfo.Exists)
  433. {
  434. dirinfo.Create();
  435. }
  436. }
  437. catch (Exception e)
  438. {
  439. m_log.Warn(
  440. "BAD NEWS. You won't be able to save backups to directory " +
  441. state.BackupDir +
  442. " because it doesn't exist or there's a permissions issue with it. Here's the exception.",
  443. e);
  444. }
  445. }
  446. }
  447. return state;
  448. }
  449. /// <summary>
  450. /// Helper function for ParseConfig.
  451. /// </summary>
  452. /// <param name="settingName"></param>
  453. /// <param name="defaultValue"></param>
  454. /// <param name="global"></param>
  455. /// <param name="local"></param>
  456. /// <returns></returns>
  457. private bool ResolveBoolean(string settingName, bool defaultValue, IConfig global, IConfig local)
  458. {
  459. if(local != null)
  460. {
  461. return local.GetBoolean(settingName, global.GetBoolean(settingName, defaultValue));
  462. }
  463. else
  464. {
  465. return global.GetBoolean(settingName, defaultValue);
  466. }
  467. }
  468. /// <summary>
  469. /// Helper function for ParseConfig.
  470. /// </summary>
  471. /// <param name="settingName"></param>
  472. /// <param name="defaultValue"></param>
  473. /// <param name="global"></param>
  474. /// <param name="local"></param>
  475. /// <returns></returns>
  476. private double ResolveDouble(string settingName, double defaultValue, IConfig global, IConfig local)
  477. {
  478. if (local != null)
  479. {
  480. return local.GetDouble(settingName, global.GetDouble(settingName, defaultValue));
  481. }
  482. else
  483. {
  484. return global.GetDouble(settingName, defaultValue);
  485. }
  486. }
  487. /// <summary>
  488. /// Helper function for ParseConfig.
  489. /// </summary>
  490. /// <param name="settingName"></param>
  491. /// <param name="defaultValue"></param>
  492. /// <param name="global"></param>
  493. /// <param name="local"></param>
  494. /// <returns></returns>
  495. private int ResolveInt(string settingName, int defaultValue, IConfig global, IConfig local)
  496. {
  497. if (local != null)
  498. {
  499. return local.GetInt(settingName, global.GetInt(settingName, defaultValue));
  500. }
  501. else
  502. {
  503. return global.GetInt(settingName, defaultValue);
  504. }
  505. }
  506. /// <summary>
  507. /// Helper function for ParseConfig.
  508. /// </summary>
  509. /// <param name="settingName"></param>
  510. /// <param name="defaultValue"></param>
  511. /// <param name="global"></param>
  512. /// <param name="local"></param>
  513. /// <returns></returns>
  514. private string ResolveString(string settingName, string defaultValue, IConfig global, IConfig local)
  515. {
  516. if (local != null)
  517. {
  518. return local.GetString(settingName, global.GetString(settingName, defaultValue));
  519. }
  520. else
  521. {
  522. return global.GetString(settingName, defaultValue);
  523. }
  524. }
  525. /// <summary>
  526. /// Called when any auto-backup timer expires. This starts the code path for actually performing a backup.
  527. /// </summary>
  528. /// <param name="sender"></param>
  529. /// <param name="e"></param>
  530. private void HandleElapsed(object sender, ElapsedEventArgs e)
  531. {
  532. // TODO: heuristic thresholds are per-region, so we should probably run heuristics once per region
  533. // XXX: Running heuristics once per region could add undue performance penalty for something that's supposed to
  534. // check whether the region is too busy! Especially on sims with LOTS of regions.
  535. // Alternative: make heuristics thresholds global to the module rather than per-region. Less flexible,
  536. // but would allow us to be semantically correct while being easier on perf.
  537. // Alternative 2: Run heuristics once per unique set of heuristics threshold parameters! Ay yi yi...
  538. // Alternative 3: Don't support per-region heuristics at all; just accept them as a global only parameter.
  539. // Since this is pretty experimental, I haven't decided which alternative makes the most sense.
  540. if (this.m_closed)
  541. {
  542. return;
  543. }
  544. bool heuristicsRun = false;
  545. bool heuristicsPassed = false;
  546. if (!this.m_timerMap.ContainsKey((Timer) sender))
  547. {
  548. m_log.Debug("Code-up error: timerMap doesn't contain timer " + sender);
  549. }
  550. List<IScene> tmap = this.m_timerMap[(Timer) sender];
  551. if (tmap != null && tmap.Count > 0)
  552. {
  553. foreach (IScene scene in tmap)
  554. {
  555. AutoBackupModuleState state = this.m_states[scene];
  556. bool heuristics = state.BusyCheck;
  557. // Fast path: heuristics are on; already ran em; and sim is fine; OR, no heuristics for the region.
  558. if ((heuristics && heuristicsRun && heuristicsPassed) || !heuristics)
  559. {
  560. this.DoRegionBackup(scene);
  561. // Heuristics are on; ran but we're too busy -- keep going. Maybe another region will have heuristics off!
  562. }
  563. else if (heuristicsRun)
  564. {
  565. m_log.Info("[AUTO BACKUP]: Heuristics: too busy to backup " +
  566. scene.RegionInfo.RegionName + " right now.");
  567. continue;
  568. // Logical Deduction: heuristics are on but haven't been run
  569. }
  570. else
  571. {
  572. heuristicsPassed = this.RunHeuristics(scene);
  573. heuristicsRun = true;
  574. if (!heuristicsPassed)
  575. {
  576. m_log.Info("[AUTO BACKUP]: Heuristics: too busy to backup " +
  577. scene.RegionInfo.RegionName + " right now.");
  578. continue;
  579. }
  580. this.DoRegionBackup(scene);
  581. }
  582. }
  583. }
  584. }
  585. /// <summary>
  586. /// Save an OAR, register for the callback for when it's done, then call the AutoBackupScript (if applicable).
  587. /// </summary>
  588. /// <param name="scene"></param>
  589. private void DoRegionBackup(IScene scene)
  590. {
  591. if (scene.RegionStatus != RegionStatus.Up)
  592. {
  593. // We won't backup a region that isn't operating normally.
  594. m_log.Warn("[AUTO BACKUP]: Not backing up region " + scene.RegionInfo.RegionName +
  595. " because its status is " + scene.RegionStatus);
  596. return;
  597. }
  598. AutoBackupModuleState state = this.m_states[scene];
  599. IRegionArchiverModule iram = scene.RequestModuleInterface<IRegionArchiverModule>();
  600. string savePath = BuildOarPath(scene.RegionInfo.RegionName,
  601. state.BackupDir,
  602. state.NamingType);
  603. if (savePath == null)
  604. {
  605. m_log.Warn("[AUTO BACKUP]: savePath is null in HandleElapsed");
  606. return;
  607. }
  608. Guid guid = Guid.NewGuid();
  609. m_pendingSaves.Add(guid, scene);
  610. state.LiveRequests.Add(guid, savePath);
  611. ((Scene) scene).EventManager.OnOarFileSaved += new EventManager.OarFileSaved(EventManager_OnOarFileSaved);
  612. iram.ArchiveRegion(savePath, guid, null);
  613. }
  614. /// <summary>
  615. /// Called by the Event Manager when the OnOarFileSaved event is fired.
  616. /// </summary>
  617. /// <param name="guid"></param>
  618. /// <param name="message"></param>
  619. void EventManager_OnOarFileSaved(Guid guid, string message)
  620. {
  621. // Ignore if the OAR save is being done by some other part of the system
  622. if (m_pendingSaves.ContainsKey(guid))
  623. {
  624. AutoBackupModuleState abms = m_states[(m_pendingSaves[guid])];
  625. ExecuteScript(abms.Script, abms.LiveRequests[guid]);
  626. m_pendingSaves.Remove(guid);
  627. abms.LiveRequests.Remove(guid);
  628. }
  629. }
  630. /// <summary>This format may turn out to be too unwieldy to keep...
  631. /// Besides, that's what ctimes are for. But then how do I name each file uniquely without using a GUID?
  632. /// Sequential numbers, right? We support those, too!</summary>
  633. private static string GetTimeString()
  634. {
  635. StringWriter sw = new StringWriter();
  636. sw.Write("_");
  637. DateTime now = DateTime.Now;
  638. sw.Write(now.Year);
  639. sw.Write("y_");
  640. sw.Write(now.Month);
  641. sw.Write("M_");
  642. sw.Write(now.Day);
  643. sw.Write("d_");
  644. sw.Write(now.Hour);
  645. sw.Write("h_");
  646. sw.Write(now.Minute);
  647. sw.Write("m_");
  648. sw.Write(now.Second);
  649. sw.Write("s");
  650. sw.Flush();
  651. string output = sw.ToString();
  652. sw.Close();
  653. return output;
  654. }
  655. /// <summary>Return value of true ==> not too busy; false ==> too busy to backup an OAR right now, or error.</summary>
  656. private bool RunHeuristics(IScene region)
  657. {
  658. try
  659. {
  660. return this.RunTimeDilationHeuristic(region) && this.RunAgentLimitHeuristic(region);
  661. }
  662. catch (Exception e)
  663. {
  664. m_log.Warn("[AUTO BACKUP]: Exception in RunHeuristics", e);
  665. return false;
  666. }
  667. }
  668. /// <summary>
  669. /// If the time dilation right at this instant is less than the threshold specified in AutoBackupDilationThreshold (default 0.5),
  670. /// then we return false and trip the busy heuristic's "too busy" path (i.e. don't save an OAR).
  671. /// AutoBackupDilationThreshold is a _LOWER BOUND_. Lower Time Dilation is bad, so if you go lower than our threshold, it's "too busy".
  672. /// </summary>
  673. /// <param name="region"></param>
  674. /// <returns>Returns true if we're not too busy; false means we've got worse time dilation than the threshold.</returns>
  675. private bool RunTimeDilationHeuristic(IScene region)
  676. {
  677. string regionName = region.RegionInfo.RegionName;
  678. return region.TimeDilation >=
  679. this.m_configSource.Configs["AutoBackupModule"].GetFloat(
  680. regionName + ".AutoBackupDilationThreshold", 0.5f);
  681. }
  682. /// <summary>
  683. /// If the root agent count right at this instant is less than the threshold specified in AutoBackupAgentThreshold (default 10),
  684. /// then we return false and trip the busy heuristic's "too busy" path (i.e., don't save an OAR).
  685. /// AutoBackupAgentThreshold is an _UPPER BOUND_. Higher Agent Count is bad, so if you go higher than our threshold, it's "too busy".
  686. /// </summary>
  687. /// <param name="region"></param>
  688. /// <returns>Returns true if we're not too busy; false means we've got more agents on the sim than the threshold.</returns>
  689. private bool RunAgentLimitHeuristic(IScene region)
  690. {
  691. string regionName = region.RegionInfo.RegionName;
  692. try
  693. {
  694. Scene scene = (Scene) region;
  695. // TODO: Why isn't GetRootAgentCount() a method in the IScene interface? Seems generally useful...
  696. return scene.GetRootAgentCount() <=
  697. this.m_configSource.Configs["AutoBackupModule"].GetInt(
  698. regionName + ".AutoBackupAgentThreshold", 10);
  699. }
  700. catch (InvalidCastException ice)
  701. {
  702. m_log.Debug(
  703. "[AUTO BACKUP]: I NEED MAINTENANCE: IScene is not a Scene; can't get root agent count!",
  704. ice);
  705. return true;
  706. // Non-obstructionist safest answer...
  707. }
  708. }
  709. /// <summary>
  710. /// Run the script or executable specified by the "AutoBackupScript" config setting.
  711. /// Of course this is a security risk if you let anyone modify OpenSim.ini and they want to run some nasty bash script.
  712. /// But there are plenty of other nasty things that can be done with an untrusted OpenSim.ini, such as running high threat level scripting functions.
  713. /// </summary>
  714. /// <param name="scriptName"></param>
  715. /// <param name="savePath"></param>
  716. private static void ExecuteScript(string scriptName, string savePath)
  717. {
  718. // Do nothing if there's no script.
  719. if (scriptName == null || scriptName.Length <= 0)
  720. {
  721. return;
  722. }
  723. try
  724. {
  725. FileInfo fi = new FileInfo(scriptName);
  726. if (fi.Exists)
  727. {
  728. ProcessStartInfo psi = new ProcessStartInfo(scriptName);
  729. psi.Arguments = savePath;
  730. psi.CreateNoWindow = true;
  731. Process proc = Process.Start(psi);
  732. proc.ErrorDataReceived += HandleProcErrorDataReceived;
  733. }
  734. }
  735. catch (Exception e)
  736. {
  737. m_log.Warn(
  738. "Exception encountered when trying to run script for oar backup " + savePath, e);
  739. }
  740. }
  741. /// <summary>
  742. /// Called if a running script process writes to stderr.
  743. /// </summary>
  744. /// <param name="sender"></param>
  745. /// <param name="e"></param>
  746. private static void HandleProcErrorDataReceived(object sender, DataReceivedEventArgs e)
  747. {
  748. m_log.Warn("ExecuteScript hook " + ((Process) sender).ProcessName +
  749. " is yacking on stderr: " + e.Data);
  750. }
  751. /// <summary>
  752. /// Quickly stop all timers from firing.
  753. /// </summary>
  754. private void StopAllTimers()
  755. {
  756. foreach (Timer t in this.m_timerMap.Keys)
  757. {
  758. t.Close();
  759. }
  760. this.m_closed = true;
  761. }
  762. /// <summary>
  763. /// Determine the next unique filename by number, for "Sequential" AutoBackupNamingType.
  764. /// </summary>
  765. /// <param name="dirName"></param>
  766. /// <param name="regionName"></param>
  767. /// <returns></returns>
  768. private static string GetNextFile(string dirName, string regionName)
  769. {
  770. FileInfo uniqueFile = null;
  771. long biggestExistingFile = GetNextOarFileNumber(dirName, regionName);
  772. biggestExistingFile++;
  773. // We don't want to overwrite the biggest existing file; we want to write to the NEXT biggest.
  774. uniqueFile =
  775. new FileInfo(dirName + Path.DirectorySeparatorChar + regionName + "_" +
  776. biggestExistingFile + ".oar");
  777. return uniqueFile.FullName;
  778. }
  779. /// <summary>
  780. /// Top-level method for creating an absolute path to an OAR backup file based on what naming scheme the user wants.
  781. /// </summary>
  782. /// <param name="regionName">Name of the region to save.</param>
  783. /// <param name="baseDir">Absolute or relative path to the directory where the file should reside.</param>
  784. /// <param name="naming">The naming scheme for the file name.</param>
  785. /// <returns></returns>
  786. private static string BuildOarPath(string regionName, string baseDir, NamingType naming)
  787. {
  788. FileInfo path = null;
  789. switch (naming)
  790. {
  791. case NamingType.Overwrite:
  792. path = new FileInfo(baseDir + Path.DirectorySeparatorChar + regionName + ".oar");
  793. return path.FullName;
  794. case NamingType.Time:
  795. path =
  796. new FileInfo(baseDir + Path.DirectorySeparatorChar + regionName +
  797. GetTimeString() + ".oar");
  798. return path.FullName;
  799. case NamingType.Sequential:
  800. // All codepaths in GetNextFile should return a file name ending in .oar
  801. path = new FileInfo(GetNextFile(baseDir, regionName));
  802. return path.FullName;
  803. default:
  804. m_log.Warn("VERY BAD: Unhandled case element " + naming);
  805. break;
  806. }
  807. return null;
  808. }
  809. /// <summary>
  810. /// Helper function for Sequential file naming type (see BuildOarPath and GetNextFile).
  811. /// </summary>
  812. /// <param name="dirName"></param>
  813. /// <param name="regionName"></param>
  814. /// <returns></returns>
  815. private static long GetNextOarFileNumber(string dirName, string regionName)
  816. {
  817. long retval = 1;
  818. DirectoryInfo di = new DirectoryInfo(dirName);
  819. FileInfo[] fi = di.GetFiles(regionName, SearchOption.TopDirectoryOnly);
  820. Array.Sort(fi, (f1, f2) => StringComparer.CurrentCultureIgnoreCase.Compare(f1.Name, f2.Name));
  821. if (fi.LongLength > 0)
  822. {
  823. long subtract = 1L;
  824. bool worked = false;
  825. Regex reg = new Regex(regionName + "_([0-9])+" + ".oar");
  826. while (!worked && subtract <= fi.LongLength)
  827. {
  828. // Pick the file with the last natural ordering
  829. string biggestFileName = fi[fi.LongLength - subtract].Name;
  830. MatchCollection matches = reg.Matches(biggestFileName);
  831. long l = 1;
  832. if (matches.Count > 0 && matches[0].Groups.Count > 0)
  833. {
  834. try
  835. {
  836. long.TryParse(matches[0].Groups[1].Value, out l);
  837. retval = l;
  838. worked = true;
  839. }
  840. catch (FormatException fe)
  841. {
  842. m_log.Warn(
  843. "[AUTO BACKUP]: Error: Can't parse long value from file name to determine next OAR backup file number!",
  844. fe);
  845. subtract++;
  846. }
  847. }
  848. else
  849. {
  850. subtract++;
  851. }
  852. }
  853. }
  854. return retval;
  855. }
  856. }
  857. }