DataSnapshotManager.cs 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425
  1. /*
  2. * Copyright (c) Contributors, http://opensimulator.org/
  3. * See CONTRIBUTORS.TXT for a full list of copyright holders.
  4. *
  5. * Redistribution and use in source and binary forms, with or without
  6. * modification, are permitted provided that the following conditions are met:
  7. * * Redistributions of source code must retain the above copyright
  8. * notice, this list of conditions and the following disclaimer.
  9. * * Redistributions in binary form must reproduce the above copyright
  10. * notice, this list of conditions and the following disclaimer in the
  11. * documentation and/or other materials provided with the distribution.
  12. * * Neither the name of the OpenSim Project nor the
  13. * names of its contributors may be used to endorse or promote products
  14. * derived from this software without specific prior written permission.
  15. *
  16. * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
  17. * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19. * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
  20. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  21. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  22. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  23. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  25. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26. *
  27. */
  28. using System;
  29. using System.Collections.Generic;
  30. using System.IO;
  31. using System.Net;
  32. using System.Reflection;
  33. using System.Text;
  34. using System.Timers;
  35. using System.Xml;
  36. using libsecondlife;
  37. using log4net;
  38. using Nini.Config;
  39. using OpenSim.Framework;
  40. using OpenSim.Framework.Communications;
  41. using OpenSim.Region.DataSnapshot.Interfaces;
  42. using OpenSim.Region.Environment.Interfaces;
  43. using OpenSim.Region.Environment.Scenes;
  44. using libsecondlife.Packets;
  45. namespace OpenSim.Region.DataSnapshot
  46. {
  47. public class DataSnapshotManager : IRegionModule, IDataSnapshot
  48. {
  49. #region Class members
  50. //Information from config
  51. private bool m_enabled = false;
  52. private bool m_configLoaded = false;
  53. private List<String> m_disabledModules = new List<String>();
  54. private Dictionary<string, string> m_gridinfo = new Dictionary<string, string>();
  55. private string m_snapsDir = "DataSnapshot";
  56. //Lists of stuff we need
  57. private List<Scene> m_scenes = new List<Scene>();
  58. private List<IDataSnapshotProvider> m_dataproviders = new List<IDataSnapshotProvider>();
  59. //Various internal objects
  60. private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  61. internal object m_syncInit = new object();
  62. //DataServices and networking
  63. private string m_dataServices = "noservices";
  64. public string m_listener_port = "9000"; //TODO: Set default port over 9000
  65. public string m_hostname = "127.0.0.1";
  66. //Update timers
  67. private Timer m_periodic = null;
  68. private int m_period = 20; // in seconds
  69. private int m_maxStales = 500;
  70. private int m_stales = 0;
  71. private Timer m_passedCheck = null;
  72. private bool m_periodPassed = false;
  73. //Program objects
  74. private SnapshotStore m_snapStore = null;
  75. private DataRequestHandler m_requests = null;
  76. #endregion
  77. #region IRegionModule
  78. public void Close()
  79. {
  80. m_log.Info("[DATASNAPSHOT]: Close called");
  81. }
  82. public void Initialise(Scene scene, IConfigSource config)
  83. {
  84. if (!m_configLoaded) {
  85. m_configLoaded = true;
  86. m_log.Info("[DATASNAPSHOT]: Loading configuration");
  87. //Read from the config for options
  88. lock (m_syncInit)
  89. {
  90. try
  91. {
  92. m_enabled = config.Configs["DataSnapshot"].GetBoolean("index_sims", m_enabled);
  93. if (config.Configs["Startup"].GetBoolean("gridmode", true))
  94. {
  95. m_gridinfo.Add("gridserverURL", config.Configs["Network"].GetString("grid_server_url", "harbl"));
  96. m_gridinfo.Add("userserverURL", config.Configs["Network"].GetString("user_server_url", "harbl"));
  97. m_gridinfo.Add("assetserverURL", config.Configs["Network"].GetString("asset_server_url", "harbl"));
  98. }
  99. else
  100. {
  101. //Non gridmode stuff
  102. }
  103. m_gridinfo.Add("Name", config.Configs["DataSnapshot"].GetString("gridname", "harbl"));
  104. m_period = config.Configs["DataSnapshot"].GetInt("default_snapshot_period", m_period);
  105. m_maxStales = config.Configs["DataSnapshot"].GetInt("max_changes_before_update", m_maxStales);
  106. m_snapsDir = config.Configs["DataSnapshot"].GetString("snapshot_cache_directory", m_snapsDir);
  107. m_dataServices = config.Configs["DataSnapshot"].GetString("data_services", m_dataServices);
  108. m_listener_port = config.Configs["Network"].GetString("http_listener_port", m_listener_port);
  109. String[] annoying_string_array = config.Configs["DataSnapshot"].GetString("disable_modules", "").Split(".".ToCharArray());
  110. foreach (String bloody_wanker in annoying_string_array) {
  111. m_disabledModules.Add(bloody_wanker);
  112. }
  113. } catch (Exception) {
  114. m_log.Info("[DATASNAPSHOT]: Could not load configuration. DataSnapshot will be disabled.");
  115. m_enabled = false;
  116. return;
  117. }
  118. }
  119. if (m_enabled)
  120. {
  121. //Create update timer
  122. m_periodic = new Timer();
  123. m_periodic.Interval = m_period * 1000;
  124. m_periodic.Elapsed += SnapshotTimerCallback;
  125. //Create update eligibility timer
  126. m_passedCheck = new Timer();
  127. m_passedCheck.Interval = m_period * 1000;
  128. m_passedCheck.Elapsed += UpdateEligibilityCallback;
  129. m_passedCheck.Start();
  130. //Hand it the first scene, assuming that all scenes have the same BaseHTTPServer
  131. m_requests = new DataRequestHandler(scene, this);
  132. m_hostname = scene.RegionInfo.ExternalHostName;
  133. m_snapStore = new SnapshotStore(m_snapsDir, m_gridinfo, m_listener_port, m_hostname);
  134. MakeEverythingStale();
  135. if (m_dataServices != "noservices")
  136. NotifyDataServices(m_dataServices);
  137. }
  138. }
  139. if (m_enabled)
  140. {
  141. m_log.Info("[DATASNAPSHOT]: Scene added to module.");
  142. m_snapStore.AddScene(scene);
  143. m_scenes.Add(scene);
  144. Assembly currentasm = Assembly.GetExecutingAssembly();
  145. foreach (Type pluginType in currentasm.GetTypes())
  146. {
  147. if (pluginType.IsPublic)
  148. {
  149. if (!pluginType.IsAbstract)
  150. {
  151. if (pluginType.GetInterface("IDataSnapshotProvider") != null)
  152. {
  153. IDataSnapshotProvider module = (IDataSnapshotProvider)Activator.CreateInstance(pluginType);
  154. module.Initialize(scene, this);
  155. module.OnStale += MarkDataStale;
  156. m_dataproviders.Add(module);
  157. m_snapStore.AddProvider(module);
  158. m_log.Info("[DATASNAPSHOT]: Added new data provider type: " + pluginType.Name);
  159. }
  160. }
  161. }
  162. }
  163. //scene.OnRestart += OnSimRestart;
  164. scene.EventManager.OnShutdown += delegate() { OnSimRestart(scene.RegionInfo); };
  165. }
  166. else
  167. {
  168. m_log.Warn("[DATASNAPSHOT]: Data snapshot disabled, not adding scene to module (or anything else).");
  169. }
  170. }
  171. public bool IsSharedModule
  172. {
  173. get { return true; }
  174. }
  175. public string Name
  176. {
  177. get { return "External Data Generator"; }
  178. }
  179. public void PostInitialise()
  180. {
  181. }
  182. #endregion
  183. #region Associated helper functions
  184. public Scene SceneForName(string name)
  185. {
  186. foreach (Scene scene in m_scenes)
  187. if (scene.RegionInfo.RegionName == name)
  188. return scene;
  189. return null;
  190. }
  191. public Scene SceneForUUID(LLUUID id)
  192. {
  193. foreach (Scene scene in m_scenes)
  194. if (scene.RegionInfo.RegionID == id)
  195. return scene;
  196. return null;
  197. }
  198. #endregion
  199. #region [Public] Snapshot storage functions
  200. /**
  201. * Reply to the http request
  202. */
  203. public XmlDocument GetSnapshot(string regionName)
  204. {
  205. XmlDocument requestedSnap = new XmlDocument();
  206. requestedSnap.AppendChild(requestedSnap.CreateXmlDeclaration("1.0", null, null));
  207. requestedSnap.AppendChild(requestedSnap.CreateWhitespace("\r\n"));
  208. XmlNode regiondata = requestedSnap.CreateNode(XmlNodeType.Element, "regiondata", "");
  209. try
  210. {
  211. if (regionName == null || regionName == "")
  212. {
  213. foreach (Scene scene in m_scenes)
  214. {
  215. regiondata.AppendChild(m_snapStore.GetScene(scene, requestedSnap));
  216. }
  217. }
  218. else
  219. {
  220. Scene scene = SceneForName(regionName);
  221. regiondata.AppendChild(m_snapStore.GetScene(scene, requestedSnap));
  222. }
  223. requestedSnap.AppendChild(regiondata);
  224. regiondata.AppendChild(requestedSnap.CreateWhitespace("\r\n"));
  225. }
  226. catch (XmlException e)
  227. {
  228. m_log.Warn("[DATASNAPSHOT]: XmlException while trying to load snapshot: " + e.ToString());
  229. requestedSnap = GetErrorMessage(regionName, e);
  230. }
  231. catch (Exception e)
  232. {
  233. m_log.Warn("[DATASNAPSHOT]: Caught unknown exception while trying to load snapshot: " + e.StackTrace);
  234. requestedSnap = GetErrorMessage(regionName, e);
  235. }
  236. return requestedSnap;
  237. }
  238. private XmlDocument GetErrorMessage(string regionName, Exception e)
  239. {
  240. XmlDocument errorMessage = new XmlDocument();
  241. XmlNode error = errorMessage.CreateNode(XmlNodeType.Element, "error", "");
  242. XmlNode region = errorMessage.CreateNode(XmlNodeType.Element, "region", "");
  243. region.InnerText = regionName;
  244. XmlNode exception = errorMessage.CreateNode(XmlNodeType.Element, "exception", "");
  245. exception.InnerText = e.ToString();
  246. error.AppendChild(region);
  247. error.AppendChild(exception);
  248. errorMessage.AppendChild(error);
  249. return errorMessage;
  250. }
  251. #endregion
  252. #region External data services
  253. private void NotifyDataServices(string servicesStr)
  254. {
  255. Stream reply = null;
  256. string delimStr = ";";
  257. char [] delimiter = delimStr.ToCharArray();
  258. string[] services = servicesStr.Split(delimiter);
  259. for (int i = 0; i < services.Length; i++)
  260. {
  261. string url = services[i].Trim();
  262. RestClient cli = new RestClient(url);
  263. cli.AddQueryParameter("host", m_hostname);
  264. cli.AddQueryParameter("port", m_listener_port);
  265. cli.RequestMethod = "GET";
  266. try
  267. {
  268. reply = cli.Request();
  269. }
  270. catch (WebException)
  271. {
  272. m_log.Warn("[DATASNAPSHOT] Unable to notify " + url);
  273. }
  274. catch (Exception e)
  275. {
  276. m_log.Warn("[DATASNAPSHOT] Ignoring unknown exception " + e.ToString());
  277. }
  278. byte[] response = new byte[1024];
  279. int n = 0;
  280. try
  281. {
  282. n = reply.Read(response, 0, 1024);
  283. }
  284. catch (Exception e)
  285. {
  286. m_log.Warn("[DATASNAPSHOT] Unable to decode reply from data service. Ignoring. " + e.StackTrace);
  287. }
  288. // This is not quite working, so...
  289. string responseStr = Encoding.UTF8.GetString(response);
  290. m_log.Info("[DATASNAPSHOT] data service notified: " + url);
  291. }
  292. }
  293. #endregion
  294. #region Latency-based update functions
  295. public void MarkDataStale(IDataSnapshotProvider provider)
  296. {
  297. //Behavior here: Wait m_period seconds, then update if there has not been a request in m_period seconds
  298. //or m_maxStales has been exceeded
  299. m_stales++;
  300. if ((m_stales >= m_maxStales) && m_periodPassed)
  301. SnapshotTimerCallback(m_periodic, null);
  302. else if (m_periodic.Enabled == false)
  303. m_periodic.Start();
  304. else
  305. {
  306. m_periodic.Stop();
  307. m_periodic.Start();
  308. }
  309. }
  310. private void SnapshotTimerCallback(object timer, ElapsedEventArgs args)
  311. {
  312. m_log.Debug("[DATASNAPSHOT]: Marking scenes for snapshot updates.");
  313. //Finally generate those snapshot updates
  314. MakeEverythingStale();
  315. //Stop the update delay timer
  316. m_periodic.Stop();
  317. //Reset the eligibility flag and timer
  318. m_periodPassed = false;
  319. m_passedCheck.Stop();
  320. m_passedCheck.Start();
  321. }
  322. private void UpdateEligibilityCallback(object timer, ElapsedEventArgs args)
  323. {
  324. //Set eligibility, so we can start making updates
  325. m_periodPassed = true;
  326. }
  327. public void MakeEverythingStale()
  328. {
  329. m_log.Debug("[DATASNAPSHOT]: Marking all scenes as stale.");
  330. foreach (Scene scene in m_scenes)
  331. {
  332. m_snapStore.ForceSceneStale(scene);
  333. }
  334. }
  335. #endregion
  336. public void OnSimRestart(RegionInfo thisRegion)
  337. {
  338. m_log.Info("[DATASNAPSHOT]: Region " + thisRegion.RegionName + " is restarting, removing from indexing");
  339. Scene restartedScene = SceneForUUID(thisRegion.RegionID);
  340. m_scenes.Remove(restartedScene);
  341. m_snapStore.RemoveScene(restartedScene);
  342. //Getting around the fact that we can't remove objects from a collection we are enumerating over
  343. List<IDataSnapshotProvider> providersToRemove = new List<IDataSnapshotProvider>();
  344. foreach (IDataSnapshotProvider provider in m_dataproviders)
  345. {
  346. if (provider.GetParentScene == restartedScene)
  347. {
  348. providersToRemove.Add(provider);
  349. }
  350. }
  351. foreach (IDataSnapshotProvider provider in providersToRemove)
  352. {
  353. m_dataproviders.Remove(provider);
  354. m_snapStore.RemoveProvider(provider);
  355. }
  356. m_snapStore.RemoveScene(restartedScene);
  357. }
  358. }
  359. }