DataSnapshotManager.cs 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461
  1. /*
  2. * Copyright (c) Contributors, http://opensimulator.org/
  3. * See CONTRIBUTORS.TXT for a full list of copyright holders.
  4. *
  5. * Redistribution and use in source and binary forms, with or without
  6. * modification, are permitted provided that the following conditions are met:
  7. * * Redistributions of source code must retain the above copyright
  8. * notice, this list of conditions and the following disclaimer.
  9. * * Redistributions in binary form must reproduce the above copyright
  10. * notice, this list of conditions and the following disclaimer in the
  11. * documentation and/or other materials provided with the distribution.
  12. * * Neither the name of the OpenSimulator Project nor the
  13. * names of its contributors may be used to endorse or promote products
  14. * derived from this software without specific prior written permission.
  15. *
  16. * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
  17. * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19. * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
  20. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  21. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  22. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  23. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  25. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26. *
  27. */
  28. using System;
  29. using System.Collections.Generic;
  30. using System.IO;
  31. using System.Net;
  32. using System.Reflection;
  33. using System.Xml;
  34. using log4net;
  35. using Nini.Config;
  36. using OpenMetaverse;
  37. using Mono.Addins;
  38. using OpenSim.Framework;
  39. using OpenSim.Framework.Communications;
  40. using OpenSim.Region.DataSnapshot.Interfaces;
  41. using OpenSim.Region.Framework.Interfaces;
  42. using OpenSim.Region.Framework.Scenes;
  43. [assembly: Addin("DataSnapshot", "0.1")]
  44. [assembly: AddinDependency("OpenSim", "0.5")]
  45. namespace OpenSim.Region.DataSnapshot
  46. {
  47. [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "DataSnapshotManager")]
  48. public class DataSnapshotManager : ISharedRegionModule, IDataSnapshot
  49. {
  50. #region Class members
  51. //Information from config
  52. private bool m_enabled = false;
  53. private bool m_configLoaded = false;
  54. private List<string> m_disabledModules = new List<string>();
  55. private Dictionary<string, string> m_gridinfo = new Dictionary<string, string>();
  56. private string m_snapsDir = "DataSnapshot";
  57. private string m_exposure_level = "minimum";
  58. //Lists of stuff we need
  59. private List<Scene> m_scenes = new List<Scene>();
  60. private List<IDataSnapshotProvider> m_dataproviders = new List<IDataSnapshotProvider>();
  61. //Various internal objects
  62. private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  63. internal object m_syncInit = new object();
  64. //DataServices and networking
  65. private string m_dataServices = "noservices";
  66. public string m_listener_port = ConfigSettings.DefaultRegionHttpPort.ToString();
  67. public string m_hostname = "127.0.0.1";
  68. private UUID m_Secret = UUID.Random();
  69. private bool m_servicesNotified = false;
  70. //Update timers
  71. private int m_period = 20; // in seconds
  72. private int m_maxStales = 500;
  73. private int m_stales = 0;
  74. private int m_lastUpdate = 0;
  75. //Program objects
  76. private SnapshotStore m_snapStore = null;
  77. #endregion
  78. #region Properties
  79. public string ExposureLevel
  80. {
  81. get { return m_exposure_level; }
  82. }
  83. public UUID Secret
  84. {
  85. get { return m_Secret; }
  86. }
  87. #endregion
  88. #region Region Module interface
  89. public void Initialise(IConfigSource config)
  90. {
  91. if (!m_configLoaded)
  92. {
  93. m_configLoaded = true;
  94. //m_log.Debug("[DATASNAPSHOT]: Loading configuration");
  95. //Read from the config for options
  96. lock (m_syncInit)
  97. {
  98. try
  99. {
  100. m_enabled = config.Configs["DataSnapshot"].GetBoolean("index_sims", m_enabled);
  101. string gatekeeper = Util.GetConfigVarFromSections<string>(config, "GatekeeperURI",
  102. new string[] { "Startup", "Hypergrid", "GridService" }, String.Empty);
  103. // Legacy. Remove soon!
  104. if (string.IsNullOrEmpty(gatekeeper))
  105. {
  106. IConfig conf = config.Configs["GridService"];
  107. if (conf != null)
  108. gatekeeper = conf.GetString("Gatekeeper", gatekeeper);
  109. }
  110. if (!string.IsNullOrEmpty(gatekeeper))
  111. m_gridinfo.Add("gatekeeperURL", gatekeeper);
  112. m_gridinfo.Add(
  113. "name", config.Configs["DataSnapshot"].GetString("gridname", "the lost continent of hippo"));
  114. m_exposure_level = config.Configs["DataSnapshot"].GetString("data_exposure", m_exposure_level);
  115. m_period = config.Configs["DataSnapshot"].GetInt("default_snapshot_period", m_period);
  116. m_maxStales = config.Configs["DataSnapshot"].GetInt("max_changes_before_update", m_maxStales);
  117. m_snapsDir = config.Configs["DataSnapshot"].GetString("snapshot_cache_directory", m_snapsDir);
  118. m_dataServices = config.Configs["DataSnapshot"].GetString("data_services", m_dataServices);
  119. m_listener_port = config.Configs["Network"].GetString("http_listener_port", m_listener_port);
  120. String[] annoying_string_array = config.Configs["DataSnapshot"].GetString("disable_modules", "").Split(".".ToCharArray());
  121. foreach (String bloody_wanker in annoying_string_array)
  122. {
  123. m_disabledModules.Add(bloody_wanker);
  124. }
  125. m_lastUpdate = Environment.TickCount;
  126. }
  127. catch (Exception)
  128. {
  129. m_log.Warn("[DATASNAPSHOT]: Could not load configuration. DataSnapshot will be disabled.");
  130. m_enabled = false;
  131. return;
  132. }
  133. }
  134. }
  135. }
  136. public void AddRegion(Scene scene)
  137. {
  138. if (!m_enabled)
  139. return;
  140. m_log.DebugFormat("[DATASNAPSHOT]: Module added to Scene {0}.", scene.RegionInfo.RegionName);
  141. if (!m_servicesNotified)
  142. {
  143. m_hostname = scene.RegionInfo.ExternalHostName;
  144. m_snapStore = new SnapshotStore(m_snapsDir, m_gridinfo, m_listener_port, m_hostname);
  145. //Hand it the first scene, assuming that all scenes have the same BaseHTTPServer
  146. new DataRequestHandler(scene, this);
  147. if (m_dataServices != "" && m_dataServices != "noservices")
  148. NotifyDataServices(m_dataServices, "online");
  149. m_servicesNotified = true;
  150. }
  151. m_scenes.Add(scene);
  152. m_snapStore.AddScene(scene);
  153. Assembly currentasm = Assembly.GetExecutingAssembly();
  154. foreach (Type pluginType in currentasm.GetTypes())
  155. {
  156. if (pluginType.IsPublic)
  157. {
  158. if (!pluginType.IsAbstract)
  159. {
  160. if (pluginType.GetInterface("IDataSnapshotProvider") != null)
  161. {
  162. IDataSnapshotProvider module = (IDataSnapshotProvider)Activator.CreateInstance(pluginType);
  163. module.Initialize(scene, this);
  164. module.OnStale += MarkDataStale;
  165. m_dataproviders.Add(module);
  166. m_snapStore.AddProvider(module);
  167. m_log.Debug("[DATASNAPSHOT]: Added new data provider type: " + pluginType.Name);
  168. }
  169. }
  170. }
  171. }
  172. }
  173. public void RemoveRegion(Scene scene)
  174. {
  175. if (!m_enabled)
  176. return;
  177. m_log.Info("[DATASNAPSHOT]: Region " + scene.RegionInfo.RegionName + " is being removed, removing from indexing");
  178. Scene restartedScene = SceneForUUID(scene.RegionInfo.RegionID);
  179. m_scenes.Remove(restartedScene);
  180. m_snapStore.RemoveScene(restartedScene);
  181. //Getting around the fact that we can't remove objects from a collection we are enumerating over
  182. List<IDataSnapshotProvider> providersToRemove = new List<IDataSnapshotProvider>();
  183. foreach (IDataSnapshotProvider provider in m_dataproviders)
  184. {
  185. if (provider.GetParentScene == restartedScene)
  186. {
  187. providersToRemove.Add(provider);
  188. }
  189. }
  190. foreach (IDataSnapshotProvider provider in providersToRemove)
  191. {
  192. m_dataproviders.Remove(provider);
  193. m_snapStore.RemoveProvider(provider);
  194. }
  195. m_snapStore.RemoveScene(restartedScene);
  196. }
  197. public void PostInitialise()
  198. {
  199. }
  200. public void RegionLoaded(Scene scene)
  201. {
  202. if (!m_enabled)
  203. return;
  204. m_log.DebugFormat("[DATASNAPSHOT]: Marking scene {0} as stale.", scene.RegionInfo.RegionName);
  205. m_snapStore.ForceSceneStale(scene);
  206. }
  207. public void Close()
  208. {
  209. if (!m_enabled)
  210. return;
  211. if (m_enabled && m_dataServices != "" && m_dataServices != "noservices")
  212. NotifyDataServices(m_dataServices, "offline");
  213. }
  214. public string Name
  215. {
  216. get { return "External Data Generator"; }
  217. }
  218. public Type ReplaceableInterface
  219. {
  220. get { return null; }
  221. }
  222. #endregion
  223. #region Associated helper functions
  224. public Scene SceneForName(string name)
  225. {
  226. foreach (Scene scene in m_scenes)
  227. if (scene.RegionInfo.RegionName == name)
  228. return scene;
  229. return null;
  230. }
  231. public Scene SceneForUUID(UUID id)
  232. {
  233. foreach (Scene scene in m_scenes)
  234. if (scene.RegionInfo.RegionID == id)
  235. return scene;
  236. return null;
  237. }
  238. #endregion
  239. #region [Public] Snapshot storage functions
  240. /**
  241. * Reply to the http request
  242. */
  243. public XmlDocument GetSnapshot(string regionName)
  244. {
  245. CheckStale();
  246. XmlDocument requestedSnap = new XmlDocument();
  247. requestedSnap.AppendChild(requestedSnap.CreateXmlDeclaration("1.0", null, null));
  248. requestedSnap.AppendChild(requestedSnap.CreateWhitespace("\r\n"));
  249. XmlNode regiondata = requestedSnap.CreateNode(XmlNodeType.Element, "regiondata", "");
  250. try
  251. {
  252. if (string.IsNullOrEmpty(regionName))
  253. {
  254. XmlNode timerblock = requestedSnap.CreateNode(XmlNodeType.Element, "expire", "");
  255. timerblock.InnerText = m_period.ToString();
  256. regiondata.AppendChild(timerblock);
  257. regiondata.AppendChild(requestedSnap.CreateWhitespace("\r\n"));
  258. foreach (Scene scene in m_scenes)
  259. {
  260. regiondata.AppendChild(m_snapStore.GetScene(scene, requestedSnap));
  261. }
  262. }
  263. else
  264. {
  265. Scene scene = SceneForName(regionName);
  266. regiondata.AppendChild(m_snapStore.GetScene(scene, requestedSnap));
  267. }
  268. requestedSnap.AppendChild(regiondata);
  269. regiondata.AppendChild(requestedSnap.CreateWhitespace("\r\n"));
  270. }
  271. catch (XmlException e)
  272. {
  273. m_log.Warn("[DATASNAPSHOT]: XmlException while trying to load snapshot: " + e.ToString());
  274. requestedSnap = GetErrorMessage(regionName, e);
  275. }
  276. catch (Exception e)
  277. {
  278. m_log.Warn("[DATASNAPSHOT]: Caught unknown exception while trying to load snapshot: " + e.StackTrace);
  279. requestedSnap = GetErrorMessage(regionName, e);
  280. }
  281. return requestedSnap;
  282. }
  283. private XmlDocument GetErrorMessage(string regionName, Exception e)
  284. {
  285. XmlDocument errorMessage = new XmlDocument();
  286. XmlNode error = errorMessage.CreateNode(XmlNodeType.Element, "error", "");
  287. XmlNode region = errorMessage.CreateNode(XmlNodeType.Element, "region", "");
  288. region.InnerText = regionName;
  289. XmlNode exception = errorMessage.CreateNode(XmlNodeType.Element, "exception", "");
  290. exception.InnerText = e.ToString();
  291. error.AppendChild(region);
  292. error.AppendChild(exception);
  293. errorMessage.AppendChild(error);
  294. return errorMessage;
  295. }
  296. #endregion
  297. #region External data services
  298. private void NotifyDataServices(string servicesStr, string serviceName)
  299. {
  300. Stream reply = null;
  301. string delimStr = ";";
  302. char [] delimiter = delimStr.ToCharArray();
  303. string[] services = servicesStr.Split(delimiter);
  304. for (int i = 0; i < services.Length; i++)
  305. {
  306. string url = services[i].Trim();
  307. RestClient cli = new RestClient(url);
  308. cli.AddQueryParameter("service", serviceName);
  309. cli.AddQueryParameter("host", m_hostname);
  310. cli.AddQueryParameter("port", m_listener_port);
  311. cli.AddQueryParameter("secret", m_Secret.ToString());
  312. cli.RequestMethod = "GET";
  313. try
  314. {
  315. reply = cli.Request();
  316. }
  317. catch (WebException)
  318. {
  319. m_log.Warn("[DATASNAPSHOT]: Unable to notify " + url);
  320. }
  321. catch (Exception e)
  322. {
  323. m_log.Warn("[DATASNAPSHOT]: Ignoring unknown exception " + e.ToString());
  324. }
  325. byte[] response = new byte[1024];
  326. // int n = 0;
  327. try
  328. {
  329. // n = reply.Read(response, 0, 1024);
  330. reply.Read(response, 0, 1024);
  331. }
  332. catch (Exception e)
  333. {
  334. m_log.WarnFormat("[DATASNAPSHOT]: Unable to decode reply from data service. Ignoring. {0}", e.StackTrace);
  335. }
  336. // This is not quite working, so...
  337. // string responseStr = Util.UTF8.GetString(response);
  338. m_log.Info("[DATASNAPSHOT]: data service " + url + " notified. Secret: " + m_Secret);
  339. }
  340. }
  341. #endregion
  342. #region Latency-based update functions
  343. public void MarkDataStale(IDataSnapshotProvider provider)
  344. {
  345. //Behavior here: Wait m_period seconds, then update if there has not been a request in m_period seconds
  346. //or m_maxStales has been exceeded
  347. m_stales++;
  348. }
  349. private void CheckStale()
  350. {
  351. // Wrap check
  352. if (Environment.TickCount < m_lastUpdate)
  353. {
  354. m_lastUpdate = Environment.TickCount;
  355. }
  356. if (m_stales >= m_maxStales)
  357. {
  358. if (Environment.TickCount - m_lastUpdate >= 20000)
  359. {
  360. m_stales = 0;
  361. m_lastUpdate = Environment.TickCount;
  362. MakeEverythingStale();
  363. }
  364. }
  365. else
  366. {
  367. if (m_lastUpdate + 1000 * m_period < Environment.TickCount)
  368. {
  369. m_stales = 0;
  370. m_lastUpdate = Environment.TickCount;
  371. MakeEverythingStale();
  372. }
  373. }
  374. }
  375. public void MakeEverythingStale()
  376. {
  377. m_log.Debug("[DATASNAPSHOT]: Marking all scenes as stale.");
  378. foreach (Scene scene in m_scenes)
  379. {
  380. m_snapStore.ForceSceneStale(scene);
  381. }
  382. }
  383. #endregion
  384. }
  385. }