AssetServicesConnector.cs 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651
  1. /*
  2. * Copyright (c) Contributors, http://opensimulator.org/
  3. * See CONTRIBUTORS.TXT for a full list of copyright holders.
  4. *
  5. * Redistribution and use in source and binary forms, with or without
  6. * modification, are permitted provided that the following conditions are met:
  7. * * Redistributions of source code must retain the above copyright
  8. * notice, this list of conditions and the following disclaimer.
  9. * * Redistributions in binary form must reproduce the above copyright
  10. * notice, this list of conditions and the following disclaimer in the
  11. * documentation and/or other materials provided with the distribution.
  12. * * Neither the name of the OpenSimulator Project nor the
  13. * names of its contributors may be used to endorse or promote products
  14. * derived from this software without specific prior written permission.
  15. *
  16. * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
  17. * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19. * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
  20. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  21. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  22. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  23. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  25. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26. */
  27. using log4net;
  28. using System;
  29. using System.Threading;
  30. using System.Collections.Generic;
  31. using System.Collections.Concurrent;
  32. using System.IO;
  33. using System.Reflection;
  34. using System.Timers;
  35. using Nini.Config;
  36. using OpenSim.Framework;
  37. using OpenSim.Framework.Monitoring;
  38. using OpenSim.Services.Interfaces;
  39. using OpenMetaverse;
  40. namespace OpenSim.Services.Connectors
  41. {
  42. public class AssetServicesConnector : BaseServiceConnector, IAssetService
  43. {
  44. private static readonly ILog m_log =
  45. LogManager.GetLogger(
  46. MethodBase.GetCurrentMethod().DeclaringType);
  47. const int MAXSENDRETRIESLEN = 30;
  48. private string m_ServerURI = String.Empty;
  49. private IAssetCache m_Cache = null;
  50. private int m_retryCounter;
  51. private bool m_inRetries;
  52. private List<AssetBase>[] m_sendRetries = new List<AssetBase>[MAXSENDRETRIESLEN];
  53. private System.Timers.Timer m_retryTimer;
  54. private int m_maxAssetRequestConcurrency = 30;
  55. private delegate void AssetRetrievedEx(AssetBase asset);
  56. // Keeps track of concurrent requests for the same asset, so that it's only loaded once.
  57. // Maps: Asset ID -> Handlers which will be called when the asset has been loaded
  58. // private Dictionary<string, AssetRetrievedEx> m_AssetHandlers = new Dictionary<string, AssetRetrievedEx>();
  59. private Dictionary<string, List<AssetRetrievedEx>> m_AssetHandlers = new Dictionary<string, List<AssetRetrievedEx>>();
  60. private Dictionary<string, string> m_UriMap = new Dictionary<string, string>();
  61. private Thread[] m_fetchThreads;
  62. public int MaxAssetRequestConcurrency
  63. {
  64. get { return m_maxAssetRequestConcurrency; }
  65. set { m_maxAssetRequestConcurrency = value; }
  66. }
  67. public AssetServicesConnector()
  68. {
  69. }
  70. public AssetServicesConnector(string serverURI)
  71. {
  72. m_ServerURI = serverURI.TrimEnd('/');
  73. }
  74. public AssetServicesConnector(IConfigSource source)
  75. : base(source, "AssetService")
  76. {
  77. Initialise(source);
  78. }
  79. public virtual void Initialise(IConfigSource source)
  80. {
  81. IConfig netconfig = source.Configs["Network"];
  82. if (netconfig != null)
  83. m_maxAssetRequestConcurrency = netconfig.GetInt("MaxRequestConcurrency",m_maxAssetRequestConcurrency);
  84. IConfig assetConfig = source.Configs["AssetService"];
  85. if (assetConfig == null)
  86. {
  87. m_log.Error("[ASSET CONNECTOR]: AssetService missing from OpenSim.ini");
  88. throw new Exception("Asset connector init error");
  89. }
  90. string serviceURI = assetConfig.GetString("AssetServerURI",
  91. String.Empty);
  92. m_ServerURI = serviceURI;
  93. if (serviceURI == String.Empty)
  94. {
  95. m_log.Error("[ASSET CONNECTOR]: No Server URI named in section AssetService");
  96. throw new Exception("Asset connector init error");
  97. }
  98. m_retryTimer = new System.Timers.Timer();
  99. m_retryTimer.Elapsed += new ElapsedEventHandler(retryCheck);
  100. m_retryTimer.AutoReset = true;
  101. m_retryTimer.Interval = 60000;
  102. Uri serverUri = new Uri(m_ServerURI);
  103. string groupHost = serverUri.Host;
  104. for (int i = 0 ; i < 256 ; i++)
  105. {
  106. string prefix = i.ToString("x2");
  107. groupHost = assetConfig.GetString("AssetServerHost_"+prefix, groupHost);
  108. m_UriMap[prefix] = groupHost;
  109. //m_log.DebugFormat("[ASSET]: Using {0} for prefix {1}", groupHost, prefix);
  110. }
  111. m_fetchThreads = new Thread[2];
  112. for (int i = 0 ; i < 2 ; i++)
  113. {
  114. m_fetchThreads[i] = WorkManager.StartThread(AssetRequestProcessor, String.Format("GetAssetsWorker{0}", i));
  115. }
  116. }
  117. private string MapServer(string id)
  118. {
  119. if (m_UriMap.Count == 0)
  120. return m_ServerURI;
  121. UriBuilder serverUri = new UriBuilder(m_ServerURI);
  122. string prefix = id.Substring(0, 2).ToLower();
  123. string host;
  124. // HG URLs will not be valid UUIDS
  125. if (m_UriMap.ContainsKey(prefix))
  126. host = m_UriMap[prefix];
  127. else
  128. host = m_UriMap["00"];
  129. serverUri.Host = host;
  130. // m_log.DebugFormat("[ASSET]: Using {0} for host name for prefix {1}", host, prefix);
  131. string ret = serverUri.Uri.AbsoluteUri;
  132. if (ret.EndsWith("/"))
  133. ret = ret.Substring(0, ret.Length - 1);
  134. return ret;
  135. }
  136. protected void retryCheck(object source, ElapsedEventArgs e)
  137. {
  138. lock(m_sendRetries)
  139. {
  140. if(m_inRetries)
  141. return;
  142. m_inRetries = true;
  143. }
  144. m_retryCounter++;
  145. if(m_retryCounter >= 61 ) // avoid overflow 60 is max in use below
  146. m_retryCounter = 1;
  147. int inUse = 0;
  148. int nextlevel;
  149. int timefactor;
  150. List<AssetBase> retrylist;
  151. // we need to go down
  152. for(int i = MAXSENDRETRIESLEN - 1; i >= 0; i--)
  153. {
  154. lock(m_sendRetries)
  155. retrylist = m_sendRetries[i];
  156. if(retrylist == null)
  157. continue;
  158. inUse++;
  159. nextlevel = i + 1;
  160. //We exponentially fall back on frequency until we reach one attempt per hour
  161. //The net result is that we end up in the queue for roughly 24 hours..
  162. //24 hours worth of assets could be a lot, so the hope is that the region admin
  163. //will have gotten the asset connector back online quickly!
  164. if(i == 0)
  165. timefactor = 1;
  166. else
  167. {
  168. timefactor = 1 << nextlevel;
  169. if (timefactor > 60)
  170. timefactor = 60;
  171. }
  172. if(m_retryCounter < timefactor)
  173. continue; // to update inUse;
  174. if (m_retryCounter % timefactor != 0)
  175. continue;
  176. // a list to retry
  177. lock(m_sendRetries)
  178. m_sendRetries[i] = null;
  179. // we are the only ones with a copy of this retrylist now
  180. foreach(AssetBase ass in retrylist)
  181. retryStore(ass, nextlevel);
  182. }
  183. lock(m_sendRetries)
  184. {
  185. if(inUse == 0 )
  186. m_retryTimer.Stop();
  187. m_inRetries = false;
  188. }
  189. }
  190. protected void SetCache(IAssetCache cache)
  191. {
  192. m_Cache = cache;
  193. }
  194. public AssetBase Get(string id)
  195. {
  196. string uri = MapServer(id) + "/assets/" + id;
  197. AssetBase asset = null;
  198. if (m_Cache != null)
  199. {
  200. if (!m_Cache.Get(id, out asset))
  201. return null;
  202. }
  203. if (asset == null || asset.Data == null || asset.Data.Length == 0)
  204. {
  205. // XXX: Commented out for now since this has either never been properly operational or not for some time
  206. // as m_maxAssetRequestConcurrency was being passed as the timeout, not a concurrency limiting option.
  207. // Wasn't noticed before because timeout wasn't actually used.
  208. // Not attempting concurrency setting for now as this omission was discovered in release candidate
  209. // phase for OpenSimulator 0.8. Need to revisit afterwards.
  210. // asset
  211. // = SynchronousRestObjectRequester.MakeRequest<int, AssetBase>(
  212. // "GET", uri, 0, m_maxAssetRequestConcurrency);
  213. asset = SynchronousRestObjectRequester.MakeRequest<int, AssetBase>("GET", uri, 0, m_Auth);
  214. if (m_Cache != null)
  215. {
  216. if (asset != null)
  217. m_Cache.Cache(asset);
  218. else
  219. m_Cache.CacheNegative(id);
  220. }
  221. }
  222. return asset;
  223. }
  224. public AssetBase GetCached(string id)
  225. {
  226. // m_log.DebugFormat("[ASSET SERVICE CONNECTOR]: Cache request for {0}", id);
  227. AssetBase asset = null;
  228. if (m_Cache != null)
  229. {
  230. m_Cache.Get(id, out asset);
  231. }
  232. return asset;
  233. }
  234. public AssetMetadata GetMetadata(string id)
  235. {
  236. if (m_Cache != null)
  237. {
  238. AssetBase fullAsset;
  239. if (!m_Cache.Get(id, out fullAsset))
  240. return null;
  241. if (fullAsset != null)
  242. return fullAsset.Metadata;
  243. }
  244. string uri = MapServer(id) + "/assets/" + id + "/metadata";
  245. AssetMetadata asset = SynchronousRestObjectRequester.MakeRequest<int, AssetMetadata>("GET", uri, 0, m_Auth);
  246. return asset;
  247. }
  248. public byte[] GetData(string id)
  249. {
  250. if (m_Cache != null)
  251. {
  252. AssetBase fullAsset;
  253. if (!m_Cache.Get(id, out fullAsset))
  254. return null;
  255. if (fullAsset != null)
  256. return fullAsset.Data;
  257. }
  258. using (RestClient rc = new RestClient(MapServer(id)))
  259. {
  260. rc.AddResourcePath("assets");
  261. rc.AddResourcePath(id);
  262. rc.AddResourcePath("data");
  263. rc.RequestMethod = "GET";
  264. using (Stream s = rc.Request(m_Auth))
  265. {
  266. if (s == null)
  267. return null;
  268. if (s.Length > 0)
  269. {
  270. byte[] ret = new byte[s.Length];
  271. s.Read(ret, 0, (int)s.Length);
  272. return ret;
  273. }
  274. }
  275. return null;
  276. }
  277. }
  278. private class QueuedAssetRequest
  279. {
  280. public string uri;
  281. public string id;
  282. }
  283. private BlockingCollection<QueuedAssetRequest> m_requestQueue = new BlockingCollection<QueuedAssetRequest>();
  284. private void AssetRequestProcessor()
  285. {
  286. QueuedAssetRequest r;
  287. while (true)
  288. {
  289. if(!m_requestQueue.TryTake(out r, 4500) || r == null)
  290. {
  291. Watchdog.UpdateThread();
  292. continue;
  293. }
  294. Watchdog.UpdateThread();
  295. string uri = r.uri;
  296. string id = r.id;
  297. try
  298. {
  299. AssetBase a = SynchronousRestObjectRequester.MakeRequest<int, AssetBase>("GET", uri, 0, 30000, m_Auth);
  300. if (a != null && m_Cache != null)
  301. m_Cache.Cache(a);
  302. List<AssetRetrievedEx> handlers;
  303. lock (m_AssetHandlers)
  304. {
  305. handlers = m_AssetHandlers[id];
  306. m_AssetHandlers.Remove(id);
  307. }
  308. if(handlers != null)
  309. {
  310. Util.FireAndForget(x =>
  311. {
  312. foreach (AssetRetrievedEx h in handlers)
  313. {
  314. try { h.Invoke(a); }
  315. catch { }
  316. }
  317. handlers.Clear();
  318. });
  319. }
  320. }
  321. catch { }
  322. }
  323. }
  324. public bool Get(string id, Object sender, AssetRetrieved handler)
  325. {
  326. string uri = MapServer(id) + "/assets/" + id;
  327. AssetBase asset = null;
  328. if (m_Cache != null)
  329. {
  330. if (!m_Cache.Get(id, out asset))
  331. return false;
  332. }
  333. if (asset == null || asset.Data == null || asset.Data.Length == 0)
  334. {
  335. lock (m_AssetHandlers)
  336. {
  337. AssetRetrievedEx handlerEx = new AssetRetrievedEx(delegate(AssetBase _asset) { handler(id, sender, _asset); });
  338. List<AssetRetrievedEx> handlers;
  339. if (m_AssetHandlers.TryGetValue(id, out handlers))
  340. {
  341. // Someone else is already loading this asset. It will notify our handler when done.
  342. handlers.Add(handlerEx);
  343. return true;
  344. }
  345. handlers = new List<AssetRetrievedEx>();
  346. handlers.Add(handlerEx);
  347. m_AssetHandlers.Add(id, handlers);
  348. QueuedAssetRequest request = new QueuedAssetRequest();
  349. request.id = id;
  350. request.uri = uri;
  351. m_requestQueue.Add(request);
  352. }
  353. }
  354. else
  355. {
  356. handler(id, sender, asset);
  357. }
  358. return true;
  359. }
  360. public virtual bool[] AssetsExist(string[] ids)
  361. {
  362. string uri = m_ServerURI + "/get_assets_exist";
  363. bool[] exist = null;
  364. try
  365. {
  366. exist = SynchronousRestObjectRequester.MakeRequest<string[], bool[]>("POST", uri, ids, m_Auth);
  367. }
  368. catch (Exception)
  369. {
  370. // This is most likely to happen because the server doesn't support this function,
  371. // so just silently return "doesn't exist" for all the assets.
  372. }
  373. if (exist == null)
  374. exist = new bool[ids.Length];
  375. return exist;
  376. }
  377. string stringUUIDZero = UUID.Zero.ToString();
  378. public string Store(AssetBase asset)
  379. {
  380. // Have to assign the asset ID here. This isn't likely to
  381. // trigger since current callers don't pass emtpy IDs
  382. // We need the asset ID to route the request to the proper
  383. // cluster member, so we can't have the server assign one.
  384. if (asset.ID == string.Empty || asset.ID == stringUUIDZero)
  385. {
  386. if (asset.FullID == UUID.Zero)
  387. {
  388. asset.FullID = UUID.Random();
  389. }
  390. m_log.WarnFormat("[Assets] Zero ID: {0}",asset.Name);
  391. asset.ID = asset.FullID.ToString();
  392. }
  393. if (asset.FullID == UUID.Zero)
  394. {
  395. UUID uuid = UUID.Zero;
  396. if (UUID.TryParse(asset.ID, out uuid))
  397. {
  398. asset.FullID = uuid;
  399. }
  400. if(asset.FullID == UUID.Zero)
  401. {
  402. m_log.WarnFormat("[Assets] Zero IDs: {0}",asset.Name);
  403. asset.FullID = UUID.Random();
  404. asset.ID = asset.FullID.ToString();
  405. }
  406. }
  407. if (m_Cache != null)
  408. m_Cache.Cache(asset);
  409. if (asset.Temporary || asset.Local)
  410. {
  411. return asset.ID;
  412. }
  413. string uri = MapServer(asset.FullID.ToString()) + "/assets/";
  414. string newID = null;
  415. try
  416. {
  417. newID = SynchronousRestObjectRequester.
  418. MakeRequest<AssetBase, string>("POST", uri, asset, 100000, m_Auth);
  419. }
  420. catch
  421. {
  422. newID = null;
  423. }
  424. if (newID == null || newID == String.Empty || newID == stringUUIDZero)
  425. {
  426. //The asset upload failed, try later
  427. lock(m_sendRetries)
  428. {
  429. if (m_sendRetries[0] == null)
  430. m_sendRetries[0] = new List<AssetBase>();
  431. List<AssetBase> m_queue = m_sendRetries[0];
  432. m_queue.Add(asset);
  433. m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later",
  434. asset.ID.ToString(), asset.Type.ToString());
  435. m_retryTimer.Start();
  436. }
  437. }
  438. else
  439. {
  440. if (newID != asset.ID)
  441. {
  442. // Placing this here, so that this work with old asset servers that don't send any reply back
  443. // SynchronousRestObjectRequester returns somethins that is not an empty string
  444. asset.ID = newID;
  445. if (m_Cache != null)
  446. m_Cache.Cache(asset);
  447. }
  448. }
  449. return asset.ID;
  450. }
  451. public void retryStore(AssetBase asset, int nextRetryLevel)
  452. {
  453. /* this may be bad, so excluding
  454. if (m_Cache != null && !m_Cache.Check(asset.ID))
  455. {
  456. m_log.WarnFormat("[Assets] Upload giveup asset bc no longer in local cache: {0}",
  457. asset.ID.ToString();
  458. return; // if no longer in cache, it was deleted or expired
  459. }
  460. */
  461. string uri = MapServer(asset.FullID.ToString()) + "/assets/";
  462. string newID = null;
  463. try
  464. {
  465. newID = SynchronousRestObjectRequester.
  466. MakeRequest<AssetBase, string>("POST", uri, asset, 100000, m_Auth);
  467. }
  468. catch
  469. {
  470. newID = null;
  471. }
  472. if (newID == null || newID == String.Empty || newID == stringUUIDZero)
  473. {
  474. if(nextRetryLevel >= MAXSENDRETRIESLEN)
  475. m_log.WarnFormat("[Assets] Upload giveup after several retries id: {0} type {1}",
  476. asset.ID.ToString(), asset.Type.ToString());
  477. else
  478. {
  479. lock(m_sendRetries)
  480. {
  481. if (m_sendRetries[nextRetryLevel] == null)
  482. {
  483. m_sendRetries[nextRetryLevel] = new List<AssetBase>();
  484. }
  485. List<AssetBase> m_queue = m_sendRetries[nextRetryLevel];
  486. m_queue.Add(asset);
  487. m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later",
  488. asset.ID.ToString(), asset.Type.ToString());
  489. }
  490. }
  491. }
  492. else
  493. {
  494. m_log.InfoFormat("[Assets] Upload of {0} succeeded after {1} failed attempts", asset.ID.ToString(), nextRetryLevel.ToString());
  495. if (newID != asset.ID)
  496. {
  497. asset.ID = newID;
  498. if (m_Cache != null)
  499. m_Cache.Cache(asset);
  500. }
  501. }
  502. }
  503. public bool UpdateContent(string id, byte[] data)
  504. {
  505. AssetBase asset = null;
  506. if (m_Cache != null)
  507. m_Cache.Get(id, out asset);
  508. if (asset == null)
  509. {
  510. AssetMetadata metadata = GetMetadata(id);
  511. if (metadata == null)
  512. return false;
  513. asset = new AssetBase(metadata.FullID, metadata.Name, metadata.Type, UUID.Zero.ToString());
  514. asset.Metadata = metadata;
  515. }
  516. asset.Data = data;
  517. string uri = MapServer(id) + "/assets/" + id;
  518. if (SynchronousRestObjectRequester.MakeRequest<AssetBase, bool>("POST", uri, asset, m_Auth))
  519. {
  520. if (m_Cache != null)
  521. m_Cache.Cache(asset);
  522. return true;
  523. }
  524. return false;
  525. }
  526. public bool Delete(string id)
  527. {
  528. string uri = MapServer(id) + "/assets/" + id;
  529. if (SynchronousRestObjectRequester.MakeRequest<int, bool>("DELETE", uri, 0, m_Auth))
  530. {
  531. if (m_Cache != null)
  532. m_Cache.Expire(id);
  533. return true;
  534. }
  535. return false;
  536. }
  537. }
  538. }