WebUtil.cs 49 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271
  1. /*
  2. * Copyright (c) Contributors, http://opensimulator.org/
  3. * See CONTRIBUTORS.TXT for a full list of copyright holders.
  4. *
  5. * Redistribution and use in source and binary forms, with or without
  6. * modification, are permitted provided that the following conditions are met:
  7. * * Redistributions of source code must retain the above copyright
  8. * notice, this list of conditions and the following disclaimer.
  9. * * Redistributions in binary form must reproduce the above copyright
  10. * notice, this list of conditions and the following disclaimer in the
  11. * documentation and/or other materials provided with the distribution.
  12. * * Neither the name of the OpenSimulator Project nor the
  13. * names of its contributors may be used to endorse or promote products
  14. * derived from this software without specific prior written permission.
  15. *
  16. * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
  17. * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19. * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
  20. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  21. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  22. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  23. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  25. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26. */
  27. using System;
  28. using System.Collections;
  29. using System.Collections.Generic;
  30. using System.Collections.Specialized;
  31. using System.Globalization;
  32. using System.IO;
  33. using System.IO.Compression;
  34. using System.Net;
  35. using System.Net.Security;
  36. using System.Reflection;
  37. using System.Text;
  38. using System.Web;
  39. using System.Xml;
  40. using System.Xml.Serialization;
  41. using log4net;
  42. using OpenMetaverse.StructuredData;
  43. namespace OpenSim.Framework
  44. {
  45. /// <summary>
  46. /// Miscellaneous static methods and extension methods related to the web
  47. /// </summary>
  48. public static class WebUtil
  49. {
  50. private static readonly ILog m_log =
  51. LogManager.GetLogger(
  52. MethodBase.GetCurrentMethod().DeclaringType);
  53. /// <summary>
  54. /// Control the printing of certain debug messages.
  55. /// </summary>
  56. /// <remarks>
  57. /// If DebugLevel >= 3 then short notices about outgoing HTTP requests are logged.
  58. /// </remarks>
  59. public static int DebugLevel { get; set; }
  60. /// <summary>
  61. /// Request number for diagnostic purposes.
  62. /// </summary>
  63. public static int RequestNumber { get; internal set; }
  64. /// <summary>
  65. /// Control where OSD requests should be serialized per endpoint.
  66. /// </summary>
  67. public static bool SerializeOSDRequestsPerEndpoint { get; set; }
  68. /// <summary>
  69. /// this is the header field used to communicate the local request id
  70. /// used for performance and debugging
  71. /// </summary>
  72. public const string OSHeaderRequestID = "opensim-request-id";
  73. /// <summary>
  74. /// Number of milliseconds a call can take before it is considered
  75. /// a "long" call for warning & debugging purposes
  76. /// </summary>
  77. public const int LongCallTime = 3000;
  78. /// <summary>
  79. /// The maximum length of any data logged because of a long request time.
  80. /// </summary>
  81. /// <remarks>
  82. /// This is to truncate any really large post data, such as an asset. In theory, the first section should
  83. /// give us useful information about the call (which agent it relates to if applicable, etc.).
  84. /// </remarks>
  85. public const int MaxRequestDiagLength = 100;
  86. /// <summary>
  87. /// Dictionary of end points
  88. /// </summary>
  89. private static Dictionary<string,object> m_endpointSerializer = new Dictionary<string,object>();
  90. private static object EndPointLock(string url)
  91. {
  92. System.Uri uri = new System.Uri(url);
  93. string endpoint = string.Format("{0}:{1}",uri.Host,uri.Port);
  94. lock (m_endpointSerializer)
  95. {
  96. object eplock = null;
  97. if (! m_endpointSerializer.TryGetValue(endpoint,out eplock))
  98. {
  99. eplock = new object();
  100. m_endpointSerializer.Add(endpoint,eplock);
  101. // m_log.WarnFormat("[WEB UTIL] add a new host to end point serializer {0}",endpoint);
  102. }
  103. return eplock;
  104. }
  105. }
  106. #region JSONRequest
  107. /// <summary>
  108. /// PUT JSON-encoded data to a web service that returns LLSD or
  109. /// JSON data
  110. /// </summary>
  111. public static OSDMap PutToServiceCompressed(string url, OSDMap data, int timeout)
  112. {
  113. return ServiceOSDRequest(url,data, "PUT", timeout, true);
  114. }
  115. public static OSDMap PutToService(string url, OSDMap data, int timeout)
  116. {
  117. return ServiceOSDRequest(url,data, "PUT", timeout, false);
  118. }
  119. public static OSDMap PostToService(string url, OSDMap data, int timeout)
  120. {
  121. return ServiceOSDRequest(url, data, "POST", timeout, false);
  122. }
  123. public static OSDMap PostToServiceCompressed(string url, OSDMap data, int timeout)
  124. {
  125. return ServiceOSDRequest(url, data, "POST", timeout, true);
  126. }
  127. public static OSDMap GetFromService(string url, int timeout)
  128. {
  129. return ServiceOSDRequest(url, null, "GET", timeout, false);
  130. }
  131. public static OSDMap ServiceOSDRequest(string url, OSDMap data, string method, int timeout, bool compressed)
  132. {
  133. if (SerializeOSDRequestsPerEndpoint)
  134. {
  135. lock (EndPointLock(url))
  136. {
  137. return ServiceOSDRequestWorker(url, data, method, timeout, compressed);
  138. }
  139. }
  140. else
  141. {
  142. return ServiceOSDRequestWorker(url, data, method, timeout, compressed);
  143. }
  144. }
  145. public static void LogOutgoingDetail(Stream outputStream)
  146. {
  147. using (StreamReader reader = new StreamReader(Util.Copy(outputStream), Encoding.UTF8))
  148. {
  149. string output;
  150. if (DebugLevel == 5)
  151. {
  152. const int sampleLength = 80;
  153. char[] sampleChars = new char[sampleLength];
  154. reader.Read(sampleChars, 0, sampleLength);
  155. output = new string(sampleChars);
  156. }
  157. else
  158. {
  159. output = reader.ReadToEnd();
  160. }
  161. LogOutgoingDetail(output);
  162. }
  163. }
  164. public static void LogOutgoingDetail(string output)
  165. {
  166. if (DebugLevel == 5)
  167. {
  168. output = output.Substring(0, 80);
  169. output = output + "...";
  170. }
  171. m_log.DebugFormat("[WEB UTIL]: {0}", output.Replace("\n", @"\n"));
  172. }
  173. private static OSDMap ServiceOSDRequestWorker(string url, OSDMap data, string method, int timeout, bool compressed)
  174. {
  175. int reqnum = RequestNumber++;
  176. if (DebugLevel >= 3)
  177. m_log.DebugFormat(
  178. "[WEB UTIL]: HTTP OUT {0} ServiceOSD {1} {2} (timeout {3}, compressed {4})",
  179. reqnum, method, url, timeout, compressed);
  180. string errorMessage = "unknown error";
  181. int tickstart = Util.EnvironmentTickCount();
  182. int tickdata = 0;
  183. string strBuffer = null;
  184. try
  185. {
  186. HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
  187. request.Method = method;
  188. request.Timeout = timeout;
  189. request.KeepAlive = false;
  190. request.MaximumAutomaticRedirections = 10;
  191. request.ReadWriteTimeout = timeout / 4;
  192. request.Headers[OSHeaderRequestID] = reqnum.ToString();
  193. // If there is some input, write it into the request
  194. if (data != null)
  195. {
  196. strBuffer = OSDParser.SerializeJsonString(data);
  197. if (DebugLevel >= 5)
  198. LogOutgoingDetail(strBuffer);
  199. byte[] buffer = System.Text.Encoding.UTF8.GetBytes(strBuffer);
  200. if (compressed)
  201. {
  202. request.ContentType = "application/x-gzip";
  203. using (MemoryStream ms = new MemoryStream())
  204. {
  205. using (GZipStream comp = new GZipStream(ms, CompressionMode.Compress))
  206. {
  207. comp.Write(buffer, 0, buffer.Length);
  208. // We need to close the gzip stream before we write it anywhere
  209. // because apparently something important related to gzip compression
  210. // gets written on the strteam upon Dispose()
  211. }
  212. byte[] buf = ms.ToArray();
  213. request.ContentLength = buf.Length; //Count bytes to send
  214. using (Stream requestStream = request.GetRequestStream())
  215. requestStream.Write(buf, 0, (int)buf.Length);
  216. }
  217. }
  218. else
  219. {
  220. request.ContentType = "application/json";
  221. request.ContentLength = buffer.Length; //Count bytes to send
  222. using (Stream requestStream = request.GetRequestStream())
  223. requestStream.Write(buffer, 0, buffer.Length); //Send it
  224. }
  225. }
  226. // capture how much time was spent writing, this may seem silly
  227. // but with the number concurrent requests, this often blocks
  228. tickdata = Util.EnvironmentTickCountSubtract(tickstart);
  229. using (WebResponse response = request.GetResponse())
  230. {
  231. using (Stream responseStream = response.GetResponseStream())
  232. {
  233. string responseStr = null;
  234. responseStr = responseStream.GetStreamString();
  235. // m_log.DebugFormat("[WEB UTIL]: <{0}> response is <{1}>",reqnum,responseStr);
  236. return CanonicalizeResults(responseStr);
  237. }
  238. }
  239. }
  240. catch (WebException we)
  241. {
  242. errorMessage = we.Message;
  243. if (we.Status == WebExceptionStatus.ProtocolError)
  244. {
  245. using (HttpWebResponse webResponse = (HttpWebResponse)we.Response)
  246. errorMessage = String.Format("[{0}] {1}", webResponse.StatusCode, webResponse.StatusDescription);
  247. }
  248. }
  249. catch (Exception ex)
  250. {
  251. errorMessage = ex.Message;
  252. }
  253. finally
  254. {
  255. int tickdiff = Util.EnvironmentTickCountSubtract(tickstart);
  256. if (tickdiff > LongCallTime)
  257. m_log.InfoFormat(
  258. "[WEB UTIL]: Slow ServiceOSD request {0} {1} {2} took {3}ms, {4}ms writing, {5}",
  259. reqnum,
  260. method,
  261. url,
  262. tickdiff,
  263. tickdata,
  264. strBuffer != null
  265. ? (strBuffer.Length > MaxRequestDiagLength ? strBuffer.Remove(MaxRequestDiagLength) : strBuffer)
  266. : "");
  267. else if (DebugLevel >= 4)
  268. m_log.DebugFormat(
  269. "[WEB UTIL]: HTTP OUT {0} took {1}ms, {2}ms writing",
  270. reqnum, tickdiff, tickdata);
  271. }
  272. m_log.DebugFormat(
  273. "[WEB UTIL]: ServiceOSD request {0} {1} {2} FAILED: {3}", reqnum, url, method, errorMessage);
  274. return ErrorResponseMap(errorMessage);
  275. }
  276. /// <summary>
  277. /// Since there are no consistencies in the way web requests are
  278. /// formed, we need to do a little guessing about the result format.
  279. /// Keys:
  280. /// Success|success == the success fail of the request
  281. /// _RawResult == the raw string that came back
  282. /// _Result == the OSD unpacked string
  283. /// </summary>
  284. private static OSDMap CanonicalizeResults(string response)
  285. {
  286. OSDMap result = new OSDMap();
  287. // Default values
  288. result["Success"] = OSD.FromBoolean(true);
  289. result["success"] = OSD.FromBoolean(true);
  290. result["_RawResult"] = OSD.FromString(response);
  291. result["_Result"] = new OSDMap();
  292. if (response.Equals("true",System.StringComparison.OrdinalIgnoreCase))
  293. return result;
  294. if (response.Equals("false",System.StringComparison.OrdinalIgnoreCase))
  295. {
  296. result["Success"] = OSD.FromBoolean(false);
  297. result["success"] = OSD.FromBoolean(false);
  298. return result;
  299. }
  300. try
  301. {
  302. OSD responseOSD = OSDParser.Deserialize(response);
  303. if (responseOSD.Type == OSDType.Map)
  304. {
  305. result["_Result"] = (OSDMap)responseOSD;
  306. return result;
  307. }
  308. }
  309. catch
  310. {
  311. // don't need to treat this as an error... we're just guessing anyway
  312. // m_log.DebugFormat("[WEB UTIL] couldn't decode <{0}>: {1}",response,e.Message);
  313. }
  314. return result;
  315. }
  316. #endregion JSONRequest
  317. #region FormRequest
  318. /// <summary>
  319. /// POST URL-encoded form data to a web service that returns LLSD or
  320. /// JSON data
  321. /// </summary>
  322. public static OSDMap PostToService(string url, NameValueCollection data)
  323. {
  324. return ServiceFormRequest(url,data,10000);
  325. }
  326. public static OSDMap ServiceFormRequest(string url, NameValueCollection data, int timeout)
  327. {
  328. lock (EndPointLock(url))
  329. {
  330. return ServiceFormRequestWorker(url,data,timeout);
  331. }
  332. }
  333. private static OSDMap ServiceFormRequestWorker(string url, NameValueCollection data, int timeout)
  334. {
  335. int reqnum = RequestNumber++;
  336. string method = (data != null && data["RequestMethod"] != null) ? data["RequestMethod"] : "unknown";
  337. if (DebugLevel >= 3)
  338. m_log.DebugFormat(
  339. "[WEB UTIL]: HTTP OUT {0} ServiceForm {1} {2} (timeout {3})",
  340. reqnum, method, url, timeout);
  341. string errorMessage = "unknown error";
  342. int tickstart = Util.EnvironmentTickCount();
  343. int tickdata = 0;
  344. string queryString = null;
  345. try
  346. {
  347. HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(url);
  348. request.Method = "POST";
  349. request.Timeout = timeout;
  350. request.KeepAlive = false;
  351. request.MaximumAutomaticRedirections = 10;
  352. request.ReadWriteTimeout = timeout / 4;
  353. request.Headers[OSHeaderRequestID] = reqnum.ToString();
  354. if (data != null)
  355. {
  356. queryString = BuildQueryString(data);
  357. if (DebugLevel >= 5)
  358. LogOutgoingDetail(queryString);
  359. byte[] buffer = System.Text.Encoding.UTF8.GetBytes(queryString);
  360. request.ContentLength = buffer.Length;
  361. request.ContentType = "application/x-www-form-urlencoded";
  362. using (Stream requestStream = request.GetRequestStream())
  363. requestStream.Write(buffer, 0, buffer.Length);
  364. }
  365. // capture how much time was spent writing, this may seem silly
  366. // but with the number concurrent requests, this often blocks
  367. tickdata = Util.EnvironmentTickCountSubtract(tickstart);
  368. using (WebResponse response = request.GetResponse())
  369. {
  370. using (Stream responseStream = response.GetResponseStream())
  371. {
  372. string responseStr = null;
  373. responseStr = responseStream.GetStreamString();
  374. OSD responseOSD = OSDParser.Deserialize(responseStr);
  375. if (responseOSD.Type == OSDType.Map)
  376. return (OSDMap)responseOSD;
  377. }
  378. }
  379. }
  380. catch (WebException we)
  381. {
  382. errorMessage = we.Message;
  383. if (we.Status == WebExceptionStatus.ProtocolError)
  384. {
  385. using (HttpWebResponse webResponse = (HttpWebResponse)we.Response)
  386. errorMessage = String.Format("[{0}] {1}",webResponse.StatusCode,webResponse.StatusDescription);
  387. }
  388. }
  389. catch (Exception ex)
  390. {
  391. errorMessage = ex.Message;
  392. }
  393. finally
  394. {
  395. int tickdiff = Util.EnvironmentTickCountSubtract(tickstart);
  396. if (tickdiff > LongCallTime)
  397. m_log.InfoFormat(
  398. "[WEB UTIL]: Slow ServiceForm request {0} {1} {2} took {3}ms, {4}ms writing, {5}",
  399. reqnum,
  400. method,
  401. url,
  402. tickdiff,
  403. tickdata,
  404. queryString != null
  405. ? (queryString.Length > MaxRequestDiagLength) ? queryString.Remove(MaxRequestDiagLength) : queryString
  406. : "");
  407. else if (DebugLevel >= 4)
  408. m_log.DebugFormat(
  409. "[WEB UTIL]: HTTP OUT {0} took {1}ms, {2}ms writing",
  410. reqnum, tickdiff, tickdata);
  411. }
  412. m_log.WarnFormat("[WEB UTIL]: ServiceForm request {0} {1} {2} failed: {2}", reqnum, method, url, errorMessage);
  413. return ErrorResponseMap(errorMessage);
  414. }
  415. /// <summary>
  416. /// Create a response map for an error, trying to keep
  417. /// the result formats consistent
  418. /// </summary>
  419. private static OSDMap ErrorResponseMap(string msg)
  420. {
  421. OSDMap result = new OSDMap();
  422. result["Success"] = "False";
  423. result["Message"] = OSD.FromString("Service request failed: " + msg);
  424. return result;
  425. }
  426. #endregion FormRequest
  427. #region Uri
  428. /// <summary>
  429. /// Combines a Uri that can contain both a base Uri and relative path
  430. /// with a second relative path fragment
  431. /// </summary>
  432. /// <param name="uri">Starting (base) Uri</param>
  433. /// <param name="fragment">Relative path fragment to append to the end
  434. /// of the Uri</param>
  435. /// <returns>The combined Uri</returns>
  436. /// <remarks>This is similar to the Uri constructor that takes a base
  437. /// Uri and the relative path, except this method can append a relative
  438. /// path fragment on to an existing relative path</remarks>
  439. public static Uri Combine(this Uri uri, string fragment)
  440. {
  441. string fragment1 = uri.Fragment;
  442. string fragment2 = fragment;
  443. if (!fragment1.EndsWith("/"))
  444. fragment1 = fragment1 + '/';
  445. if (fragment2.StartsWith("/"))
  446. fragment2 = fragment2.Substring(1);
  447. return new Uri(uri, fragment1 + fragment2);
  448. }
  449. /// <summary>
  450. /// Combines a Uri that can contain both a base Uri and relative path
  451. /// with a second relative path fragment. If the fragment is absolute,
  452. /// it will be returned without modification
  453. /// </summary>
  454. /// <param name="uri">Starting (base) Uri</param>
  455. /// <param name="fragment">Relative path fragment to append to the end
  456. /// of the Uri, or an absolute Uri to return unmodified</param>
  457. /// <returns>The combined Uri</returns>
  458. public static Uri Combine(this Uri uri, Uri fragment)
  459. {
  460. if (fragment.IsAbsoluteUri)
  461. return fragment;
  462. string fragment1 = uri.Fragment;
  463. string fragment2 = fragment.ToString();
  464. if (!fragment1.EndsWith("/"))
  465. fragment1 = fragment1 + '/';
  466. if (fragment2.StartsWith("/"))
  467. fragment2 = fragment2.Substring(1);
  468. return new Uri(uri, fragment1 + fragment2);
  469. }
  470. /// <summary>
  471. /// Appends a query string to a Uri that may or may not have existing
  472. /// query parameters
  473. /// </summary>
  474. /// <param name="uri">Uri to append the query to</param>
  475. /// <param name="query">Query string to append. Can either start with ?
  476. /// or just containg key/value pairs</param>
  477. /// <returns>String representation of the Uri with the query string
  478. /// appended</returns>
  479. public static string AppendQuery(this Uri uri, string query)
  480. {
  481. if (String.IsNullOrEmpty(query))
  482. return uri.ToString();
  483. if (query[0] == '?' || query[0] == '&')
  484. query = query.Substring(1);
  485. string uriStr = uri.ToString();
  486. if (uriStr.Contains("?"))
  487. return uriStr + '&' + query;
  488. else
  489. return uriStr + '?' + query;
  490. }
  491. #endregion Uri
  492. #region NameValueCollection
  493. /// <summary>
  494. /// Convert a NameValueCollection into a query string. This is the
  495. /// inverse of HttpUtility.ParseQueryString()
  496. /// </summary>
  497. /// <param name="parameters">Collection of key/value pairs to convert</param>
  498. /// <returns>A query string with URL-escaped values</returns>
  499. public static string BuildQueryString(NameValueCollection parameters)
  500. {
  501. List<string> items = new List<string>(parameters.Count);
  502. foreach (string key in parameters.Keys)
  503. {
  504. string[] values = parameters.GetValues(key);
  505. if (values != null)
  506. {
  507. foreach (string value in values)
  508. items.Add(String.Concat(key, "=", HttpUtility.UrlEncode(value ?? String.Empty)));
  509. }
  510. }
  511. return String.Join("&", items.ToArray());
  512. }
  513. /// <summary>
  514. ///
  515. /// </summary>
  516. /// <param name="collection"></param>
  517. /// <param name="key"></param>
  518. /// <returns></returns>
  519. public static string GetOne(this NameValueCollection collection, string key)
  520. {
  521. string[] values = collection.GetValues(key);
  522. if (values != null && values.Length > 0)
  523. return values[0];
  524. return null;
  525. }
  526. #endregion NameValueCollection
  527. #region Stream
  528. /// <summary>
  529. /// Copies the contents of one stream to another, starting at the
  530. /// current position of each stream
  531. /// </summary>
  532. /// <param name="copyFrom">The stream to copy from, at the position
  533. /// where copying should begin</param>
  534. /// <param name="copyTo">The stream to copy to, at the position where
  535. /// bytes should be written</param>
  536. /// <param name="maximumBytesToCopy">The maximum bytes to copy</param>
  537. /// <returns>The total number of bytes copied</returns>
  538. /// <remarks>
  539. /// Copying begins at the streams' current positions. The positions are
  540. /// NOT reset after copying is complete.
  541. /// NOTE!! .NET 4.0 adds the method 'Stream.CopyTo(stream, bufferSize)'.
  542. /// This function could be replaced with that method once we move
  543. /// totally to .NET 4.0. For versions before, this routine exists.
  544. /// This routine used to be named 'CopyTo' but the int parameter has
  545. /// a different meaning so this method was renamed to avoid any confusion.
  546. /// </remarks>
  547. public static int CopyStream(this Stream copyFrom, Stream copyTo, int maximumBytesToCopy)
  548. {
  549. byte[] buffer = new byte[4096];
  550. int readBytes;
  551. int totalCopiedBytes = 0;
  552. while ((readBytes = copyFrom.Read(buffer, 0, Math.Min(4096, maximumBytesToCopy))) > 0)
  553. {
  554. int writeBytes = Math.Min(maximumBytesToCopy, readBytes);
  555. copyTo.Write(buffer, 0, writeBytes);
  556. totalCopiedBytes += writeBytes;
  557. maximumBytesToCopy -= writeBytes;
  558. }
  559. return totalCopiedBytes;
  560. }
  561. /// <summary>
  562. /// Converts an entire stream to a string, regardless of current stream
  563. /// position
  564. /// </summary>
  565. /// <param name="stream">The stream to convert to a string</param>
  566. /// <returns></returns>
  567. /// <remarks>When this method is done, the stream position will be
  568. /// reset to its previous position before this method was called</remarks>
  569. public static string GetStreamString(this Stream stream)
  570. {
  571. string value = null;
  572. if (stream != null && stream.CanRead)
  573. {
  574. long rewindPos = -1;
  575. if (stream.CanSeek)
  576. {
  577. rewindPos = stream.Position;
  578. stream.Seek(0, SeekOrigin.Begin);
  579. }
  580. StreamReader reader = new StreamReader(stream);
  581. value = reader.ReadToEnd();
  582. if (rewindPos >= 0)
  583. stream.Seek(rewindPos, SeekOrigin.Begin);
  584. }
  585. return value;
  586. }
  587. #endregion Stream
  588. public class QBasedComparer : IComparer
  589. {
  590. public int Compare(Object x, Object y)
  591. {
  592. float qx = GetQ(x);
  593. float qy = GetQ(y);
  594. return qy.CompareTo(qx); // descending order
  595. }
  596. private float GetQ(Object o)
  597. {
  598. // Example: image/png;q=0.9
  599. float qvalue = 1F;
  600. if (o is String)
  601. {
  602. string mime = (string)o;
  603. string[] parts = mime.Split(';');
  604. if (parts.Length > 1)
  605. {
  606. string[] kvp = parts[1].Split('=');
  607. if (kvp.Length == 2 && kvp[0] == "q")
  608. float.TryParse(kvp[1], NumberStyles.Number, CultureInfo.InvariantCulture, out qvalue);
  609. }
  610. }
  611. return qvalue;
  612. }
  613. }
  614. /// <summary>
  615. /// Takes the value of an Accept header and returns the preferred types
  616. /// ordered by q value (if it exists).
  617. /// Example input: image/jpg;q=0.7, image/png;q=0.8, image/jp2
  618. /// Exmaple output: ["jp2", "png", "jpg"]
  619. /// NOTE: This doesn't handle the semantics of *'s...
  620. /// </summary>
  621. /// <param name="accept"></param>
  622. /// <returns></returns>
  623. public static string[] GetPreferredImageTypes(string accept)
  624. {
  625. if (string.IsNullOrEmpty(accept))
  626. return new string[0];
  627. string[] types = accept.Split(new char[] { ',' });
  628. if (types.Length > 0)
  629. {
  630. List<string> list = new List<string>(types);
  631. list.RemoveAll(delegate(string s) { return !s.ToLower().StartsWith("image"); });
  632. ArrayList tlist = new ArrayList(list);
  633. tlist.Sort(new QBasedComparer());
  634. string[] result = new string[tlist.Count];
  635. for (int i = 0; i < tlist.Count; i++)
  636. {
  637. string mime = (string)tlist[i];
  638. string[] parts = mime.Split(new char[] { ';' });
  639. string[] pair = parts[0].Split(new char[] { '/' });
  640. if (pair.Length == 2)
  641. result[i] = pair[1].ToLower();
  642. else // oops, we don't know what this is...
  643. result[i] = pair[0];
  644. }
  645. return result;
  646. }
  647. return new string[0];
  648. }
  649. }
  650. public static class AsynchronousRestObjectRequester
  651. {
  652. private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  653. /// <summary>
  654. /// Perform an asynchronous REST request.
  655. /// </summary>
  656. /// <param name="verb">GET or POST</param>
  657. /// <param name="requestUrl"></param>
  658. /// <param name="obj"></param>
  659. /// <param name="action"></param>
  660. /// <returns></returns>
  661. ///
  662. /// <exception cref="System.Net.WebException">Thrown if we encounter a
  663. /// network issue while posting the request. You'll want to make
  664. /// sure you deal with this as they're not uncommon</exception>
  665. //
  666. public static void MakeRequest<TRequest, TResponse>(string verb,
  667. string requestUrl, TRequest obj, Action<TResponse> action)
  668. {
  669. MakeRequest<TRequest, TResponse>(verb, requestUrl, obj, action, 0);
  670. }
  671. public static void MakeRequest<TRequest, TResponse>(string verb,
  672. string requestUrl, TRequest obj, Action<TResponse> action,
  673. int maxConnections)
  674. {
  675. int reqnum = WebUtil.RequestNumber++;
  676. if (WebUtil.DebugLevel >= 3)
  677. m_log.DebugFormat(
  678. "[WEB UTIL]: HTTP OUT {0} AsynchronousRequestObject {1} {2}",
  679. reqnum, verb, requestUrl);
  680. int tickstart = Util.EnvironmentTickCount();
  681. int tickdata = 0;
  682. Type type = typeof(TRequest);
  683. WebRequest request = WebRequest.Create(requestUrl);
  684. HttpWebRequest ht = (HttpWebRequest)request;
  685. if (maxConnections > 0 && ht.ServicePoint.ConnectionLimit < maxConnections)
  686. ht.ServicePoint.ConnectionLimit = maxConnections;
  687. WebResponse response = null;
  688. TResponse deserial = default(TResponse);
  689. XmlSerializer deserializer = new XmlSerializer(typeof(TResponse));
  690. request.Method = verb;
  691. MemoryStream buffer = null;
  692. if (verb == "POST")
  693. {
  694. request.ContentType = "text/xml";
  695. buffer = new MemoryStream();
  696. XmlWriterSettings settings = new XmlWriterSettings();
  697. settings.Encoding = Encoding.UTF8;
  698. using (XmlWriter writer = XmlWriter.Create(buffer, settings))
  699. {
  700. XmlSerializer serializer = new XmlSerializer(type);
  701. serializer.Serialize(writer, obj);
  702. writer.Flush();
  703. }
  704. int length = (int)buffer.Length;
  705. request.ContentLength = length;
  706. if (WebUtil.DebugLevel >= 5)
  707. WebUtil.LogOutgoingDetail(buffer);
  708. request.BeginGetRequestStream(delegate(IAsyncResult res)
  709. {
  710. Stream requestStream = request.EndGetRequestStream(res);
  711. requestStream.Write(buffer.ToArray(), 0, length);
  712. requestStream.Close();
  713. // capture how much time was spent writing
  714. tickdata = Util.EnvironmentTickCountSubtract(tickstart);
  715. request.BeginGetResponse(delegate(IAsyncResult ar)
  716. {
  717. response = request.EndGetResponse(ar);
  718. Stream respStream = null;
  719. try
  720. {
  721. respStream = response.GetResponseStream();
  722. deserial = (TResponse)deserializer.Deserialize(
  723. respStream);
  724. }
  725. catch (System.InvalidOperationException)
  726. {
  727. }
  728. finally
  729. {
  730. // Let's not close this
  731. //buffer.Close();
  732. respStream.Close();
  733. response.Close();
  734. }
  735. action(deserial);
  736. }, null);
  737. }, null);
  738. }
  739. else
  740. {
  741. request.BeginGetResponse(delegate(IAsyncResult res2)
  742. {
  743. try
  744. {
  745. // If the server returns a 404, this appears to trigger a System.Net.WebException even though that isn't
  746. // documented in MSDN
  747. response = request.EndGetResponse(res2);
  748. Stream respStream = null;
  749. try
  750. {
  751. respStream = response.GetResponseStream();
  752. deserial = (TResponse)deserializer.Deserialize(respStream);
  753. }
  754. catch (System.InvalidOperationException)
  755. {
  756. }
  757. finally
  758. {
  759. respStream.Close();
  760. response.Close();
  761. }
  762. }
  763. catch (WebException e)
  764. {
  765. if (e.Status == WebExceptionStatus.ProtocolError)
  766. {
  767. if (e.Response is HttpWebResponse)
  768. {
  769. using (HttpWebResponse httpResponse = (HttpWebResponse)e.Response)
  770. {
  771. if (httpResponse.StatusCode != HttpStatusCode.NotFound)
  772. {
  773. // We don't appear to be handling any other status codes, so log these feailures to that
  774. // people don't spend unnecessary hours hunting phantom bugs.
  775. m_log.DebugFormat(
  776. "[ASYNC REQUEST]: Request {0} {1} failed with unexpected status code {2}",
  777. verb, requestUrl, httpResponse.StatusCode);
  778. }
  779. }
  780. }
  781. }
  782. else
  783. {
  784. m_log.ErrorFormat(
  785. "[ASYNC REQUEST]: Request {0} {1} failed with status {2} and message {3}",
  786. verb, requestUrl, e.Status, e.Message);
  787. }
  788. }
  789. catch (Exception e)
  790. {
  791. m_log.ErrorFormat(
  792. "[ASYNC REQUEST]: Request {0} {1} failed with exception {2}{3}",
  793. verb, requestUrl, e.Message, e.StackTrace);
  794. }
  795. // m_log.DebugFormat("[ASYNC REQUEST]: Received {0}", deserial.ToString());
  796. try
  797. {
  798. action(deserial);
  799. }
  800. catch (Exception e)
  801. {
  802. m_log.ErrorFormat(
  803. "[ASYNC REQUEST]: Request {0} {1} callback failed with exception {2}{3}",
  804. verb, requestUrl, e.Message, e.StackTrace);
  805. }
  806. }, null);
  807. }
  808. int tickdiff = Util.EnvironmentTickCountSubtract(tickstart);
  809. if (tickdiff > WebUtil.LongCallTime)
  810. {
  811. string originalRequest = null;
  812. if (buffer != null)
  813. {
  814. originalRequest = Encoding.UTF8.GetString(buffer.ToArray());
  815. if (originalRequest.Length > WebUtil.MaxRequestDiagLength)
  816. originalRequest = originalRequest.Remove(WebUtil.MaxRequestDiagLength);
  817. }
  818. m_log.InfoFormat(
  819. "[ASYNC REQUEST]: Slow request {0} {1} {2} took {3}ms, {4}ms writing, {5}",
  820. reqnum,
  821. verb,
  822. requestUrl,
  823. tickdiff,
  824. tickdata,
  825. originalRequest);
  826. }
  827. else if (WebUtil.DebugLevel >= 4)
  828. {
  829. m_log.DebugFormat(
  830. "[WEB UTIL]: HTTP OUT {0} took {1}ms, {2}ms writing",
  831. reqnum, tickdiff, tickdata);
  832. }
  833. }
  834. }
  835. public static class SynchronousRestFormsRequester
  836. {
  837. private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  838. /// <summary>
  839. /// Perform a synchronous REST request.
  840. /// </summary>
  841. /// <param name="verb"></param>
  842. /// <param name="requestUrl"></param>
  843. /// <param name="obj"> </param>
  844. /// <param name="timeoutsecs"> </param>
  845. /// <returns></returns>
  846. ///
  847. /// <exception cref="System.Net.WebException">Thrown if we encounter a network issue while posting
  848. /// the request. You'll want to make sure you deal with this as they're not uncommon</exception>
  849. public static string MakeRequest(string verb, string requestUrl, string obj, int timeoutsecs)
  850. {
  851. int reqnum = WebUtil.RequestNumber++;
  852. if (WebUtil.DebugLevel >= 3)
  853. m_log.DebugFormat(
  854. "[WEB UTIL]: HTTP OUT {0} SynchronousRestForms {1} {2}",
  855. reqnum, verb, requestUrl);
  856. int tickstart = Util.EnvironmentTickCount();
  857. int tickdata = 0;
  858. WebRequest request = WebRequest.Create(requestUrl);
  859. request.Method = verb;
  860. if (timeoutsecs > 0)
  861. request.Timeout = timeoutsecs * 1000;
  862. string respstring = String.Empty;
  863. using (MemoryStream buffer = new MemoryStream())
  864. {
  865. if ((verb == "POST") || (verb == "PUT"))
  866. {
  867. request.ContentType = "application/x-www-form-urlencoded";
  868. int length = 0;
  869. using (StreamWriter writer = new StreamWriter(buffer))
  870. {
  871. writer.Write(obj);
  872. writer.Flush();
  873. }
  874. length = (int)obj.Length;
  875. request.ContentLength = length;
  876. if (WebUtil.DebugLevel >= 5)
  877. WebUtil.LogOutgoingDetail(buffer);
  878. Stream requestStream = null;
  879. try
  880. {
  881. requestStream = request.GetRequestStream();
  882. requestStream.Write(buffer.ToArray(), 0, length);
  883. }
  884. catch (Exception e)
  885. {
  886. m_log.DebugFormat(
  887. "[FORMS]: exception occured {0} {1}: {2}{3}", verb, requestUrl, e.Message, e.StackTrace);
  888. }
  889. finally
  890. {
  891. if (requestStream != null)
  892. requestStream.Close();
  893. // capture how much time was spent writing
  894. tickdata = Util.EnvironmentTickCountSubtract(tickstart);
  895. }
  896. }
  897. try
  898. {
  899. using (WebResponse resp = request.GetResponse())
  900. {
  901. if (resp.ContentLength != 0)
  902. {
  903. Stream respStream = null;
  904. try
  905. {
  906. using (respStream = resp.GetResponseStream())
  907. using (StreamReader reader = new StreamReader(respStream))
  908. respstring = reader.ReadToEnd();
  909. }
  910. catch (Exception e)
  911. {
  912. m_log.DebugFormat(
  913. "[FORMS]: Exception occured on receiving {0} {1}: {2}{3}",
  914. verb, requestUrl, e.Message, e.StackTrace);
  915. }
  916. finally
  917. {
  918. if (respStream != null)
  919. respStream.Close();
  920. }
  921. }
  922. }
  923. }
  924. catch (System.InvalidOperationException)
  925. {
  926. // This is what happens when there is invalid XML
  927. m_log.DebugFormat("[FORMS]: InvalidOperationException on receiving {0} {1}", verb, requestUrl);
  928. }
  929. }
  930. int tickdiff = Util.EnvironmentTickCountSubtract(tickstart);
  931. if (tickdiff > WebUtil.LongCallTime)
  932. m_log.InfoFormat(
  933. "[FORMS]: Slow request {0} {1} {2} took {3}ms, {4}ms writing, {5}",
  934. reqnum,
  935. verb,
  936. requestUrl,
  937. tickdiff,
  938. tickdata,
  939. obj.Length > WebUtil.MaxRequestDiagLength ? obj.Remove(WebUtil.MaxRequestDiagLength) : obj);
  940. else if (WebUtil.DebugLevel >= 4)
  941. m_log.DebugFormat(
  942. "[WEB UTIL]: HTTP OUT {0} took {1}ms, {2}ms writing",
  943. reqnum, tickdiff, tickdata);
  944. return respstring;
  945. }
  946. public static string MakeRequest(string verb, string requestUrl, string obj)
  947. {
  948. return MakeRequest(verb, requestUrl, obj, -1);
  949. }
  950. }
  951. public class SynchronousRestObjectRequester
  952. {
  953. private static readonly ILog m_log =
  954. LogManager.GetLogger(
  955. MethodBase.GetCurrentMethod().DeclaringType);
  956. /// <summary>
  957. /// Perform a synchronous REST request.
  958. /// </summary>
  959. /// <param name="verb"></param>
  960. /// <param name="requestUrl"></param>
  961. /// <param name="obj"> </param>
  962. /// <returns></returns>
  963. ///
  964. /// <exception cref="System.Net.WebException">Thrown if we encounter a network issue while posting
  965. /// the request. You'll want to make sure you deal with this as they're not uncommon</exception>
  966. public static TResponse MakeRequest<TRequest, TResponse>(string verb, string requestUrl, TRequest obj)
  967. {
  968. return MakeRequest<TRequest, TResponse>(verb, requestUrl, obj, 0);
  969. }
  970. public static TResponse MakeRequest<TRequest, TResponse>(string verb, string requestUrl, TRequest obj, int pTimeout)
  971. {
  972. return MakeRequest<TRequest, TResponse>(verb, requestUrl, obj, pTimeout, 0);
  973. }
  974. public static TResponse MakeRequest<TRequest, TResponse>(string verb, string requestUrl, TRequest obj, int pTimeout, int maxConnections)
  975. {
  976. int reqnum = WebUtil.RequestNumber++;
  977. if (WebUtil.DebugLevel >= 3)
  978. m_log.DebugFormat(
  979. "[WEB UTIL]: HTTP OUT {0} SynchronousRestObject {1} {2}",
  980. reqnum, verb, requestUrl);
  981. int tickstart = Util.EnvironmentTickCount();
  982. int tickdata = 0;
  983. Type type = typeof(TRequest);
  984. TResponse deserial = default(TResponse);
  985. WebRequest request = WebRequest.Create(requestUrl);
  986. HttpWebRequest ht = (HttpWebRequest)request;
  987. if (maxConnections > 0 && ht.ServicePoint.ConnectionLimit < maxConnections)
  988. ht.ServicePoint.ConnectionLimit = maxConnections;
  989. request.Method = verb;
  990. MemoryStream buffer = null;
  991. if ((verb == "POST") || (verb == "PUT"))
  992. {
  993. request.ContentType = "text/xml";
  994. buffer = new MemoryStream();
  995. XmlWriterSettings settings = new XmlWriterSettings();
  996. settings.Encoding = Encoding.UTF8;
  997. using (XmlWriter writer = XmlWriter.Create(buffer, settings))
  998. {
  999. XmlSerializer serializer = new XmlSerializer(type);
  1000. serializer.Serialize(writer, obj);
  1001. writer.Flush();
  1002. }
  1003. int length = (int)buffer.Length;
  1004. request.ContentLength = length;
  1005. if (WebUtil.DebugLevel >= 5)
  1006. WebUtil.LogOutgoingDetail(buffer);
  1007. Stream requestStream = null;
  1008. try
  1009. {
  1010. requestStream = request.GetRequestStream();
  1011. requestStream.Write(buffer.ToArray(), 0, length);
  1012. }
  1013. catch (Exception e)
  1014. {
  1015. m_log.DebugFormat(
  1016. "[SynchronousRestObjectRequester]: Exception in making request {0} {1}: {2}{3}",
  1017. verb, requestUrl, e.Message, e.StackTrace);
  1018. return deserial;
  1019. }
  1020. finally
  1021. {
  1022. if (requestStream != null)
  1023. requestStream.Close();
  1024. // capture how much time was spent writing
  1025. tickdata = Util.EnvironmentTickCountSubtract(tickstart);
  1026. }
  1027. }
  1028. try
  1029. {
  1030. using (HttpWebResponse resp = (HttpWebResponse)request.GetResponse())
  1031. {
  1032. if (resp.ContentLength != 0)
  1033. {
  1034. using (Stream respStream = resp.GetResponseStream())
  1035. {
  1036. XmlSerializer deserializer = new XmlSerializer(typeof(TResponse));
  1037. deserial = (TResponse)deserializer.Deserialize(respStream);
  1038. }
  1039. }
  1040. else
  1041. {
  1042. m_log.DebugFormat(
  1043. "[SynchronousRestObjectRequester]: Oops! no content found in response stream from {0} {1}",
  1044. verb, requestUrl);
  1045. }
  1046. }
  1047. }
  1048. catch (WebException e)
  1049. {
  1050. using (HttpWebResponse hwr = (HttpWebResponse)e.Response)
  1051. {
  1052. if (hwr != null && hwr.StatusCode == HttpStatusCode.NotFound)
  1053. return deserial;
  1054. else
  1055. m_log.ErrorFormat(
  1056. "[SynchronousRestObjectRequester]: WebException for {0} {1} {2}: {3} {4}",
  1057. verb, requestUrl, typeof(TResponse).ToString(), e.Message, e.StackTrace);
  1058. }
  1059. }
  1060. catch (System.InvalidOperationException)
  1061. {
  1062. // This is what happens when there is invalid XML
  1063. m_log.DebugFormat(
  1064. "[SynchronousRestObjectRequester]: Invalid XML from {0} {1} {2}",
  1065. verb, requestUrl, typeof(TResponse).ToString());
  1066. }
  1067. catch (Exception e)
  1068. {
  1069. m_log.DebugFormat(
  1070. "[SynchronousRestObjectRequester]: Exception on response from {0} {1}: {2}{3}",
  1071. verb, requestUrl, e.Message, e.StackTrace);
  1072. }
  1073. int tickdiff = Util.EnvironmentTickCountSubtract(tickstart);
  1074. if (tickdiff > WebUtil.LongCallTime)
  1075. {
  1076. string originalRequest = null;
  1077. if (buffer != null)
  1078. {
  1079. originalRequest = Encoding.UTF8.GetString(buffer.ToArray());
  1080. if (originalRequest.Length > WebUtil.MaxRequestDiagLength)
  1081. originalRequest = originalRequest.Remove(WebUtil.MaxRequestDiagLength);
  1082. }
  1083. m_log.InfoFormat(
  1084. "[SynchronousRestObjectRequester]: Slow request {0} {1} {2} took {3}ms, {4}ms writing, {5}",
  1085. reqnum,
  1086. verb,
  1087. requestUrl,
  1088. tickdiff,
  1089. tickdata,
  1090. originalRequest);
  1091. }
  1092. else if (WebUtil.DebugLevel >= 4)
  1093. {
  1094. m_log.DebugFormat(
  1095. "[WEB UTIL]: HTTP OUT {0} took {1}ms, {2}ms writing",
  1096. reqnum, tickdiff, tickdata);
  1097. }
  1098. return deserial;
  1099. }
  1100. }
  1101. }