ScriptsHttpRequests.cs 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767
  1. /*
  2. * Copyright (c) Contributors, http://opensimulator.org/
  3. * See CONTRIBUTORS.TXT for a full list of copyright holders.
  4. *
  5. * Redistribution and use in source and binary forms, with or without
  6. * modification, are permitted provided that the following conditions are met:
  7. * * Redistributions of source code must retain the above copyright
  8. * notice, this list of conditions and the following disclaimer.
  9. * * Redistributions in binary form must reproduce the above copyright
  10. * notice, this list of conditions and the following disclaimer in the
  11. * documentation and/or other materials provided with the distribution.
  12. * * Neither the name of the OpenSimulator Project nor the
  13. * names of its contributors may be used to endorse or promote products
  14. * derived from this software without specific prior written permission.
  15. *
  16. * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
  17. * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  18. * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19. * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
  20. * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  21. * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  22. * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  23. * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  24. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  25. * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26. */
  27. using System;
  28. using System.Collections.Generic;
  29. using System.Collections.Concurrent;
  30. using System.IO;
  31. using System.Net;
  32. using System.Net.Security;
  33. using System.Security.Cryptography.X509Certificates;
  34. using System.Threading;
  35. using Nini.Config;
  36. using OpenMetaverse;
  37. using OpenSim.Framework;
  38. using OpenSim.Framework.Monitoring;
  39. using OpenSim.Region.Framework.Interfaces;
  40. using OpenSim.Region.Framework.Scenes;
  41. using Mono.Addins;
  42. using System.Net.Http;
  43. using System.Security.Authentication;
  44. using System.Net.Http.Headers;
  45. /*****************************************************
  46. *
  47. * ScriptsHttpRequests
  48. *
  49. * Implements the llHttpRequest and http_response
  50. * callback.
  51. *
  52. * This is a non shared module with shared static parts
  53. * **************************************************/
  54. namespace OpenSim.Region.CoreModules.Scripting.HttpRequest
  55. {
  56. [Extension(Path = "/OpenSim/RegionModules", NodeName = "RegionModule", Id = "HttpRequestModule")]
  57. public class HttpRequestModule : INonSharedRegionModule, IHttpRequestModule
  58. {
  59. private struct ThrottleData
  60. {
  61. public double lastTime;
  62. public float control;
  63. }
  64. // private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
  65. private static HttpClient VeriFyCertClient = null;
  66. private static HttpClient VeriFyNoCertClient = null;
  67. private static readonly object m_mainLock = new();
  68. private static int m_numberScenes;
  69. private static readonly string m_name = "HttpScriptRequests";
  70. private static OutboundUrlFilter m_outboundUrlFilter;
  71. private static int m_HttpBodyMaxLenMAX = 16384;
  72. private static float m_primPerSec = 1.0f;
  73. private static float m_primBurst = 3.0f;
  74. private static float m_primOwnerPerSec = 25.0f;
  75. private static float m_primOwnerBurst = 5.0f;
  76. public static JobEngine m_jobEngine = null;
  77. private static Dictionary<UUID, HttpRequestClass> m_pendingRequests;
  78. //this are per region/module
  79. private readonly ConcurrentQueue<HttpRequestClass> m_CompletedRequests = new();
  80. private readonly ConcurrentDictionary<uint, ThrottleData> m_RequestsThrottle = new();
  81. private readonly ConcurrentDictionary<UUID, ThrottleData> m_OwnerRequestsThrottle = new();
  82. public HttpRequestModule()
  83. {
  84. }
  85. #region INonSharedRegionModule Members
  86. public void Initialise(IConfigSource config)
  87. {
  88. lock (m_mainLock)
  89. {
  90. // shared items
  91. if (m_jobEngine is null)
  92. {
  93. WebProxy proxy = null;
  94. string proxyurl = config.Configs["Startup"].GetString("HttpProxy");
  95. if (!string.IsNullOrEmpty(proxyurl))
  96. {
  97. string[] proxyexceptsArray = null;
  98. string proxyexcepts = config.Configs["Startup"].GetString("HttpProxyExceptions");
  99. if (!string.IsNullOrEmpty(proxyexcepts))
  100. {
  101. proxyexceptsArray = proxyexcepts.Split(';');
  102. if(proxyexceptsArray.Length == 0)
  103. proxyexceptsArray = null;
  104. }
  105. proxy = proxyexceptsArray is null ?
  106. new WebProxy(proxyurl, true) :
  107. new WebProxy(proxyurl, true, proxyexceptsArray);
  108. }
  109. m_HttpBodyMaxLenMAX = config.Configs["Network"].GetInt("HttpBodyMaxLenMAX", m_HttpBodyMaxLenMAX);
  110. m_outboundUrlFilter = new OutboundUrlFilter("Script HTTP request module", config);
  111. int maxThreads = 8;
  112. IConfig httpConfig = config.Configs["ScriptsHttpRequestModule"];
  113. int httpTimeout = 30000;
  114. if (httpConfig is not null)
  115. {
  116. maxThreads = httpConfig.GetInt("MaxPoolThreads", maxThreads);
  117. m_primBurst = httpConfig.GetFloat("PrimRequestsBurst", m_primBurst);
  118. m_primPerSec = httpConfig.GetFloat("PrimRequestsPerSec", m_primPerSec);
  119. m_primOwnerBurst = httpConfig.GetFloat("PrimOwnerRequestsBurst", m_primOwnerBurst);
  120. m_primOwnerPerSec = httpConfig.GetFloat("PrimOwnerRequestsPerSec", m_primOwnerPerSec);
  121. httpTimeout = httpConfig.GetInt("RequestsTimeOut", httpTimeout);
  122. if (httpTimeout > 60000)
  123. httpTimeout = 60000;
  124. else if (httpTimeout < 200)
  125. httpTimeout = 200;
  126. }
  127. if (VeriFyNoCertClient is null)
  128. {
  129. SocketsHttpHandler shhnc = new()
  130. {
  131. AllowAutoRedirect = false,
  132. AutomaticDecompression = DecompressionMethods.None,
  133. ConnectTimeout = TimeSpan.FromMilliseconds(httpTimeout),
  134. PreAuthenticate = false,
  135. UseCookies = false,
  136. MaxConnectionsPerServer = maxThreads < 10 ? maxThreads : 10,
  137. PooledConnectionLifetime = TimeSpan.FromMinutes(3)
  138. };
  139. //shhnc.SslOptions.ClientCertificates = null,
  140. shhnc.SslOptions.EnabledSslProtocols = SslProtocols.Tls | SslProtocols.Tls11 | SslProtocols.Tls12 | SslProtocols.Tls13;
  141. shhnc.SslOptions.CertificateRevocationCheckMode = X509RevocationMode.NoCheck;
  142. shhnc.SslOptions.RemoteCertificateValidationCallback = (message, cert, chain, errors) =>
  143. {
  144. errors &= ~(SslPolicyErrors.RemoteCertificateChainErrors | SslPolicyErrors.RemoteCertificateNameMismatch);
  145. return errors == SslPolicyErrors.None;
  146. };
  147. if (proxy is null)
  148. shhnc.UseProxy = false;
  149. else
  150. {
  151. shhnc.Proxy = proxy;
  152. shhnc.UseProxy = true;
  153. }
  154. VeriFyNoCertClient = new HttpClient(shhnc)
  155. {
  156. Timeout = TimeSpan.FromMilliseconds(httpTimeout),
  157. MaxResponseContentBufferSize = 2 * m_HttpBodyMaxLenMAX,
  158. };
  159. VeriFyNoCertClient.DefaultRequestHeaders.ExpectContinue = false;
  160. VeriFyNoCertClient.DefaultRequestHeaders.ConnectionClose = true;
  161. }
  162. if (VeriFyCertClient is null)
  163. {
  164. SocketsHttpHandler shh = new()
  165. {
  166. AllowAutoRedirect = false,
  167. AutomaticDecompression = DecompressionMethods.None,
  168. ConnectTimeout = TimeSpan.FromMilliseconds((double)httpTimeout),
  169. PreAuthenticate = false,
  170. UseCookies = false,
  171. MaxConnectionsPerServer = maxThreads < 10 ? maxThreads : 10,
  172. PooledConnectionLifetime = TimeSpan.FromMinutes(3)
  173. };
  174. //shhnc.SslOptions.ClientCertificates = null,
  175. shh.SslOptions.EnabledSslProtocols = SslProtocols.Tls | SslProtocols.Tls11 | SslProtocols.Tls12 | SslProtocols.Tls13;
  176. shh.SslOptions.CertificateRevocationCheckMode = X509RevocationMode.NoCheck;
  177. shh.SslOptions.RemoteCertificateValidationCallback = (message, cert, chain, errors) =>
  178. {
  179. errors &= ~SslPolicyErrors.RemoteCertificateChainErrors;
  180. return errors == SslPolicyErrors.None;
  181. };
  182. if (proxy is null)
  183. shh.UseProxy = false;
  184. else
  185. {
  186. shh.Proxy = proxy;
  187. shh.UseProxy = true;
  188. }
  189. VeriFyCertClient = new HttpClient(shh)
  190. {
  191. Timeout = TimeSpan.FromMilliseconds(httpTimeout),
  192. MaxResponseContentBufferSize = 2 * m_HttpBodyMaxLenMAX
  193. };
  194. VeriFyCertClient.DefaultRequestHeaders.ExpectContinue = false;
  195. VeriFyCertClient.DefaultRequestHeaders.ConnectionClose = true;
  196. }
  197. m_pendingRequests = new Dictionary<UUID, HttpRequestClass>();
  198. m_jobEngine = new JobEngine("ScriptsHttpReq", "ScriptsHttpReq", 2000, maxThreads);
  199. m_jobEngine.Start();
  200. }
  201. }
  202. }
  203. public void AddRegion(Scene scene)
  204. {
  205. scene.RegisterModuleInterface<IHttpRequestModule>(this);
  206. Interlocked.Increment(ref m_numberScenes);
  207. }
  208. public void RemoveRegion(Scene scene)
  209. {
  210. scene.UnregisterModuleInterface<IHttpRequestModule>(this);
  211. }
  212. public void PostInitialise()
  213. {
  214. }
  215. public void RegionLoaded(Scene scene)
  216. {
  217. }
  218. public void Close()
  219. {
  220. int n = Interlocked.Decrement(ref m_numberScenes);
  221. if (n == 0)
  222. {
  223. lock(m_mainLock)
  224. {
  225. if (m_jobEngine is not null)
  226. {
  227. m_jobEngine.Stop();
  228. m_jobEngine = null;
  229. }
  230. VeriFyCertClient?.Dispose();
  231. VeriFyCertClient = null;
  232. VeriFyNoCertClient?.Dispose();
  233. VeriFyNoCertClient = null;
  234. }
  235. }
  236. }
  237. public string Name
  238. {
  239. get { return m_name; }
  240. }
  241. public Type ReplaceableInterface
  242. {
  243. get { return null; }
  244. }
  245. #endregion
  246. #region IHttpRequestModule Members
  247. public UUID MakeHttpRequest(string url, string parameters, string body)
  248. {
  249. return UUID.Zero;
  250. }
  251. public HttpClient GetHttpClient(bool verify)
  252. {
  253. return verify ? VeriFyCertClient : VeriFyNoCertClient;
  254. }
  255. public bool CheckThrottle(uint localID, UUID ownerID)
  256. {
  257. double now = Util.GetTimeStamp();
  258. bool ret;
  259. if (m_RequestsThrottle.TryGetValue(localID, out ThrottleData th))
  260. {
  261. double delta = now - th.lastTime;
  262. th.lastTime = now;
  263. float add = (float)(m_primPerSec * delta);
  264. th.control += add;
  265. if (th.control > m_primBurst)
  266. {
  267. th.control = m_primBurst - 1;
  268. ret = true;
  269. }
  270. else
  271. {
  272. ret = th.control > 0;
  273. if (ret)
  274. th.control--;
  275. }
  276. }
  277. else
  278. {
  279. th = new ThrottleData()
  280. {
  281. lastTime = now,
  282. control = m_primBurst - 1,
  283. };
  284. ret = true;
  285. }
  286. m_RequestsThrottle[localID] = th;
  287. if(!ret)
  288. return false;
  289. if (m_OwnerRequestsThrottle.TryGetValue(ownerID, out th))
  290. {
  291. double delta = now - th.lastTime;
  292. th.lastTime = now;
  293. float add = (float)(m_primOwnerPerSec * delta);
  294. th.control += add;
  295. if (th.control > m_primOwnerBurst)
  296. th.control = m_primOwnerBurst - 1;
  297. else
  298. {
  299. ret = th.control > 0;
  300. if (ret)
  301. th.control--;
  302. }
  303. }
  304. else
  305. {
  306. th = new ThrottleData()
  307. {
  308. lastTime = now,
  309. control = m_primBurst - 1
  310. };
  311. }
  312. m_OwnerRequestsThrottle[ownerID] = th;
  313. return ret;
  314. }
  315. public UUID StartHttpRequest(uint localID, UUID itemID, string url,
  316. List<string> parameters, Dictionary<string, string> headers, string body)
  317. {
  318. UUID reqID = UUID.Random();
  319. HttpRequestClass htc = new();
  320. // Partial implementation: support for parameter flags needed
  321. // see http://wiki.secondlife.com/wiki/LlHTTPRequest
  322. //
  323. // Parameters are expected in {key, value, ... , key, value}
  324. if (parameters is not null)
  325. {
  326. for (int i = 0; i < parameters.Count; i += 2)
  327. {
  328. switch (Int32.Parse(parameters[i]))
  329. {
  330. case (int)HttpRequestConstants.HTTP_METHOD:
  331. htc.HttpMethod = parameters[i + 1];
  332. break;
  333. case (int)HttpRequestConstants.HTTP_MIMETYPE:
  334. htc.HttpMIMEType = parameters[i + 1];
  335. break;
  336. case (int)HttpRequestConstants.HTTP_BODY_MAXLENGTH:
  337. if(int.TryParse(parameters[i + 1], out int len))
  338. {
  339. if(len > m_HttpBodyMaxLenMAX)
  340. len = m_HttpBodyMaxLenMAX;
  341. else if(len < 64) //???
  342. len = 64;
  343. htc.HttpBodyMaxLen = len;
  344. }
  345. break;
  346. case (int)HttpRequestConstants.HTTP_VERIFY_CERT:
  347. htc.HttpVerifyCert = (int.Parse(parameters[i + 1]) != 0);
  348. break;
  349. case (int)HttpRequestConstants.HTTP_VERBOSE_THROTTLE:
  350. break;
  351. case (int)HttpRequestConstants.HTTP_CUSTOM_HEADER:
  352. // should not happen
  353. //Parameters are in pairs and custom header takes
  354. //arguments in pairs so adjust for header marker.
  355. ++i;
  356. //Maximum of 8 headers are allowed based on the
  357. //Second Life documentation for llHTTPRequest.
  358. for (int count = 1; count <= 8; ++count)
  359. {
  360. //Not enough parameters remaining for a header?
  361. if (parameters.Count - i < 2)
  362. break;
  363. int nexti = i + 2;
  364. if (nexti >= parameters.Count || Char.IsDigit(parameters[nexti][0]))
  365. break;
  366. i = nexti;
  367. }
  368. break;
  369. case (int)HttpRequestConstants.HTTP_PRAGMA_NO_CACHE:
  370. htc.HttpPragmaNoCache = (int.Parse(parameters[i + 1]) != 0);
  371. break;
  372. }
  373. }
  374. }
  375. htc.RequestModule = this;
  376. htc.LocalID = localID;
  377. htc.ItemID = itemID;
  378. htc.Url = url;
  379. htc.ReqID = reqID;
  380. htc.OutboundBody = body;
  381. htc.Headers = headers;
  382. lock (m_mainLock)
  383. m_pendingRequests.Add(reqID, htc);
  384. htc.Process();
  385. return reqID;
  386. }
  387. /// <summary>
  388. /// Would a caller to this module be allowed to make a request to the given URL?
  389. /// </summary>
  390. /// <returns></returns>
  391. public bool CheckAllowed(Uri url)
  392. {
  393. return m_outboundUrlFilter.CheckAllowed(url);
  394. }
  395. public void StopHttpRequest(uint localID, UUID m_itemID)
  396. {
  397. List<UUID> toremove = new();
  398. lock (m_mainLock)
  399. {
  400. foreach (HttpRequestClass tmpReq in m_pendingRequests.Values)
  401. {
  402. if(m_itemID.Equals(tmpReq.ItemID))
  403. {
  404. tmpReq.Stop();
  405. toremove.Add(tmpReq.ReqID);
  406. }
  407. }
  408. foreach(UUID id in toremove)
  409. m_pendingRequests.Remove(id);
  410. }
  411. if (m_RequestsThrottle.TryGetValue(localID, out ThrottleData th))
  412. {
  413. if (th.control + m_primOwnerPerSec * (Util.GetTimeStamp() - th.lastTime) >= m_primBurst)
  414. m_RequestsThrottle.TryRemove(localID, out _);
  415. }
  416. }
  417. /*
  418. * TODO
  419. * Not sure how important ordering is is here - the next first
  420. * one completed in the list is returned, based soley on its list
  421. * position, not the order in which the request was started or
  422. * finished. I thought about setting up a queue for this, but
  423. * it will need some refactoring and this works 'enough' right now
  424. */
  425. public void GotCompletedRequest(HttpRequestClass req)
  426. {
  427. lock (m_mainLock)
  428. {
  429. m_pendingRequests.Remove(req.ReqID);
  430. if (!req.Removed)
  431. m_CompletedRequests.Enqueue(req);
  432. }
  433. }
  434. public IServiceRequest GetNextCompletedRequest()
  435. {
  436. if(m_CompletedRequests.TryDequeue(out HttpRequestClass req))
  437. return req;
  438. return null;
  439. }
  440. public void RemoveCompletedRequest(UUID reqId)
  441. {
  442. lock (m_mainLock)
  443. {
  444. if (m_pendingRequests.TryGetValue(reqId, out HttpRequestClass tmpReq))
  445. {
  446. tmpReq.Stop();
  447. m_pendingRequests.Remove(reqId);
  448. }
  449. }
  450. }
  451. #endregion
  452. }
  453. public class HttpRequestClass : IServiceRequest
  454. {
  455. private static readonly string[] s_wellKnownContentHeaders = {
  456. "Content-Disposition",
  457. "Content-Encoding",
  458. "Content-Language",
  459. "Content-Length",
  460. "Content-Location",
  461. "Content-MD5",
  462. "Content-Range",
  463. "Content-Type",
  464. "Expires",
  465. "Last-Modified"
  466. };
  467. private bool IsWellKnownContentHeader(string header)
  468. {
  469. foreach (string contentHeaderName in s_wellKnownContentHeaders)
  470. {
  471. if (string.Equals(header, contentHeaderName, StringComparison.OrdinalIgnoreCase))
  472. return true;
  473. }
  474. return false;
  475. }
  476. private void AddHeader(string headerName, string value, HttpRequestMessage request)
  477. {
  478. if (IsWellKnownContentHeader(headerName))
  479. {
  480. request.Content ??= new ByteArrayContent(Array.Empty<byte>());
  481. request.Content.Headers.TryAddWithoutValidation(headerName, value);
  482. }
  483. else
  484. request.Headers.TryAddWithoutValidation(headerName, value);
  485. }
  486. /// <summary>
  487. /// Module that made this request.
  488. /// </summary>
  489. public HttpRequestModule RequestModule { get; set; }
  490. public bool HttpVerifyCert = true;
  491. public bool Removed;
  492. // Parameter members and default values
  493. public int HttpBodyMaxLen = 2048;
  494. public string HttpMethod = "GET";
  495. public string HttpMIMEType = "text/plain;charset=utf-8";
  496. public bool HttpPragmaNoCache = false;
  497. // Request info
  498. public bool Finished { get; }
  499. public UUID ReqID { get; set; }
  500. public UUID ItemID { get; set;}
  501. public uint LocalID { get; set;}
  502. /// <summary>
  503. /// Number of HTTP redirects that this request has been through.
  504. /// </summary>
  505. public int Redirects { get; private set; }
  506. /// <summary>
  507. /// Maximum number of HTTP redirects allowed for this request.
  508. /// </summary>
  509. public int MaxRedirects { get; set; } = 10;
  510. public string OutboundBody;
  511. public string ResponseBody;
  512. public Dictionary<string, string> Headers;
  513. public int Status;
  514. public string Url;
  515. public void Process()
  516. {
  517. HttpRequestModule.m_jobEngine?.QueueJob("", SendRequest);
  518. }
  519. public void SendRequest()
  520. {
  521. if (Removed)
  522. return;
  523. HttpResponseMessage responseMessage = null;
  524. HttpRequestMessage request = null;
  525. try
  526. {
  527. HttpClient client = RequestModule.GetHttpClient(HttpVerifyCert);
  528. request = new (new HttpMethod(HttpMethod), Url);
  529. int datalen;
  530. if (!string.IsNullOrEmpty(OutboundBody))
  531. {
  532. byte[] data = Util.UTF8.GetBytes(OutboundBody);
  533. datalen = data.Length;
  534. request.Content = new ByteArrayContent(data);
  535. }
  536. else
  537. datalen = -1;
  538. foreach (KeyValuePair<string, string> entry in Headers)
  539. AddHeader(entry.Key, entry.Value, request);
  540. if (HttpPragmaNoCache)
  541. request.Headers.TryAddWithoutValidation("Pragma", "no-cache");
  542. request.Headers.TransferEncodingChunked = false;
  543. request.Headers.ConnectionClose = true;
  544. if (datalen > 0)
  545. {
  546. request.Content.Headers.TryAddWithoutValidation("Content-Type", HttpMIMEType);
  547. request.Content.Headers.TryAddWithoutValidation("Content-Length", datalen.ToString());
  548. }
  549. if (Removed)
  550. return;
  551. responseMessage = client.Send(request, HttpCompletionOption.ResponseHeadersRead);
  552. if (Removed)
  553. return;
  554. Status = (int)responseMessage.StatusCode;
  555. if (responseMessage.Content is not null)
  556. {
  557. int len;
  558. if(responseMessage.Content.Headers is not null && responseMessage.Content.Headers.ContentLength is long l)
  559. len = (int)l;
  560. else
  561. len = -1;
  562. Stream resStream = responseMessage.Content.ReadAsStream();
  563. if(resStream is not null)
  564. {
  565. int maxBytes = (len < 0 || len > HttpBodyMaxLen) ? HttpBodyMaxLen : len;
  566. byte[] buf = new byte[maxBytes];
  567. int totalBodyBytes = 0;
  568. int count;
  569. do
  570. {
  571. count = resStream.Read(buf, totalBodyBytes, maxBytes - totalBodyBytes);
  572. totalBodyBytes += count;
  573. } while (count > 0 && totalBodyBytes < maxBytes); // any more data to read?
  574. resStream.Dispose();
  575. if (totalBodyBytes > 0)
  576. {
  577. string tempString = Util.UTF8.GetString(buf, 0, totalBodyBytes);
  578. ResponseBody = tempString.Replace("\r", "");
  579. }
  580. }
  581. }
  582. }
  583. catch (HttpRequestException e)
  584. {
  585. Status = e.StatusCode is null ? 499 : (int)e.StatusCode;
  586. ResponseBody = e.Message;
  587. }
  588. //catch (Exception e)
  589. catch
  590. {
  591. // Don't crash on anything else
  592. }
  593. finally
  594. {
  595. if (!Removed)
  596. {
  597. // We need to resubmit ?
  598. if (Status == (int)HttpStatusCode.MovedPermanently ||
  599. Status == (int)HttpStatusCode.Found ||
  600. Status == (int)HttpStatusCode.SeeOther ||
  601. Status == (int)HttpStatusCode.TemporaryRedirect)
  602. {
  603. if (Redirects >= MaxRedirects)
  604. {
  605. Status = 499;//.ClientErrorJoker;
  606. ResponseBody = "Number of redirects exceeded max redirects";
  607. RequestModule.GotCompletedRequest(this);
  608. }
  609. else if (responseMessage is not null && responseMessage.Headers is not null)
  610. {
  611. Uri locationUri = responseMessage.Headers.Location;
  612. if (locationUri == null)
  613. {
  614. Status = 499;//ClientErrorJoker;
  615. ResponseBody = "HTTP redirect code but no location header";
  616. RequestModule.GotCompletedRequest(this);
  617. }
  618. else
  619. {
  620. bool validredir = true;
  621. if(!locationUri.IsAbsoluteUri)
  622. {
  623. Uri reqUri = responseMessage.RequestMessage.RequestUri;
  624. string newloc = reqUri.Scheme +"://" + reqUri.DnsSafeHost + ":" +
  625. reqUri.Port +"/" + locationUri.OriginalString;
  626. if (!Uri.TryCreate(newloc, UriKind.RelativeOrAbsolute, out locationUri))
  627. {
  628. Status = 499;//ClientErrorJoker;
  629. ResponseBody = "HTTP redirect code but invalid location header";
  630. RequestModule.GotCompletedRequest(this);
  631. validredir = false;
  632. }
  633. }
  634. if(validredir)
  635. {
  636. if (!RequestModule.CheckAllowed(locationUri))
  637. {
  638. Status = 499;//ClientErrorJoker;
  639. ResponseBody = "URL from HTTP redirect blocked: " + locationUri.AbsoluteUri;
  640. RequestModule.GotCompletedRequest(this);
  641. }
  642. else
  643. {
  644. Status = 0;
  645. Url = locationUri.AbsoluteUri;
  646. Redirects++;
  647. ResponseBody = null;
  648. //m_log.DebugFormat("Redirecting to [{0}]", Url);
  649. Process();
  650. }
  651. }
  652. else
  653. {
  654. Status = 499;//ClientErrorJoker;
  655. ResponseBody = "HTTP redirect code but invalid location header";
  656. RequestModule.GotCompletedRequest(this);
  657. }
  658. }
  659. }
  660. }
  661. else
  662. {
  663. ResponseBody ??= string.Empty;
  664. RequestModule.GotCompletedRequest(this);
  665. }
  666. }
  667. responseMessage?.Dispose();
  668. request.Dispose();
  669. }
  670. }
  671. public void Stop()
  672. {
  673. Removed = true;
  674. }
  675. }
  676. }