From 19229aff64c6c2adc458fc5647ee7aa07e85cd56 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Tue, 16 May 2023 22:57:23 +0100 Subject: [PATCH] replace HttpWebRequest by HttpClient on another place --- .../Framework/Servers/BaseOpenSimServer.cs | 10 +- .../ServiceAuth/BasicHttpAuthentication.cs | 10 +- .../ServiceAuth/CompoundAuthentication.cs | 7 + .../ServiceAuth/DisallowLlHttpRequest.cs | 4 +- OpenSim/Framework/ServiceAuth/IServiceAuth.cs | 2 + OpenSim/Framework/Util.cs | 2 +- OpenSim/Framework/WebUtil.cs | 266 ++++++++++++++---- OpenSim/Server/ServerMain.cs | 2 + 8 files changed, 244 insertions(+), 59 deletions(-) diff --git a/OpenSim/Framework/Servers/BaseOpenSimServer.cs b/OpenSim/Framework/Servers/BaseOpenSimServer.cs index 63cc4bb0a2..1309351e2b 100644 --- a/OpenSim/Framework/Servers/BaseOpenSimServer.cs +++ b/OpenSim/Framework/Servers/BaseOpenSimServer.cs @@ -27,25 +27,17 @@ using System; using System.Collections.Generic; -using System.IO; using System.Reflection; using System.Text; -using System.Text.RegularExpressions; using System.Threading; using System.Timers; using System.Net; using System.Net.Security; using System.Security.Cryptography.X509Certificates; using log4net; -using log4net.Appender; -using log4net.Core; -using log4net.Repository; using OpenMetaverse; -using OpenMetaverse.StructuredData; using OpenSim.Framework; -using OpenSim.Framework.Console; using OpenSim.Framework.Monitoring; -using OpenSim.Framework.Servers; using OpenSim.Framework.Servers.HttpServer; using Timer=System.Timers.Timer; using Nini.Config; @@ -125,6 +117,8 @@ namespace OpenSim.Framework.Servers m_NoVerifyCertHostname = startupConfig.GetBoolean("NoVerifyCertHostname", m_NoVerifyCertHostname); ServicePointManager.ServerCertificateValidationCallback = ValidateServerCertificate; + WebUtil.SetupHTTPClients(m_NoVerifyCertChain, m_NoVerifyCertHostname, null, 32 ); + int logShowStatsSeconds = startupConfig.GetInt("LogShowStatsSeconds", m_periodDiagnosticTimerMS / 1000); m_periodDiagnosticTimerMS = logShowStatsSeconds * 1000; m_periodicDiagnosticsTimer.Elapsed += new ElapsedEventHandler(LogDiagnostics); diff --git a/OpenSim/Framework/ServiceAuth/BasicHttpAuthentication.cs b/OpenSim/Framework/ServiceAuth/BasicHttpAuthentication.cs index 4f16a4644b..c1c94e1645 100644 --- a/OpenSim/Framework/ServiceAuth/BasicHttpAuthentication.cs +++ b/OpenSim/Framework/ServiceAuth/BasicHttpAuthentication.cs @@ -33,6 +33,7 @@ using System.Reflection; using Nini.Config; using log4net; +using System.Net.Http.Headers; namespace OpenSim.Framework.ServiceAuth { @@ -54,14 +55,13 @@ namespace OpenSim.Framework.ServiceAuth public BasicHttpAuthentication(IConfigSource config, string section) { -// remove_me = section; m_Username = Util.GetConfigVarFromSections(config, "HttpAuthUsername", new string[] { "Network", section }, string.Empty); m_Password = Util.GetConfigVarFromSections(config, "HttpAuthPassword", new string[] { "Network", section }, string.Empty); string str = m_Username + ":" + m_Password; byte[] encData_byte = Util.UTF8.GetBytes(str); m_CredentialsB64 = Convert.ToBase64String(encData_byte); -// m_log.DebugFormat("[HTTP BASIC AUTH]: {0} {1} [{2}]", m_Username, m_Password, section); + //m_log.DebugFormat("[HTTP BASIC AUTH]: {0} {1} [{2}]", m_Username, m_Password, section); } public void AddAuthorization(NameValueCollection headers) @@ -70,6 +70,12 @@ namespace OpenSim.Framework.ServiceAuth headers["Authorization"] = "Basic " + m_CredentialsB64; } + public void AddAuthorization(HttpRequestHeaders headers) + { + //m_log.DebugFormat("[HTTP BASIC AUTH]: Adding authorization for {0}", remove_me); + headers.TryAddWithoutValidation("Authorization","Basic " + m_CredentialsB64); + } + public bool Authenticate(string data) { string recovered = Util.Base64ToString(data); diff --git a/OpenSim/Framework/ServiceAuth/CompoundAuthentication.cs b/OpenSim/Framework/ServiceAuth/CompoundAuthentication.cs index 79d6ff429b..4b13a80b5b 100644 --- a/OpenSim/Framework/ServiceAuth/CompoundAuthentication.cs +++ b/OpenSim/Framework/ServiceAuth/CompoundAuthentication.cs @@ -30,6 +30,7 @@ using System.Collections.Generic; using System.Collections.Specialized; using System.Linq; using System.Net; +using System.Net.Http.Headers; namespace OpenSim.Framework.ServiceAuth { @@ -62,6 +63,12 @@ namespace OpenSim.Framework.ServiceAuth auth.AddAuthorization(headers); } + public void AddAuthorization(HttpRequestHeaders headers) + { + foreach (IServiceAuth auth in m_authentications) + auth.AddAuthorization(headers); + } + public bool Authenticate(string data) { return m_authentications.TrueForAll(a => a.Authenticate(data)); diff --git a/OpenSim/Framework/ServiceAuth/DisallowLlHttpRequest.cs b/OpenSim/Framework/ServiceAuth/DisallowLlHttpRequest.cs index e0c413bac9..e2f64ce99f 100644 --- a/OpenSim/Framework/ServiceAuth/DisallowLlHttpRequest.cs +++ b/OpenSim/Framework/ServiceAuth/DisallowLlHttpRequest.cs @@ -28,6 +28,7 @@ using System; using System.Collections.Specialized; using System.Net; +using System.Net.Http.Headers; namespace OpenSim.Framework.ServiceAuth { @@ -35,7 +36,8 @@ namespace OpenSim.Framework.ServiceAuth { public string Name { get { return "DisallowllHTTPRequest"; } } - public void AddAuthorization(NameValueCollection headers) {} + public void AddAuthorization(NameValueCollection headers) { } + public void AddAuthorization(HttpRequestHeaders headers) { } public bool Authenticate(string data) { diff --git a/OpenSim/Framework/ServiceAuth/IServiceAuth.cs b/OpenSim/Framework/ServiceAuth/IServiceAuth.cs index 5f744cbe10..4fe309dd79 100644 --- a/OpenSim/Framework/ServiceAuth/IServiceAuth.cs +++ b/OpenSim/Framework/ServiceAuth/IServiceAuth.cs @@ -29,6 +29,7 @@ using System; using System.Net; using System.Collections.Generic; using System.Collections.Specialized; +using System.Net.Http.Headers; namespace OpenSim.Framework.ServiceAuth { @@ -44,5 +45,6 @@ namespace OpenSim.Framework.ServiceAuth bool Authenticate(string data); bool Authenticate(NameValueCollection headers, AddHeaderDelegate d, out HttpStatusCode statusCode); void AddAuthorization(NameValueCollection headers); + void AddAuthorization(HttpRequestHeaders headers); } } diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs index 61568676d8..ecfefe8899 100644 --- a/OpenSim/Framework/Util.cs +++ b/OpenSim/Framework/Util.cs @@ -422,7 +422,7 @@ namespace OpenSim.Framework [MethodImpl(MethodImplOptions.AggressiveInlining)] public static ulong RegionWorldLocToHandle(uint X, uint Y) { - ulong handle = X & 0xffffff00; // make sure it matchs grid coord points. + ulong handle = X & 0xffffff00; // make sure it matches grid coord points. handle <<= 32; // to higher half handle |= (Y & 0xffffff00); return handle; diff --git a/OpenSim/Framework/WebUtil.cs b/OpenSim/Framework/WebUtil.cs index c5bf5870af..f0817be85e 100644 --- a/OpenSim/Framework/WebUtil.cs +++ b/OpenSim/Framework/WebUtil.cs @@ -44,6 +44,9 @@ using log4net; using Nwc.XmlRpc; using OpenMetaverse.StructuredData; using OpenSim.Framework.ServiceAuth; +using System.Net.Http; +using System.Security.Authentication; +using System.Runtime.CompilerServices; namespace OpenSim.Framework { @@ -56,6 +59,12 @@ namespace OpenSim.Framework { private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + public static SocketsHttpHandler SharedSocketsHttpHandler = null; + public static HttpClient SharedHttpClient = null; + + public static SocketsHttpHandler SharedSocketsHttpHandlerWithRedir = null; + public static HttpClient SharedHttpClientWithRedir = null; + public static ExpiringKey GlobalExpiringBadURLs = new(30000); /// /// Control the printing of certain debug messages. @@ -104,35 +113,187 @@ namespace OpenSim.Framework } #region JSONRequest + public static void SetupHTTPClients(bool NoVerifyCertChain, bool NoVerifyCertHostname, IWebProxy proxy, int MaxConnectionsPerServer ) + { + SocketsHttpHandler shh = new() + { + AllowAutoRedirect = false, + AutomaticDecompression = DecompressionMethods.None, + ConnectTimeout = TimeSpan.FromMilliseconds(10000), + PreAuthenticate = false, + UseCookies = false, + MaxConnectionsPerServer = MaxConnectionsPerServer, + PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(30000), + PooledConnectionLifetime = TimeSpan.FromMinutes(3) + }; + //shh.SslOptions.ClientCertificates = null, + shh.SslOptions.EnabledSslProtocols = SslProtocols.Tls | SslProtocols.Tls11 | SslProtocols.Tls12 | SslProtocols.Tls13; + if (NoVerifyCertChain) + { + shh.SslOptions.CertificateRevocationCheckMode = X509RevocationMode.NoCheck; + if (NoVerifyCertHostname) + { + shh.SslOptions.RemoteCertificateValidationCallback = (message, cert, chain, errors) => + { + errors &= ~(SslPolicyErrors.RemoteCertificateChainErrors | SslPolicyErrors.RemoteCertificateNameMismatch); + return errors == SslPolicyErrors.None; + }; + } + else + { + shh.SslOptions.RemoteCertificateValidationCallback = (message, cert, chain, errors) => + { + errors &= ~SslPolicyErrors.RemoteCertificateChainErrors; + return errors == SslPolicyErrors.None; + }; + } + } + else + { + shh.SslOptions.CertificateRevocationCheckMode = X509RevocationMode.NoCheck; + if (NoVerifyCertHostname) + { + shh.SslOptions.RemoteCertificateValidationCallback = (message, cert, chain, errors) => + { + errors &= ~SslPolicyErrors.RemoteCertificateNameMismatch; + return errors == SslPolicyErrors.None; + }; + } + else + { + shh.SslOptions.RemoteCertificateValidationCallback = (message, cert, chain, errors) => + { + return errors == SslPolicyErrors.None; + }; + } + } + + if (proxy is null) + shh.UseProxy = false; + else + { + shh.Proxy = proxy; + shh.UseProxy = true; + } + + var client = new HttpClient(shh) + { + Timeout = TimeSpan.FromMilliseconds(30000), + }; + client.DefaultRequestHeaders.ExpectContinue = false; + + SharedSocketsHttpHandler = shh; + SharedHttpClient = client; + + // **************** + + shh = new() + { + AllowAutoRedirect = true, + MaxAutomaticRedirections = 10, + AutomaticDecompression = DecompressionMethods.None, + ConnectTimeout = TimeSpan.FromMilliseconds(10000), + PreAuthenticate = false, + UseCookies = false, + MaxConnectionsPerServer = MaxConnectionsPerServer, + PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(30000), + PooledConnectionLifetime = TimeSpan.FromMinutes(3) + }; + //shh.SslOptions.ClientCertificates = null, + shh.SslOptions.EnabledSslProtocols = SslProtocols.Tls | SslProtocols.Tls11 | SslProtocols.Tls12 | SslProtocols.Tls13; + if (NoVerifyCertChain) + { + shh.SslOptions.CertificateRevocationCheckMode = X509RevocationMode.NoCheck; + if (NoVerifyCertHostname) + { + shh.SslOptions.RemoteCertificateValidationCallback = (message, cert, chain, errors) => + { + errors &= ~(SslPolicyErrors.RemoteCertificateChainErrors | SslPolicyErrors.RemoteCertificateNameMismatch); + return errors == SslPolicyErrors.None; + }; + } + else + { + shh.SslOptions.RemoteCertificateValidationCallback = (message, cert, chain, errors) => + { + errors &= ~SslPolicyErrors.RemoteCertificateChainErrors; + return errors == SslPolicyErrors.None; + }; + } + } + else + { + shh.SslOptions.CertificateRevocationCheckMode = X509RevocationMode.NoCheck; + if (NoVerifyCertHostname) + { + shh.SslOptions.RemoteCertificateValidationCallback = (message, cert, chain, errors) => + { + errors &= ~SslPolicyErrors.RemoteCertificateNameMismatch; + return errors == SslPolicyErrors.None; + }; + } + else + { + shh.SslOptions.RemoteCertificateValidationCallback = (message, cert, chain, errors) => + { + return errors == SslPolicyErrors.None; + }; + } + } + + if (proxy is null) + shh.UseProxy = false; + else + { + shh.Proxy = proxy; + shh.UseProxy = true; + } + + client = new HttpClient(shh) + { + Timeout = TimeSpan.FromMilliseconds(30000), + }; + client.DefaultRequestHeaders.ExpectContinue = false; + + SharedSocketsHttpHandlerWithRedir = shh; + SharedHttpClientWithRedir = client; + } + /// /// PUT JSON-encoded data to a web service that returns LLSD or /// JSON data /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static OSDMap PutToServiceCompressed(string url, OSDMap data, int timeout) { return ServiceOSDRequest(url, data, "PUT", timeout, true, false); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static OSDMap PutToService(string url, OSDMap data, int timeout) { return ServiceOSDRequest(url, data, "PUT", timeout, false, false); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static OSDMap PostToService(string url, OSDMap data, int timeout, bool rpc) { return ServiceOSDRequest(url, data, "POST", timeout, false, rpc); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static OSDMap PostToServiceCompressed(string url, OSDMap data, int timeout) { return ServiceOSDRequest(url, data, "POST", timeout, true, false); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static OSDMap GetFromService(string url, int timeout) { return ServiceOSDRequest(url, null, "GET", timeout, false, false); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void LogOutgoingDetail(Stream outputStream) { LogOutgoingDetail("", outputStream); @@ -158,11 +319,13 @@ namespace OpenSim.Framework LogOutgoingDetail(context, output); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void LogOutgoingDetail(string type, int reqnum, string output) { LogOutgoingDetail($"{type} {reqnum}: ", output); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void LogOutgoingDetail(string context, string output) { if (DebugLevel == 5) @@ -174,11 +337,13 @@ namespace OpenSim.Framework m_log.DebugFormat($"[LOGHTTP]: {context}{Util.BinaryToASCII(output)}"); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void LogResponseDetail(int reqnum, Stream inputStream) { LogOutgoingDetail($"RESPONSE {reqnum}: ", inputStream); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void LogResponseDetail(int reqnum, string input) { LogOutgoingDetail($"RESPONSE {reqnum}: ", input); @@ -195,27 +360,14 @@ namespace OpenSim.Framework int tickstart = Util.EnvironmentTickCount(); int sendlen = 0; int rcvlen = 0; - HttpWebRequest request; + HttpResponseMessage responseMessage = null; + HttpRequestMessage request = null; try { - request = (HttpWebRequest)WebRequest.Create(url); - request.Method = method; - request.Timeout = timeout; - request.KeepAlive = keepalive; - request.MaximumAutomaticRedirections = 10; - request.ReadWriteTimeout = timeout / 2; - request.Headers[OSHeaderRequestID] = reqnum.ToString(); - request.AllowWriteStreamBuffering = false; - } - catch (Exception ex) - { - m_log.Debug($"[WEB UTIL]: SvcOSD error creating request {ex.Message}"); - return ErrorResponseMap(ex.Message); - } + HttpClient client = SharedHttpClientWithRedir; + + request = new(new HttpMethod(method), url); - try - { - // If there is some input, write it into the request if (data is not null) { byte[] buffer; @@ -228,43 +380,60 @@ namespace OpenSim.Framework else buffer = OSDParser.SerializeJsonToBytes(data); - request.ContentType = rpc ? "application/json-rpc" : "application/json"; - - if (compressed) + if (buffer.Length > 0) { - request.Headers["X-Content-Encoding"] = "gzip"; // can't set "Content-Encoding" because old OpenSims fail if they get an unrecognized Content-Encoding - - using MemoryStream ms = new(); - using (GZipStream comp = new(ms, CompressionMode.Compress, true)) + if (compressed) { - comp.Write(buffer, 0, buffer.Length); + using MemoryStream ms = new(); + using (GZipStream comp = new(ms, CompressionMode.Compress, true)) + { + comp.Write(buffer, 0, buffer.Length); + } + buffer = ms.ToArray(); + + request.Headers.TryAddWithoutValidation("X-Content-Encoding", "gzip"); // can't set "Content-Encoding" because old OpenSims fail if they get an unrecognized Content-Encoding } - buffer = ms.ToArray(); + + sendlen = buffer.Length; + request.Content = new ByteArrayContent(buffer); + request.Content.Headers.TryAddWithoutValidation("Content-Type", + rpc ? "application/json-rpc" : "application/json"); + request.Content.Headers.TryAddWithoutValidation("Content-Length", sendlen.ToString()); } - - sendlen = buffer.Length; - request.ContentLength = buffer.Length; //Count bytes to send - using (Stream requestStream = request.GetRequestStream()) - requestStream.Write(buffer, 0, buffer.Length); //Send it - buffer = null; } - using HttpWebResponse response = (HttpWebResponse)request.GetResponse(); - using StreamReader reader = new(response.GetResponseStream()); - string responseStr = reader.ReadToEnd(); - if (WebUtil.DebugLevel >= 5) - WebUtil.LogResponseDetail(reqnum, responseStr); - rcvlen = responseStr.Length; - return CanonicalizeResults(responseStr); - } - catch (WebException we) - { - errorMessage = we.Message; - if (we.Status == WebExceptionStatus.ProtocolError) + request.Headers.ExpectContinue = false; + request.Headers.TransferEncodingChunked = false; + if(keepalive) { - using HttpWebResponse webResponse = (HttpWebResponse)we.Response; - errorMessage = $"[{webResponse.StatusCode}] {webResponse.StatusDescription}"; + request.Headers.TryAddWithoutValidation("Keep-Alive", "timeout=30, max=10"); + request.Headers.TryAddWithoutValidation("Connection", "Keep-Alive"); } + else + request.Headers.TryAddWithoutValidation("Connection", "close"); + + request.Headers.TryAddWithoutValidation(OSHeaderRequestID, reqnum.ToString()); + + responseMessage = client.Send(request, HttpCompletionOption.ResponseHeadersRead); + + int Status = (int)responseMessage.StatusCode; + + Stream resStream = responseMessage.Content.ReadAsStream(); + if (resStream is not null) + { + using StreamReader reader = new(resStream); + string responseStr = reader.ReadToEnd(); + if (WebUtil.DebugLevel >= 5) + WebUtil.LogResponseDetail(reqnum, responseStr); + rcvlen = responseStr.Length; + resStream.Dispose(); + return CanonicalizeResults(responseStr); + } + } + catch (HttpRequestException e) + { + int Status = e.StatusCode is null ? 499 : (int)e.StatusCode; + errorMessage = $"[{Status}] {e.Message}"; } catch (Exception ex) { @@ -273,6 +442,9 @@ namespace OpenSim.Framework } finally { + request?.Dispose(); + responseMessage?.Dispose(); + int tickdiff = Util.EnvironmentTickCountSubtract(tickstart); if (tickdiff > LongCallTime) { diff --git a/OpenSim/Server/ServerMain.cs b/OpenSim/Server/ServerMain.cs index 354ef53c65..c0984281a9 100644 --- a/OpenSim/Server/ServerMain.cs +++ b/OpenSim/Server/ServerMain.cs @@ -102,6 +102,8 @@ namespace OpenSim.Server ServicePointManager.UseNagleAlgorithm = false; ServicePointManager.ServerCertificateValidationCallback = ValidateServerCertificate; + WebUtil.SetupHTTPClients(m_NoVerifyCertChain, m_NoVerifyCertHostname, null, 32); + m_Server = new HttpServerBase("R.O.B.U.S.T.", args); string registryLocation;