From f326cc89013490c5d8ed53b7917cc824244ffc1e Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Fri, 22 May 2020 14:00:12 +0100 Subject: [PATCH] reduce garbage on WebFetchInvDesc --- .../FetchInventory/FetchInvDescHandler.cs | 164 +++++++++--------- .../FetchInvDescServerConnector.cs | 12 +- .../FetchInventoryDescendents2HandlerTests.cs | 34 ++-- .../Linden/Caps/WebFetchInvDescModule.cs | 21 +-- 4 files changed, 111 insertions(+), 120 deletions(-) diff --git a/OpenSim/Capabilities/Handlers/FetchInventory/FetchInvDescHandler.cs b/OpenSim/Capabilities/Handlers/FetchInventory/FetchInvDescHandler.cs index a9cb134143..a476e91b5a 100644 --- a/OpenSim/Capabilities/Handlers/FetchInventory/FetchInvDescHandler.cs +++ b/OpenSim/Capabilities/Handlers/FetchInventory/FetchInvDescHandler.cs @@ -38,10 +38,10 @@ using OpenMetaverse; using OpenMetaverse.StructuredData; using OpenSim.Framework; using OpenSim.Framework.Capabilities; -using OpenSim.Region.Framework.Interfaces; using OpenSim.Framework.Servers.HttpServer; using OpenSim.Services.Interfaces; -using Caps = OpenSim.Framework.Capabilities.Caps; +using OSDMap = OpenMetaverse.StructuredData.OSDMap; +using OSDArray = OpenMetaverse.StructuredData.OSDArray; namespace OpenSim.Capabilities.Handlers { @@ -50,10 +50,10 @@ namespace OpenSim.Capabilities.Handlers private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + private static byte[] EmptyResponse = Util.UTF8NBGetbytes("folders"); private IInventoryService m_InventoryService; private ILibraryService m_LibraryService; private IScene m_Scene; -// private object m_fetchLock = new Object(); public FetchInvDescHandler(IInventoryService invService, ILibraryService libService, IScene s) { @@ -62,62 +62,66 @@ namespace OpenSim.Capabilities.Handlers m_Scene = s; } - public string FetchInventoryDescendentsRequest(string request, string path, string param, IOSHttpRequest httpRequest, IOSHttpResponse httpResponse) - { - using (MemoryStream ms = new MemoryStream(Utils.StringToBytes(request), false)) - { - return FetchInventoryDescendentsRequest(ms, path, param, httpRequest, httpResponse); - } - } - - public string FetchInventoryDescendentsRequest(Stream request, string path, string param, IOSHttpRequest httpRequest, IOSHttpResponse httpResponse) + public void FetchInventoryDescendentsRequest(IOSHttpRequest httpRequest, IOSHttpResponse httpResponse) { //m_log.DebugFormat("[XXX]: FetchInventoryDescendentsRequest in {0}, {1}", (m_Scene == null) ? "none" : m_Scene.Name, request); - ArrayList foldersrequested = null; + List folders = null; try { - Hashtable hash = (Hashtable)LLSD.LLSDDeserialize(request); - foldersrequested = (ArrayList)hash["folders"]; - hash = null; + OSDArray foldersrequested = null; + OSD tmp = OSDParser.DeserializeLLSDXml(httpRequest.InputStream); + OSDMap map = (OSDMap)tmp; + if(map.TryGetValue("folders", out tmp) && tmp is OSDArray) + foldersrequested = tmp as OSDArray; + + if (foldersrequested == null || foldersrequested.Count == 0) + { + httpResponse.RawBuffer = EmptyResponse; + return; + } + + folders = new List(foldersrequested.Count); + for (int i = 0; i < foldersrequested.Count; i++) + { + LLSDFetchInventoryDescendents llsdRequest = new LLSDFetchInventoryDescendents(); + try + { + OSDMap mfolder = (OSDMap)foldersrequested[i]; + llsdRequest.folder_id = mfolder["folder_id"].AsUUID(); + llsdRequest.owner_id = mfolder["owner_id"].AsUUID(); + llsdRequest.sort_order = mfolder["sort_order"].AsInteger(); + llsdRequest.fetch_folders = mfolder["fetch_folders"].AsBoolean(); + llsdRequest.fetch_items = mfolder["fetch_items"].AsBoolean(); + } + catch (Exception e) + { + m_log.Debug("[WEB FETCH INV DESC HANDLER]: caught exception doing OSD deserialize" + e.Message); + continue; + } + folders.Add(llsdRequest); + } + foldersrequested = null; + tmp = null; } catch (Exception e) { - m_log.ErrorFormat("[FETCH INV DESC]: fail parsing request: '{0}'; path: '{1}'; exception: '{2}'", request, path, e.Message); - foldersrequested = null; + m_log.ErrorFormat("[FETCH INV DESC]: fail parsing request: {0}", e.Message); + httpResponse.RawBuffer = EmptyResponse; + return; } - if(foldersrequested == null || foldersrequested.Count == 0) - return "folders"; - - List folders = new List(); - for (int i = 0; i < foldersrequested.Count; i++) + if (folders == null || folders.Count == 0) { - Hashtable inventoryhash = (Hashtable)foldersrequested[i]; - LLSDFetchInventoryDescendents llsdRequest = new LLSDFetchInventoryDescendents(); - - try - { - LLSDHelpers.DeserialiseOSDMap(inventoryhash, llsdRequest); - } - catch (Exception e) - { - m_log.Debug("[WEB FETCH INV DESC HANDLER]: caught exception doing OSD deserialize" + e.Message); - continue; - } - - folders.Add(llsdRequest); + httpResponse.RawBuffer = EmptyResponse; + return; } - foldersrequested.Clear(); - - if(folders.Count == 0) - return "folders"; - List bad_folders = new List(); int total_folders = 0; int total_items = 0; + List invcollSet = Fetch(folders, bad_folders, ref total_folders, ref total_items); //m_log.DebugFormat("[XXX]: Got {0} folders from a request of {1}", invcollSet.Count, folders.Count); @@ -133,57 +137,57 @@ namespace OpenSim.Capabilities.Handlers StringBuilder lastresponse = new StringBuilder(mem); lastresponse.Append(""); - if(invcollSetCount > 0) + if (invcollSetCount > 0) { lastresponse.Append("folders"); int i = 0; InventoryCollection thiscoll; - for(i = 0; i < invcollSetCount; i++) + for (i = 0; i < invcollSetCount; i++) { thiscoll = invcollSet[i]; invcollSet[i] = null; LLSDxmlEncode.AddMap(lastresponse); - LLSDxmlEncode.AddElem("agent_id", thiscoll.OwnerID, lastresponse); - LLSDxmlEncode.AddElem("descendents", thiscoll.Descendents, lastresponse); - LLSDxmlEncode.AddElem("folder_id", thiscoll.FolderID, lastresponse); + LLSDxmlEncode.AddElem("agent_id", thiscoll.OwnerID, lastresponse); + LLSDxmlEncode.AddElem("descendents", thiscoll.Descendents, lastresponse); + LLSDxmlEncode.AddElem("folder_id", thiscoll.FolderID, lastresponse); - if(thiscoll.Folders == null || thiscoll.Folders.Count == 0) - LLSDxmlEncode.AddEmptyArray("categories", lastresponse); - else + if (thiscoll.Folders == null || thiscoll.Folders.Count == 0) + LLSDxmlEncode.AddEmptyArray("categories", lastresponse); + else + { + LLSDxmlEncode.AddArray("categories", lastresponse); + foreach (InventoryFolderBase invFolder in thiscoll.Folders) { - LLSDxmlEncode.AddArray("categories", lastresponse); - foreach (InventoryFolderBase invFolder in thiscoll.Folders) - { - LLSDxmlEncode.AddMap(lastresponse); + LLSDxmlEncode.AddMap(lastresponse); - LLSDxmlEncode.AddElem("folder_id", invFolder.ID, lastresponse); - LLSDxmlEncode.AddElem("parent_id", invFolder.ParentID, lastresponse); - LLSDxmlEncode.AddElem("name", invFolder.Name, lastresponse); - LLSDxmlEncode.AddElem("type", invFolder.Type, lastresponse); - LLSDxmlEncode.AddElem("preferred_type", (int)-1, lastresponse); - LLSDxmlEncode.AddElem("version", invFolder.Version, lastresponse); + LLSDxmlEncode.AddElem("folder_id", invFolder.ID, lastresponse); + LLSDxmlEncode.AddElem("parent_id", invFolder.ParentID, lastresponse); + LLSDxmlEncode.AddElem("name", invFolder.Name, lastresponse); + LLSDxmlEncode.AddElem("type", invFolder.Type, lastresponse); + LLSDxmlEncode.AddElem("preferred_type", (int)-1, lastresponse); + LLSDxmlEncode.AddElem("version", invFolder.Version, lastresponse); - LLSDxmlEncode.AddEndMap(lastresponse); - } - LLSDxmlEncode.AddEndArray(lastresponse); + LLSDxmlEncode.AddEndMap(lastresponse); + } + LLSDxmlEncode.AddEndArray(lastresponse); + } + + if (thiscoll.Items == null || thiscoll.Items.Count == 0) + LLSDxmlEncode.AddEmptyArray("items", lastresponse); + else + { + LLSDxmlEncode.AddArray("items", lastresponse); + foreach (InventoryItemBase invItem in thiscoll.Items) + { + invItem.ToLLSDxml(lastresponse); } - if(thiscoll.Items == null || thiscoll.Items.Count == 0) - LLSDxmlEncode.AddEmptyArray("items", lastresponse); - else - { - LLSDxmlEncode.AddArray("items", lastresponse); - foreach (InventoryItemBase invItem in thiscoll.Items) - { - invItem.ToLLSDxml(lastresponse); - } + LLSDxmlEncode.AddEndArray(lastresponse); + } - LLSDxmlEncode.AddEndArray(lastresponse); - } - - LLSDxmlEncode.AddElem("owner_id", thiscoll.OwnerID, lastresponse); - LLSDxmlEncode.AddElem("version", thiscoll.Version, lastresponse); + LLSDxmlEncode.AddElem("owner_id", thiscoll.OwnerID, lastresponse); + LLSDxmlEncode.AddElem("version", thiscoll.Version, lastresponse); LLSDxmlEncode.AddEndMap(lastresponse); invcollSet[i] = null; @@ -197,7 +201,7 @@ namespace OpenSim.Capabilities.Handlers } //m_log.DebugFormat("[WEB FETCH INV DESC HANDLER]: Bad folders {0}", string.Join(", ", bad_folders)); - if(bad_folders.Count > 0) + if (bad_folders.Count > 0) { lastresponse.Append("bad_folders"); foreach (UUID bad in bad_folders) @@ -210,7 +214,7 @@ namespace OpenSim.Capabilities.Handlers } lastresponse.Append(""); - return lastresponse.ToString(); + httpResponse.RawBuffer = Util.UTF8NBGetbytes(lastresponse.ToString()); } private void AddLibraryFolders(List libFolders, List result, ref int total_folders, ref int total_items) @@ -400,7 +404,7 @@ namespace OpenSim.Capabilities.Handlers if (linked != null) { - List linkedItems = new List(); + List linkedItems = new List(linked.Length); // check for broken foreach (InventoryItemBase linkedItem in linked) { diff --git a/OpenSim/Capabilities/Handlers/FetchInventory/FetchInvDescServerConnector.cs b/OpenSim/Capabilities/Handlers/FetchInventory/FetchInvDescServerConnector.cs index 9dcfaa48f1..f537bfb9f4 100644 --- a/OpenSim/Capabilities/Handlers/FetchInventory/FetchInvDescServerConnector.cs +++ b/OpenSim/Capabilities/Handlers/FetchInventory/FetchInvDescServerConnector.cs @@ -68,15 +68,9 @@ namespace OpenSim.Capabilities.Handlers ServerUtils.LoadPlugin(libService, args); FetchInvDescHandler webFetchHandler = new FetchInvDescHandler(m_InventoryService, m_LibraryService, null); - IRequestHandler reqHandler - = new RestStreamHandler( - "POST", - "/CAPS/WebFetchInvDesc/" /*+ UUID.Random()*/, - webFetchHandler.FetchInventoryDescendentsRequest, - "FetchInvDescendents", - null); - server.AddStreamHandler(reqHandler); + ISimpleStreamHandler reqHandler + = new SimpleStreamHandler( "/CAPS/WebFetchInvDesc/", webFetchHandler.FetchInventoryDescendentsRequest); + server.AddSimpleStreamHandler(reqHandler); } - } } diff --git a/OpenSim/Capabilities/Handlers/FetchInventory/Tests/FetchInventoryDescendents2HandlerTests.cs b/OpenSim/Capabilities/Handlers/FetchInventory/Tests/FetchInventoryDescendents2HandlerTests.cs index e950d2a3a1..b8ccf4b202 100644 --- a/OpenSim/Capabilities/Handlers/FetchInventory/Tests/FetchInventoryDescendents2HandlerTests.cs +++ b/OpenSim/Capabilities/Handlers/FetchInventory/Tests/FetchInventoryDescendents2HandlerTests.cs @@ -28,7 +28,7 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net; +using System.IO; using System.Text.RegularExpressions; using log4net; using log4net.Config; @@ -127,6 +127,18 @@ namespace OpenSim.Capabilities.Handlers.FetchInventory.Tests Console.WriteLine("Number of descendents: " + m_rootDescendents); } + private string dorequest(FetchInvDescHandler handler, string request) + { + TestOSHttpRequest req = new TestOSHttpRequest(); + TestOSHttpResponse resp = new TestOSHttpResponse(); + using (MemoryStream ms = new MemoryStream(Utils.StringToBytes(request), false)) + { + req.InputStream = ms; + handler.FetchInventoryDescendentsRequest(req, resp); + } + return Util.UTF8.GetString(resp.RawBuffer); + } + [Test] public void Test_001_SimpleFolder() { @@ -135,8 +147,6 @@ namespace OpenSim.Capabilities.Handlers.FetchInventory.Tests Init(); FetchInvDescHandler handler = new FetchInvDescHandler(m_scene.InventoryService, null, m_scene); - TestOSHttpRequest req = new TestOSHttpRequest(); - TestOSHttpResponse resp = new TestOSHttpResponse(); string request = "foldersfetch_folders1fetch_items1folder_id"; request += m_rootFolderID; @@ -144,7 +154,7 @@ namespace OpenSim.Capabilities.Handlers.FetchInventory.Tests request += m_userID.ToString(); request += "sort_order1"; - string llsdresponse = handler.FetchInventoryDescendentsRequest(request, "/FETCH", string.Empty, req, resp); + string llsdresponse = dorequest(handler, request); Assert.That(llsdresponse != null, Is.True, "Incorrect null response"); Assert.That(llsdresponse != string.Empty, Is.True, "Incorrect empty response"); @@ -161,8 +171,6 @@ namespace OpenSim.Capabilities.Handlers.FetchInventory.Tests TestHelpers.InMethod(); FetchInvDescHandler handler = new FetchInvDescHandler(m_scene.InventoryService, null, m_scene); - TestOSHttpRequest req = new TestOSHttpRequest(); - TestOSHttpResponse resp = new TestOSHttpResponse(); string request = "folders"; request += "fetch_folders1fetch_items1folder_id"; @@ -173,7 +181,7 @@ namespace OpenSim.Capabilities.Handlers.FetchInventory.Tests request += "owner_id00000000-0000-0000-0000-000000000001sort_order1"; request += ""; - string llsdresponse = handler.FetchInventoryDescendentsRequest(request, "/FETCH", string.Empty, req, resp); + string llsdresponse = dorequest(handler, request); Console.WriteLine(llsdresponse); string descendents = "descendents" + m_rootDescendents + ""; @@ -191,14 +199,12 @@ namespace OpenSim.Capabilities.Handlers.FetchInventory.Tests TestHelpers.InMethod(); FetchInvDescHandler handler = new FetchInvDescHandler(m_scene.InventoryService, null, m_scene); - TestOSHttpRequest req = new TestOSHttpRequest(); - TestOSHttpResponse resp = new TestOSHttpResponse(); string request = "foldersfetch_folders1fetch_items1folder_id"; request += "f0000000-0000-0000-0000-00000000000f"; request += "owner_id00000000-0000-0000-0000-000000000001sort_order1"; - string llsdresponse = handler.FetchInventoryDescendentsRequest(request, "/FETCH", string.Empty, req, resp); + string llsdresponse = dorequest(handler, request); Console.WriteLine(llsdresponse); string descendents = "descendents2"; @@ -235,8 +241,6 @@ namespace OpenSim.Capabilities.Handlers.FetchInventory.Tests TestHelpers.InMethod(); FetchInvDescHandler handler = new FetchInvDescHandler(m_scene.InventoryService, null, m_scene); - TestOSHttpRequest req = new TestOSHttpRequest(); - TestOSHttpResponse resp = new TestOSHttpResponse(); string request = "folders"; request += "fetch_folders1fetch_items1folder_id"; @@ -253,7 +257,7 @@ namespace OpenSim.Capabilities.Handlers.FetchInventory.Tests request += "owner_id00000000-0000-0000-0000-000000000000sort_order1"; request += ""; - string llsdresponse = handler.FetchInventoryDescendentsRequest(request, "/FETCH", string.Empty, req, resp); + string llsdresponse = dorequest(handler, request); Console.WriteLine(llsdresponse); string root_folder = "folder_id" + m_rootFolderID + ""; @@ -276,14 +280,12 @@ namespace OpenSim.Capabilities.Handlers.FetchInventory.Tests Init(); FetchInvDescHandler handler = new FetchInvDescHandler(m_scene.InventoryService, null, m_scene); - TestOSHttpRequest req = new TestOSHttpRequest(); - TestOSHttpResponse resp = new TestOSHttpResponse(); string request = "foldersfetch_folders1fetch_items1folder_id"; request += UUID.Zero; request += "owner_id00000000-0000-0000-0000-000000000000sort_order1"; - string llsdresponse = handler.FetchInventoryDescendentsRequest(request, "/FETCH", string.Empty, req, resp); + string llsdresponse = dorequest(handler, request); Assert.That(llsdresponse != null, Is.True, "Incorrect null response"); Assert.That(llsdresponse != string.Empty, Is.True, "Incorrect empty response"); diff --git a/OpenSim/Region/ClientStack/Linden/Caps/WebFetchInvDescModule.cs b/OpenSim/Region/ClientStack/Linden/Caps/WebFetchInvDescModule.cs index 1580eae7e2..15b65af1da 100755 --- a/OpenSim/Region/ClientStack/Linden/Caps/WebFetchInvDescModule.cs +++ b/OpenSim/Region/ClientStack/Linden/Caps/WebFetchInvDescModule.cs @@ -330,17 +330,8 @@ namespace OpenSim.Region.ClientStack.Linden } } - Hashtable response = new Hashtable(); - - response["int_response_code"] = 200; - response["content_type"] = "text/plain"; - - response["bin_response_data"] = Encoding.UTF8.GetBytes( - m_webFetchHandler.FetchInventoryDescendentsRequest( - requestinfo.request.InputStream, - String.Empty, String.Empty, null, null) - ); - + OSHttpResponse osresponse = new OSHttpResponse(requestinfo.request); + m_webFetchHandler.FetchInventoryDescendentsRequest(requestinfo.request, osresponse); requestinfo.request.InputStream.Dispose(); lock (responses) @@ -350,16 +341,16 @@ namespace OpenSim.Region.ClientStack.Linden if(dropedResponses.Contains(requestID)) { dropedResponses.Remove(requestID); - WebFetchInvDescModule.ProcessedRequestsCount++; + ProcessedRequestsCount++; return; } } - if (responses.ContainsKey(requestID)) - m_log.WarnFormat("[FETCH INVENTORY DESCENDENTS2 MODULE]: Caught in the act of loosing responses! Please report this on mantis #7054"); + Hashtable response = new Hashtable(); + response["h"] = osresponse; responses[requestID] = response; } - WebFetchInvDescModule.ProcessedRequestsCount++; + ProcessedRequestsCount++; } }