From b3893b9dc5419a775a2342fd52016a645cd34a45 Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Fri, 23 Oct 2020 19:43:31 +0100 Subject: [PATCH] make AssetServicesConnector do HG work (HGAssetServicesConnector should no not be in used now); change HGAssetBroker to be a extended derivative of it. WARNING TESTgit push origin master possible still very broken --- OpenSim/Framework/GridInfo.cs | 7 +- OpenSim/Framework/Util.cs | 49 +- .../InventoryAccess/HGAssetMapper.cs | 3 +- .../Asset/AssetServiceInConnectorModule.cs | 2 +- .../HypergridServiceInConnectorModule.cs | 8 +- OpenSim/Services/AssetService/AssetService.cs | 4 +- .../Asset/AssetServicesConnector.cs | 738 +++++++++++++----- .../Asset/HGAssetServiceConnector.cs | 60 +- 8 files changed, 616 insertions(+), 255 deletions(-) diff --git a/OpenSim/Framework/GridInfo.cs b/OpenSim/Framework/GridInfo.cs index d51acb606c..5e0f80417a 100644 --- a/OpenSim/Framework/GridInfo.cs +++ b/OpenSim/Framework/GridInfo.cs @@ -50,8 +50,9 @@ namespace OpenSim.Framework public int Port; public IPAddress IP; public readonly string Host; - public readonly string URI; + public readonly string URL; public readonly string Path; + public readonly string URI; public OSHTTPURI(string uri, bool withDNSResolve = false) { @@ -60,6 +61,7 @@ namespace OpenSim.Framework IP = null; Host = string.Empty; URI = string.Empty; + URL = string.Empty; Path = string.Empty; if (string.IsNullOrEmpty(uri)) @@ -80,7 +82,8 @@ namespace OpenSim.Framework if (Path[Path.Length - 1] == '/') Path = Path.Substring(0, Path.Length - 1); - URI = m_checkuri.Scheme + "://" + Host + ":" + Port + Path; + URL = m_checkuri.Scheme + "://" + Host + ":" + Port; + URI = URL + Path; if (withDNSResolve) { diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs index 99c3405355..8e99f720de 100644 --- a/OpenSim/Framework/Util.cs +++ b/OpenSim/Framework/Util.cs @@ -1485,30 +1485,41 @@ namespace OpenSim.Framework /// The URL: http://grid.example.com:8002 /// The asset ID: 00000000-0000-0000-0000-000000000000. Returned even if 'id' isn't foreign. /// True: this is a foreign asset ID; False: it isn't - public static bool ParseForeignAssetID(string id, out string url, out string assetID) + public static int ParseForeignAssetID(string id, out string url, out string assetID) { - url = String.Empty; - assetID = String.Empty; + url = string.Empty; + assetID = string.Empty; - if (UUID.TryParse(id, out UUID uuid)) + if (id.Length == 0) + return -1; + + if(id[0] != 'h' && id[0] != 'H') { - assetID = uuid.ToString(); - return false; + if (UUID.TryParse(id, out UUID luuid)) + { + assetID = id; + return 0; + } + return -1; } - if ((id.Length == 0) || (id[0] != 'h' && id[0] != 'H')) - return false; - - if (!Uri.TryCreate(id, UriKind.Absolute, out Uri assetUri) || assetUri.Scheme != Uri.UriSchemeHttp) - return false; - - url = "http://" + assetUri.Authority; - assetID = assetUri.LocalPath.Trim(new char[] { '/' }); - - if (!UUID.TryParse(assetID, out uuid)) - return false; - - return true; + OSHTTPURI uri = new OSHTTPURI(id, true); + if(uri.IsResolvedHost) + { + url = uri.URL; + string tmp = uri.Path; + if(tmp.Length < 36) + return -3; + if(tmp[0] =='/') + tmp = tmp.Substring(1); + if (UUID.TryParse(tmp, out UUID uuid)) + { + assetID = tmp; + return 1; + } + return -1; + } + return -2; } /// diff --git a/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs b/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs index dfd40f1286..cebbb54543 100644 --- a/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs +++ b/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs @@ -147,7 +147,7 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess asset1.Data = asset.Data; string id = m_scene.AssetService.Store(asset1); - if (String.IsNullOrEmpty(id)) + if (string.IsNullOrEmpty(id)) { if (verbose) m_log.DebugFormat("[HG ASSET MAPPER]: Asset server {0} did not accept {1}", url, asset.ID); @@ -169,7 +169,6 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess to.Name = from.Name; to.Temporary = from.Temporary; to.Type = from.Type; - } private void AdjustIdentifiers(AssetMetadata meta) diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsIn/Asset/AssetServiceInConnectorModule.cs b/OpenSim/Region/CoreModules/ServiceConnectorsIn/Asset/AssetServiceInConnectorModule.cs index 2c882e686e..034185caa0 100644 --- a/OpenSim/Region/CoreModules/ServiceConnectorsIn/Asset/AssetServiceInConnectorModule.cs +++ b/OpenSim/Region/CoreModules/ServiceConnectorsIn/Asset/AssetServiceInConnectorModule.cs @@ -96,7 +96,7 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsIn.Asset m_log.Info("[HGAssetService]: Starting..."); - Object[] args = new Object[] { m_Config, MainServer.Instance, "HGAssetService" }; + object[] args = new object[] { m_Config, MainServer.Instance, "HGAssetService" }; ServerUtils.LoadPlugin("OpenSim.Server.Handlers.dll:AssetServiceConnector", args); } diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsIn/Hypergrid/HypergridServiceInConnectorModule.cs b/OpenSim/Region/CoreModules/ServiceConnectorsIn/Hypergrid/HypergridServiceInConnectorModule.cs index 4f87595b60..9f76be0af8 100644 --- a/OpenSim/Region/CoreModules/ServiceConnectorsIn/Hypergrid/HypergridServiceInConnectorModule.cs +++ b/OpenSim/Region/CoreModules/ServiceConnectorsIn/Hypergrid/HypergridServiceInConnectorModule.cs @@ -51,7 +51,7 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsIn.Hypergrid private IConfigSource m_Config; private bool m_Registered = false; - private string m_LocalServiceDll = String.Empty; + private string m_LocalServiceDll = string.Empty; private GatekeeperServiceInConnector m_HypergridHandler; private UserAgentServerConnector m_UASHandler; @@ -71,13 +71,11 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsIn.Hypergrid if (fconfig != null) { m_LocalServiceDll = fconfig.GetString("LocalServiceModule", m_LocalServiceDll); - if (m_LocalServiceDll == String.Empty) + if (m_LocalServiceDll == string.Empty) m_log.WarnFormat("[HGGRID IN CONNECTOR]: Friends LocalServiceModule config missing"); } } - } - } public void PostInitialise() @@ -123,7 +121,7 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsIn.Hypergrid ISimulationService simService = scene.RequestModuleInterface(); IFriendsSimConnector friendsConn = scene.RequestModuleInterface(); - Object[] args = new Object[] { m_Config }; + object[] args = new object[] { m_Config }; // IFriendsService friendsService = ServerUtils.LoadPlugin(m_LocalServiceDll, args) ServerUtils.LoadPlugin(m_LocalServiceDll, args); diff --git a/OpenSim/Services/AssetService/AssetService.cs b/OpenSim/Services/AssetService/AssetService.cs index ee2e56871f..3930a68ce0 100644 --- a/OpenSim/Services/AssetService/AssetService.cs +++ b/OpenSim/Services/AssetService/AssetService.cs @@ -40,9 +40,7 @@ namespace OpenSim.Services.AssetService { public class AssetService : AssetServiceBase, IAssetService { - private static readonly ILog m_log = - LogManager.GetLogger( - MethodBase.GetCurrentMethod().DeclaringType); + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); protected static AssetService m_RootInstance; diff --git a/OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs b/OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs index 21a3a3495d..68852a1347 100644 --- a/OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs +++ b/OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs @@ -43,29 +43,30 @@ namespace OpenSim.Services.Connectors { public class AssetServicesConnector : BaseServiceConnector, IAssetService { - private static readonly ILog m_log = - LogManager.GetLogger( - MethodBase.GetCurrentMethod().DeclaringType); + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); const int MAXSENDRETRIESLEN = 30; + public object ConnectorLock = new object(); + private string m_ServerURI = String.Empty; private IAssetCache m_Cache = null; private int m_retryCounter; private bool m_inRetries; - private List[] m_sendRetries = new List[MAXSENDRETRIESLEN]; + private List[] m_sendRetries = new List[MAXSENDRETRIESLEN]; + private List[] m_sendCachedRetries = new List[MAXSENDRETRIESLEN]; private System.Timers.Timer m_retryTimer; - private int m_maxAssetRequestConcurrency = 30; + + private int m_maxAssetRequestConcurrency = 8; private delegate void AssetRetrievedEx(AssetBase asset); // Keeps track of concurrent requests for the same asset, so that it's only loaded once. // Maps: Asset ID -> Handlers which will be called when the asset has been loaded -// private Dictionary m_AssetHandlers = new Dictionary(); private Dictionary> m_AssetHandlers = new Dictionary>(); - private Dictionary m_UriMap = new Dictionary(); + private Dictionary m_UriMap; private Thread[] m_fetchThreads; @@ -84,8 +85,7 @@ namespace OpenSim.Services.Connectors m_ServerURI = serverURI.TrimEnd('/'); } - public AssetServicesConnector(IConfigSource source) - : base(source, "AssetService") + public AssetServicesConnector(IConfigSource source) { Initialise(source); } @@ -103,64 +103,78 @@ namespace OpenSim.Services.Connectors throw new Exception("Asset connector init error"); } - m_ServerURI = assetConfig.GetString("AssetServerURI", String.Empty); - if (m_ServerURI == String.Empty) + m_ServerURI = assetConfig.GetString("AssetServerURI", string.Empty); + if (string.IsNullOrEmpty(m_ServerURI)) { - m_log.Error("[ASSET CONNECTOR]: No Server URI named in section AssetService"); + IConfig netConfig = source.Configs["Network"]; + m_ServerURI = netConfig.GetString("asset_server_url", string.Empty); + } + if (string.IsNullOrEmpty(m_ServerURI)) + { + m_log.Error("[ASSET CONNECTOR]: AssetServerURI not defined in section AssetService"); throw new Exception("Asset connector init error"); } + OSHHTPHost m_GridAssetsURL = new OSHHTPHost(m_ServerURI, true); + if(!m_GridAssetsURL.IsResolvedHost) + { + m_log.Error("[ASSET CONNECTOR]: Could not parse or resolve AssetServerURI"); + throw new Exception("Asset connector init error"); + } + + m_ServerURI = m_GridAssetsURL.URI; + bool usemaps = assetConfig.GetBoolean("AssetServerIsCluster", false); + + if(usemaps) + { + m_UriMap = new Dictionary(); + for (int i = 0; i < 256; i++) + { + string prefix = i.ToString("x2"); + string groupHost = assetConfig.GetString("AssetServerHost_" + prefix, string.Empty); + if(string.IsNullOrEmpty(groupHost)) + m_UriMap[prefix] = m_ServerURI; + else + { + OSHHTPHost other = new OSHHTPHost(groupHost, true); + if(!other.IsResolvedHost) + { + m_log.Error("[ASSET CONNECTOR]: Could not parse or resolve AssetServerHost_" + prefix); + throw new Exception("Asset connector init error"); + } + m_UriMap[prefix] = other.URI; + } + } + } + else + m_UriMap = null; + m_retryTimer = new System.Timers.Timer(); m_retryTimer.Elapsed += new ElapsedEventHandler(retryCheck); m_retryTimer.AutoReset = true; m_retryTimer.Interval = 60000; - Uri serverUri = new Uri(m_ServerURI); + m_fetchThreads = new Thread[3]; - string groupHost = serverUri.Host; + Initialise(source, "AssetService"); - for (int i = 0 ; i < 256 ; i++) + for (int i = 0 ; i < m_fetchThreads.Length; i++) { - string prefix = i.ToString("x2"); - groupHost = assetConfig.GetString("AssetServerHost_"+prefix, groupHost); - - m_UriMap[prefix] = groupHost; - //m_log.DebugFormat("[ASSET]: Using {0} for prefix {1}", groupHost, prefix); - } - - m_fetchThreads = new Thread[2]; - - for (int i = 0 ; i < 2 ; i++) - { - m_fetchThreads[i] = WorkManager.StartThread(AssetRequestProcessor, String.Format("GetAssetsWorker{0}", i)); + m_fetchThreads[i] = WorkManager.StartThread(AssetRequestProcessor, string.Format("GetAssetsWorker{0}", i)); } } private string MapServer(string id) { - if (m_UriMap.Count == 0) + if (m_UriMap == null) return m_ServerURI; - UriBuilder serverUri = new UriBuilder(m_ServerURI); - string prefix = id.Substring(0, 2).ToLower(); - string host; + if (m_UriMap.TryGetValue(prefix, out string host)) + return host; - // HG URLs will not be valid UUIDS - if (m_UriMap.ContainsKey(prefix)) - host = m_UriMap[prefix]; - else - host = m_UriMap["00"]; - - serverUri.Host = host; - - // m_log.DebugFormat("[ASSET]: Using {0} for host name for prefix {1}", host, prefix); - - string ret = serverUri.Uri.AbsoluteUri; - if (ret.EndsWith("/")) - ret = ret.Substring(0, ret.Length - 1); - return ret; + return m_UriMap["00"]; } protected void retryCheck(object source, ElapsedEventArgs e) @@ -179,48 +193,94 @@ namespace OpenSim.Services.Connectors int inUse = 0; int nextlevel; int timefactor; - List retrylist; - // we need to go down - for(int i = MAXSENDRETRIESLEN - 1; i >= 0; i--) + if(m_Cache == null) { - lock(m_sendRetries) - retrylist = m_sendRetries[i]; - - if(retrylist == null) - continue; - - inUse++; - nextlevel = i + 1; - - //We exponentially fall back on frequency until we reach one attempt per hour - //The net result is that we end up in the queue for roughly 24 hours.. - //24 hours worth of assets could be a lot, so the hope is that the region admin - //will have gotten the asset connector back online quickly! - if(i == 0) - timefactor = 1; - else + List retrylist; + // we need to go down + for(int i = MAXSENDRETRIESLEN - 1; i >= 0; i--) { - timefactor = 1 << nextlevel; - if (timefactor > 60) - timefactor = 60; + lock(m_sendRetries) + retrylist = m_sendRetries[i]; + + if(retrylist == null) + continue; + + inUse++; + nextlevel = i + 1; + + //We exponentially fall back on frequency until we reach one attempt per hour + //The net result is that we end up in the queue for roughly 24 hours.. + //24 hours worth of assets could be a lot, so the hope is that the region admin + //will have gotten the asset connector back online quickly! + if(i == 0) + timefactor = 1; + else + { + timefactor = 1 << nextlevel; + if (timefactor > 60) + timefactor = 60; + } + + if(m_retryCounter < timefactor) + continue; // to update inUse; + + if (m_retryCounter % timefactor != 0) + continue; + + // a list to retry + lock(m_sendRetries) + m_sendRetries[i] = null; + + // we are the only ones with a copy of this retrylist now + foreach(AssetBase ass in retrylist) + retryStore(ass, nextlevel); } + } + else + { + List retrylist; + // we need to go down + for (int i = MAXSENDRETRIESLEN - 1; i >= 0; i--) + { + lock (m_sendRetries) + retrylist = m_sendCachedRetries[i]; - if(m_retryCounter < timefactor) - continue; // to update inUse; + if (retrylist == null) + continue; - if (m_retryCounter % timefactor != 0) - continue; + inUse++; + nextlevel = i + 1; - // a list to retry - lock(m_sendRetries) - m_sendRetries[i] = null; + //We exponentially fall back on frequency until we reach one attempt per hour + //The net result is that we end up in the queue for roughly 24 hours.. + //24 hours worth of assets could be a lot, so the hope is that the region admin + //will have gotten the asset connector back online quickly! + if (i == 0) + timefactor = 1; + else + { + timefactor = 1 << nextlevel; + if (timefactor > 60) + timefactor = 60; + } - // we are the only ones with a copy of this retrylist now - foreach(AssetBase ass in retrylist) - retryStore(ass, nextlevel); + if (m_retryCounter < timefactor) + continue; // to update inUse; + + if (m_retryCounter % timefactor != 0) + continue; + + // a list to retry + lock (m_sendRetries) + m_sendCachedRetries[i] = null; + + // we are the only ones with a copy of this retrylist now + foreach (string id in retrylist) + retryCachedStore(id, nextlevel); + } } - lock(m_sendRetries) + lock (m_sendRetries) { if(inUse == 0 ) m_retryTimer.Stop(); @@ -229,12 +289,23 @@ namespace OpenSim.Services.Connectors } } - protected void SetCache(IAssetCache cache) + public void SetCache(IAssetCache cache) { m_Cache = cache; } - public AssetBase Get(string id) + public AssetBase GetCached(string id) + { + AssetBase asset = null; + if (m_Cache != null) + { + m_Cache.Get(id, out asset); + } + + return asset; + } + + public virtual AssetBase Get(string id) { AssetBase asset = null; if (m_Cache != null) @@ -246,6 +317,7 @@ namespace OpenSim.Services.Connectors if (asset == null || asset.Data == null || asset.Data.Length == 0) { string uri = MapServer(id) + "/assets/" + id; + asset = SynchronousRestObjectRequester.MakeRequest("GET", uri, 0, m_Auth); if (m_Cache != null) { @@ -258,20 +330,39 @@ namespace OpenSim.Services.Connectors return asset; } - public AssetBase GetCached(string id) + public AssetBase GetForeign(string id) { -// m_log.DebugFormat("[ASSET SERVICE CONNECTOR]: Cache request for {0}", id); + int type = Util.ParseForeignAssetID(id, out string uri, out string uuidstr); + if (type < 0) + return null; AssetBase asset = null; if (m_Cache != null) { - m_Cache.Get(id, out asset); + if (!m_Cache.Get(uuidstr, out asset)) + return null; } + if (asset == null || asset.Data == null || asset.Data.Length == 0) + { + if (type == 0) + uri = MapServer(uuidstr) + "/assets/" + uuidstr; + else + uri = uri + "/assets/" + uuidstr; + + asset = SynchronousRestObjectRequester.MakeRequest("GET", uri, 0, m_Auth); + if (m_Cache != null) + { + if (asset != null) + m_Cache.Cache(asset); + else + m_Cache.CacheNegative(id); + } + } return asset; } - public AssetMetadata GetMetadata(string id) + public virtual AssetMetadata GetMetadata(string id) { if (m_Cache != null) { @@ -289,40 +380,87 @@ namespace OpenSim.Services.Connectors return asset; } - public byte[] GetData(string id) + public AssetMetadata GetForeignMetadata(string id) { + int type = Util.ParseForeignAssetID(id, out string uri, out string uuidstr); + if (type < 0) + return null; + if (m_Cache != null) { AssetBase fullAsset; - if (!m_Cache.Get(id, out fullAsset)) + if (!m_Cache.Get(uuidstr, out fullAsset)) + return null; + + if (fullAsset != null) + return fullAsset.Metadata; + } + + if (type == 0) + uri = MapServer(uuidstr) + "/assets/" + uuidstr + "/metadata"; + else + uri = uri + "/assets/" + uuidstr + "/metadata"; + + AssetMetadata asset = SynchronousRestObjectRequester.MakeRequest("GET", uri, 0, m_Auth); + return asset; + } + + public virtual byte[] GetData(string id) + { + if (m_Cache != null) + { + if (!m_Cache.Get(id, out AssetBase fullAsset)) return null; if (fullAsset != null) return fullAsset.Data; } - using (RestClient rc = new RestClient(MapServer(id))) - { - rc.AddResourcePath("assets"); - rc.AddResourcePath(id); - rc.AddResourcePath("data"); + string uri = MapServer(id); + using (RestClient rc = new RestClient(uri)) + { + rc.AddResourcePath("assets/" + id + "/Data"); rc.RequestMethod = "GET"; - using (Stream s = rc.Request(m_Auth)) + using (MemoryStream s = rc.Request(m_Auth)) { - if (s == null) + if (s == null || s.Length == 0) return null; - - if (s.Length > 0) - { - byte[] ret = new byte[s.Length]; - s.Read(ret, 0, (int)s.Length); - - return ret; - } + return s.ToArray(); } + } + } + + public byte[] GetForeignData(string id) + { + int type = Util.ParseForeignAssetID(id, out string uri, out string uuidstr); + if (type < 0) return null; + + if (m_Cache != null) + { + if (!m_Cache.Get(uuidstr, out AssetBase fullAsset)) + return null; + + if (fullAsset != null) + return fullAsset.Data; + } + + if (type == 0) + uri = MapServer(uuidstr); + + using (RestClient rc = new RestClient(uri)) + { + rc.AddResourcePath("assets/" + id + "/Data"); + rc.RequestMethod = "GET"; + + using (MemoryStream s = rc.Request(m_Auth)) + { + if (s == null || s.Length == 0) + return null; + return s.ToArray(); + } } } @@ -332,8 +470,106 @@ namespace OpenSim.Services.Connectors public string id; } - private BlockingCollection m_requestQueue = new BlockingCollection(); + public virtual bool Get(string id, object sender, AssetRetrieved handler) + { + AssetBase asset = null; + if (m_Cache != null) + { + if (!m_Cache.Get(id, out asset)) + return false; + } + if (asset == null) + { + string uri = MapServer(id) + "/assets/" + id; + + lock (m_AssetHandlers) + { + AssetRetrievedEx handlerEx = new AssetRetrievedEx(delegate (AssetBase _asset) { handler(id, sender, _asset); }); + + List handlers; + if (m_AssetHandlers.TryGetValue(id, out handlers)) + { + // Someone else is already loading this asset. It will notify our handler when done. + handlers.Add(handlerEx); + return true; + } + + handlers = new List(); + handlers.Add(handlerEx); + + m_AssetHandlers.Add(id, handlers); + + QueuedAssetRequest request = new QueuedAssetRequest(); + request.id = id; + request.uri = uri; + m_requestQueue.Add(request); + } + } + else + { + if (asset != null && (asset.Data == null || asset.Data.Length == 0)) + asset = null; + handler(id, sender, asset); + } + + return true; + } + + public bool GetForeign(string id, object sender, AssetRetrieved handler) + { + int type = Util.ParseForeignAssetID(id, out string uri, out string uuidstr); + if (type < 0) + return false; + + AssetBase asset = null; + if (m_Cache != null) + { + if (!m_Cache.Get(uuidstr, out asset)) + return false; + } + + if (asset == null) + { + if (type == 0) + uri = MapServer(uuidstr) + "/assets/" + uuidstr; + else + uri = uri + "/assets/" + uuidstr; + + lock (m_AssetHandlers) + { + AssetRetrievedEx handlerEx = new AssetRetrievedEx(delegate (AssetBase _asset) { handler(id, sender, _asset); }); + + List handlers; + if (m_AssetHandlers.TryGetValue(id, out handlers)) + { + // Someone else is already loading this asset. It will notify our handler when done. + handlers.Add(handlerEx); + return true; + } + + handlers = new List(); + handlers.Add(handlerEx); + + m_AssetHandlers.Add(id, handlers); + + QueuedAssetRequest request = new QueuedAssetRequest(); + request.id = id; + request.uri = uri; + m_requestQueue.Add(request); + } + } + else + { + if (asset != null && (asset.Data == null || asset.Data.Length == 0)) + asset = null; + handler(id, sender, asset); + } + + return true; + } + + private BlockingCollection m_requestQueue = new BlockingCollection(); private void AssetRequestProcessor() { QueuedAssetRequest r; @@ -347,12 +583,11 @@ namespace OpenSim.Services.Connectors } Watchdog.UpdateThread(); - string uri = r.uri; string id = r.id; try { - AssetBase a = SynchronousRestObjectRequester.MakeRequest("GET", uri, 0, 30000, m_Auth); + AssetBase a = SynchronousRestObjectRequester.MakeRequest("GET", r.uri, 0, 30000, m_Auth); if (a != null && m_Cache != null) m_Cache.Cache(a); @@ -381,50 +616,6 @@ namespace OpenSim.Services.Connectors } } - public bool Get(string id, Object sender, AssetRetrieved handler) - { - string uri = MapServer(id) + "/assets/" + id; - - AssetBase asset = null; - if (m_Cache != null) - { - if (!m_Cache.Get(id, out asset)) - return false; - } - - if (asset == null || asset.Data == null || asset.Data.Length == 0) - { - lock (m_AssetHandlers) - { - AssetRetrievedEx handlerEx = new AssetRetrievedEx(delegate(AssetBase _asset) { handler(id, sender, _asset); }); - - List handlers; - if (m_AssetHandlers.TryGetValue(id, out handlers)) - { - // Someone else is already loading this asset. It will notify our handler when done. - handlers.Add(handlerEx); - return true; - } - - handlers = new List(); - handlers.Add(handlerEx); - - m_AssetHandlers.Add(id, handlers); - - QueuedAssetRequest request = new QueuedAssetRequest(); - request.id = id; - request.uri = uri; - m_requestQueue.Add(request); - } - } - else - { - handler(id, sender, asset); - } - - return true; - } - public virtual bool[] AssetsExist(string[] ids) { string uri = m_ServerURI + "/get_assets_exist"; @@ -446,9 +637,80 @@ namespace OpenSim.Services.Connectors return exist; } + private struct AssetAndIndex + { + public string assetID; + public int index; + + public AssetAndIndex(string assetID, int index) + { + this.assetID = assetID; + this.index = index; + } + } + + public bool[] ForeignAssetsExist(string[] ids) + { + bool[] exist = new bool[ids.Length]; + + var url2assets = new Dictionary>(); + + for (int i = 0; i < ids.Length; i++) + { + int ltype = Util.ParseForeignAssetID(ids[i], out string lurl, out string luuidstr); + if (ltype > 0) + { + if (ltype == 0) + lurl = m_ServerURI; + + List < AssetAndIndex > lst; + if (!url2assets.TryGetValue(lurl, out lst)) + { + lst = new List(); + url2assets.Add(lurl, lst); + } + lst.Add(new AssetAndIndex(luuidstr, i)); + } + } + + // Query each of the servers in turn + foreach (KeyValuePair> kvp in url2assets) + { + List curAssets = kvp.Value; + string[] assetIDs = new string[curAssets.Count]; + for(int i = 0; i < assetIDs.Length;++i) + assetIDs[i] = curAssets[i].assetID; + + string uri = kvp.Key + "/get_assets_exist"; + + bool[] curExist = null; + try + { + curExist = SynchronousRestObjectRequester.MakeRequest("POST", uri, assetIDs, m_Auth); + } + catch (Exception) + { + // This is most likely to happen because the server doesn't support this function, + // so just silently return "doesn't exist" for all the assets. + } + + if(curExist != null) + { + int i = 0; + foreach (AssetAndIndex ai in curAssets) + { + exist[ai.index] = curExist[i]; + ++i; + } + } + } + + return exist; + } + string stringUUIDZero = UUID.Zero.ToString(); - public string Store(AssetBase asset) + public virtual string Store(AssetBase asset) { // Have to assign the asset ID here. This isn't likely to // trigger since current callers don't pass emtpy IDs @@ -492,23 +754,31 @@ namespace OpenSim.Services.Connectors string newID = null; try { - newID = SynchronousRestObjectRequester. - MakeRequest("POST", uri, asset, 10000, m_Auth); + newID = SynchronousRestObjectRequester.MakeRequest("POST", uri, asset, 10000, m_Auth); } catch { newID = null; } - if (newID == null || newID == String.Empty || newID == stringUUIDZero) + if (string.IsNullOrEmpty(newID) || newID == stringUUIDZero) { //The asset upload failed, try later lock(m_sendRetries) { - if (m_sendRetries[0] == null) - m_sendRetries[0] = new List(); - List m_queue = m_sendRetries[0]; - m_queue.Add(asset); + if(m_Cache == null) + { + if (m_sendRetries[0] == null) + m_sendRetries[0] = new List(); + m_sendRetries[0].Add(asset); + } + else + { + if (m_sendCachedRetries[0] == null) + m_sendCachedRetries[0] = new List(); + m_sendCachedRetries[0].Add(asset.ID); + } + m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later", asset.ID.ToString(), asset.Type.ToString()); m_retryTimer.Start(); @@ -520,26 +790,138 @@ namespace OpenSim.Services.Connectors { // Placing this here, so that this work with old asset servers that don't send any reply back // SynchronousRestObjectRequester returns somethins that is not an empty string - asset.ID = newID; + if (m_Cache != null) + m_Cache.Cache(asset); } } + + return asset.ID; + } + + public virtual string StoreForeign(AssetBase asset) + { + int type = Util.ParseForeignAssetID(asset.ID, out string uri, out string uuidstr); + if(type < 0) + return string.Empty; + + if(type != 0) + asset.ID = uuidstr; + if (m_Cache != null) m_Cache.Cache(asset); + if (asset.Temporary || asset.Local) + { + return asset.ID; + } + + if(type == 0) + uri = MapServer(uuidstr) + "/assets/"; + else + uri += "/assets/"; + + string newID = null; + try + { + newID = SynchronousRestObjectRequester.MakeRequest("POST", uri, asset, 10000, m_Auth); + } + catch + { + newID = null; + } + + if (string.IsNullOrEmpty(newID) || newID == stringUUIDZero) + { + //The asset upload failed, try later + lock (m_sendRetries) + { + if (m_Cache == null) + { + if (m_sendRetries[0] == null) + m_sendRetries[0] = new List(); + m_sendRetries[0].Add(asset); + } + else + { + if (m_sendCachedRetries[0] == null) + m_sendCachedRetries[0] = new List(); + m_sendCachedRetries[0].Add(asset.ID); + } + + m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later", + asset.ID.ToString(), asset.Type.ToString()); + m_retryTimer.Start(); + } + } + else + { + if (newID != asset.ID) + { + // Placing this here, so that this work with old asset servers that don't send any reply back + // SynchronousRestObjectRequester returns somethins that is not an empty string + asset.ID = newID; + if (m_Cache != null) + m_Cache.Cache(asset); + } + } + return asset.ID; } public void retryStore(AssetBase asset, int nextRetryLevel) { -/* this may be bad, so excluding - if (m_Cache != null && !m_Cache.Check(asset.ID)) + string uri = MapServer(asset.FullID.ToString()) + "/assets/"; + + string newID = null; + try { - m_log.WarnFormat("[Assets] Upload giveup asset bc no longer in local cache: {0}", - asset.ID.ToString(); - return; // if no longer in cache, it was deleted or expired + newID = SynchronousRestObjectRequester.MakeRequest("POST", uri, asset, 100000, m_Auth); } -*/ + catch + { + newID = null; + } + + if (string.IsNullOrEmpty(newID) || newID == stringUUIDZero) + { + if (nextRetryLevel >= MAXSENDRETRIESLEN) + m_log.WarnFormat("[Assets] Giving up on uploading after {2} retries id: {0} type {1}", + asset.ID.ToString(), asset.Type.ToString(), MAXSENDRETRIESLEN); + else + { + lock (m_sendRetries) + { + if (m_sendRetries[nextRetryLevel] == null) + m_sendRetries[nextRetryLevel] = new List(); + + List m_queue = m_sendRetries[nextRetryLevel]; + m_queue.Add(asset); + m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later", + asset.ID.ToString(), asset.Type.ToString()); + } + } + } + else + { + m_log.InfoFormat("[Assets] Upload of {0} succeeded after {1} failed attempts", asset.ID.ToString(), nextRetryLevel.ToString()); + if (newID != asset.ID) + { + asset.ID = newID; + m_Cache?.Cache(asset); + } + } + } + + public void retryCachedStore(string assetID, int nextRetryLevel) + { + m_Cache.Get(assetID,out AssetBase asset); + if(asset == null) + { + m_log.WarnFormat("[Assets] asset not in cache on uploading after {2} retries id: {0}", + assetID, MAXSENDRETRIESLEN); + } + string uri = MapServer(asset.FullID.ToString()) + "/assets/"; string newID = null; @@ -553,23 +935,21 @@ namespace OpenSim.Services.Connectors newID = null; } - if (newID == null || newID == String.Empty || newID == stringUUIDZero) + if (string.IsNullOrEmpty(newID) || newID == stringUUIDZero) { - if(nextRetryLevel >= MAXSENDRETRIESLEN) + if (nextRetryLevel >= MAXSENDRETRIESLEN) m_log.WarnFormat("[Assets] Giving up on uploading after {2} retries id: {0} type {1}", asset.ID.ToString(), asset.Type.ToString(), MAXSENDRETRIESLEN); else { - lock(m_sendRetries) + lock (m_sendRetries) { - if (m_sendRetries[nextRetryLevel] == null) - { - m_sendRetries[nextRetryLevel] = new List(); - } - List m_queue = m_sendRetries[nextRetryLevel]; - m_queue.Add(asset); - m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later", - asset.ID.ToString(), asset.Type.ToString()); + if (m_sendCachedRetries[nextRetryLevel] == null) + m_sendCachedRetries[nextRetryLevel] = new List(); + + m_sendCachedRetries[nextRetryLevel].Add(assetID); + m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later", + asset.ID.ToString(), asset.Type.ToString()); } } } @@ -578,14 +958,13 @@ namespace OpenSim.Services.Connectors m_log.InfoFormat("[Assets] Upload of {0} succeeded after {1} failed attempts", asset.ID.ToString(), nextRetryLevel.ToString()); if (newID != asset.ID) { - asset.ID = newID; + asset.ID = newID; + m_Cache?.Cache(asset); } } - - m_Cache?.Cache(asset); } - public bool UpdateContent(string id, byte[] data) + public virtual bool UpdateContent(string id, byte[] data) { AssetBase asset = null; @@ -612,8 +991,7 @@ namespace OpenSim.Services.Connectors return false; } - - public bool Delete(string id) + public virtual bool Delete(string id) { string uri = MapServer(id) + "/assets/" + id; diff --git a/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs b/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs index 7fa80b048a..e63401c4a3 100644 --- a/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs +++ b/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs @@ -41,29 +41,8 @@ namespace OpenSim.Services.Connectors { public class HGAssetServiceConnector : IAssetService { - private static readonly ILog m_log = - LogManager.GetLogger( - MethodBase.GetCurrentMethod().DeclaringType); - - private Dictionary m_endpointSerializer = new Dictionary(); - private object EndPointLock(IAssetService connector) - { - lock (m_endpointSerializer) - { - object eplock = null; - - if (! m_endpointSerializer.TryGetValue(connector, out eplock)) - { - eplock = new object(); - m_endpointSerializer.Add(connector, eplock); - // m_log.WarnFormat("[WEB UTIL] add a new host to end point serializer {0}",endpoint); - } - - return eplock; - } - } - - private Dictionary m_connectors = new Dictionary(); + private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType); + private ExpiringCacheOS m_connectors = new ExpiringCacheOS(120000); public HGAssetServiceConnector(IConfigSource source) { @@ -83,16 +62,12 @@ namespace OpenSim.Services.Connectors } } - private IAssetService GetConnector(string url) + private AssetServicesConnector GetConnector(string url) { - IAssetService connector = null; + AssetServicesConnector connector = null; lock (m_connectors) { - if (m_connectors.ContainsKey(url)) - { - connector = m_connectors[url]; - } - else + if (!m_connectors.TryGetValue(url, 120000, out connector)) { // Still not as flexible as I would like this to be, // but good enough for now @@ -108,7 +83,7 @@ namespace OpenSim.Services.Connectors string url = string.Empty; string assetID = string.Empty; - if (Util.ParseForeignAssetID(id, out url, out assetID)) + if (Util.ParseForeignAssetID(id, out url, out assetID) > 0) { IAssetService connector = GetConnector(url); return connector.Get(assetID); @@ -122,7 +97,7 @@ namespace OpenSim.Services.Connectors string url = string.Empty; string assetID = string.Empty; - if (Util.ParseForeignAssetID(id, out url, out assetID)) + if (Util.ParseForeignAssetID(id, out url, out assetID) > 0) { IAssetService connector = GetConnector(url); return connector.GetCached(assetID); @@ -136,7 +111,7 @@ namespace OpenSim.Services.Connectors string url = string.Empty; string assetID = string.Empty; - if (Util.ParseForeignAssetID(id, out url, out assetID)) + if (Util.ParseForeignAssetID(id, out url, out assetID) > 0) { IAssetService connector = GetConnector(url); return connector.GetMetadata(assetID); @@ -150,12 +125,12 @@ namespace OpenSim.Services.Connectors return null; } - public bool Get(string id, Object sender, AssetRetrieved handler) + public bool Get(string id, object sender, AssetRetrieved handler) { string url = string.Empty; string assetID = string.Empty; - if (Util.ParseForeignAssetID(id, out url, out assetID)) + if (Util.ParseForeignAssetID(id, out url, out assetID) > 0) { IAssetService connector = GetConnector(url); return connector.Get(assetID, sender, handler); @@ -164,7 +139,6 @@ namespace OpenSim.Services.Connectors return false; } - private struct AssetAndIndex { public UUID assetID; @@ -191,7 +165,7 @@ namespace OpenSim.Services.Connectors string url = string.Empty; string assetID = string.Empty; - if (Util.ParseForeignAssetID(ids[i], out url, out assetID)) + if (Util.ParseForeignAssetID(ids[i], out url, out assetID) > 0) { if (!url2assets.ContainsKey(url)) url2assets.Add(url, new List()); @@ -205,8 +179,8 @@ namespace OpenSim.Services.Connectors foreach (string url in url2assets.Keys) { - IAssetService connector = GetConnector(url); - lock (EndPointLock(connector)) + AssetServicesConnector connector = GetConnector(url); + lock (connector.ConnectorLock) { List curAssets = url2assets[url]; string[] assetIDs = curAssets.ConvertAll(a => a.assetID.ToString()).ToArray(); @@ -229,16 +203,16 @@ namespace OpenSim.Services.Connectors string url = string.Empty; string assetID = string.Empty; - if (Util.ParseForeignAssetID(asset.ID, out url, out assetID)) + if (Util.ParseForeignAssetID(asset.ID, out url, out assetID) > 0) { - IAssetService connector = GetConnector(url); + AssetServicesConnector connector = GetConnector(url); // Restore the assetID to a simple UUID asset.ID = assetID; - lock (EndPointLock(connector)) + lock ((connector.ConnectorLock)) return connector.Store(asset); } - return String.Empty; + return string.Empty; } public bool UpdateContent(string id, byte[] data)