diff --git a/OpenSim/Framework/GridInfo.cs b/OpenSim/Framework/GridInfo.cs
index d51acb606c..5e0f80417a 100644
--- a/OpenSim/Framework/GridInfo.cs
+++ b/OpenSim/Framework/GridInfo.cs
@@ -50,8 +50,9 @@ namespace OpenSim.Framework
public int Port;
public IPAddress IP;
public readonly string Host;
- public readonly string URI;
+ public readonly string URL;
public readonly string Path;
+ public readonly string URI;
public OSHTTPURI(string uri, bool withDNSResolve = false)
{
@@ -60,6 +61,7 @@ namespace OpenSim.Framework
IP = null;
Host = string.Empty;
URI = string.Empty;
+ URL = string.Empty;
Path = string.Empty;
if (string.IsNullOrEmpty(uri))
@@ -80,7 +82,8 @@ namespace OpenSim.Framework
if (Path[Path.Length - 1] == '/')
Path = Path.Substring(0, Path.Length - 1);
- URI = m_checkuri.Scheme + "://" + Host + ":" + Port + Path;
+ URL = m_checkuri.Scheme + "://" + Host + ":" + Port;
+ URI = URL + Path;
if (withDNSResolve)
{
diff --git a/OpenSim/Framework/Util.cs b/OpenSim/Framework/Util.cs
index 99c3405355..8e99f720de 100644
--- a/OpenSim/Framework/Util.cs
+++ b/OpenSim/Framework/Util.cs
@@ -1485,30 +1485,41 @@ namespace OpenSim.Framework
/// The URL: http://grid.example.com:8002
/// The asset ID: 00000000-0000-0000-0000-000000000000. Returned even if 'id' isn't foreign.
/// True: this is a foreign asset ID; False: it isn't
- public static bool ParseForeignAssetID(string id, out string url, out string assetID)
+ public static int ParseForeignAssetID(string id, out string url, out string assetID)
{
- url = String.Empty;
- assetID = String.Empty;
+ url = string.Empty;
+ assetID = string.Empty;
- if (UUID.TryParse(id, out UUID uuid))
+ if (id.Length == 0)
+ return -1;
+
+ if(id[0] != 'h' && id[0] != 'H')
{
- assetID = uuid.ToString();
- return false;
+ if (UUID.TryParse(id, out UUID luuid))
+ {
+ assetID = id;
+ return 0;
+ }
+ return -1;
}
- if ((id.Length == 0) || (id[0] != 'h' && id[0] != 'H'))
- return false;
-
- if (!Uri.TryCreate(id, UriKind.Absolute, out Uri assetUri) || assetUri.Scheme != Uri.UriSchemeHttp)
- return false;
-
- url = "http://" + assetUri.Authority;
- assetID = assetUri.LocalPath.Trim(new char[] { '/' });
-
- if (!UUID.TryParse(assetID, out uuid))
- return false;
-
- return true;
+ OSHTTPURI uri = new OSHTTPURI(id, true);
+ if(uri.IsResolvedHost)
+ {
+ url = uri.URL;
+ string tmp = uri.Path;
+ if(tmp.Length < 36)
+ return -3;
+ if(tmp[0] =='/')
+ tmp = tmp.Substring(1);
+ if (UUID.TryParse(tmp, out UUID uuid))
+ {
+ assetID = tmp;
+ return 1;
+ }
+ return -1;
+ }
+ return -2;
}
///
diff --git a/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs b/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs
index dfd40f1286..cebbb54543 100644
--- a/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs
+++ b/OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs
@@ -147,7 +147,7 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess
asset1.Data = asset.Data;
string id = m_scene.AssetService.Store(asset1);
- if (String.IsNullOrEmpty(id))
+ if (string.IsNullOrEmpty(id))
{
if (verbose)
m_log.DebugFormat("[HG ASSET MAPPER]: Asset server {0} did not accept {1}", url, asset.ID);
@@ -169,7 +169,6 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess
to.Name = from.Name;
to.Temporary = from.Temporary;
to.Type = from.Type;
-
}
private void AdjustIdentifiers(AssetMetadata meta)
diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsIn/Asset/AssetServiceInConnectorModule.cs b/OpenSim/Region/CoreModules/ServiceConnectorsIn/Asset/AssetServiceInConnectorModule.cs
index 2c882e686e..034185caa0 100644
--- a/OpenSim/Region/CoreModules/ServiceConnectorsIn/Asset/AssetServiceInConnectorModule.cs
+++ b/OpenSim/Region/CoreModules/ServiceConnectorsIn/Asset/AssetServiceInConnectorModule.cs
@@ -96,7 +96,7 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsIn.Asset
m_log.Info("[HGAssetService]: Starting...");
- Object[] args = new Object[] { m_Config, MainServer.Instance, "HGAssetService" };
+ object[] args = new object[] { m_Config, MainServer.Instance, "HGAssetService" };
ServerUtils.LoadPlugin("OpenSim.Server.Handlers.dll:AssetServiceConnector", args);
}
diff --git a/OpenSim/Region/CoreModules/ServiceConnectorsIn/Hypergrid/HypergridServiceInConnectorModule.cs b/OpenSim/Region/CoreModules/ServiceConnectorsIn/Hypergrid/HypergridServiceInConnectorModule.cs
index 4f87595b60..9f76be0af8 100644
--- a/OpenSim/Region/CoreModules/ServiceConnectorsIn/Hypergrid/HypergridServiceInConnectorModule.cs
+++ b/OpenSim/Region/CoreModules/ServiceConnectorsIn/Hypergrid/HypergridServiceInConnectorModule.cs
@@ -51,7 +51,7 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsIn.Hypergrid
private IConfigSource m_Config;
private bool m_Registered = false;
- private string m_LocalServiceDll = String.Empty;
+ private string m_LocalServiceDll = string.Empty;
private GatekeeperServiceInConnector m_HypergridHandler;
private UserAgentServerConnector m_UASHandler;
@@ -71,13 +71,11 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsIn.Hypergrid
if (fconfig != null)
{
m_LocalServiceDll = fconfig.GetString("LocalServiceModule", m_LocalServiceDll);
- if (m_LocalServiceDll == String.Empty)
+ if (m_LocalServiceDll == string.Empty)
m_log.WarnFormat("[HGGRID IN CONNECTOR]: Friends LocalServiceModule config missing");
}
}
-
}
-
}
public void PostInitialise()
@@ -123,7 +121,7 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsIn.Hypergrid
ISimulationService simService = scene.RequestModuleInterface();
IFriendsSimConnector friendsConn = scene.RequestModuleInterface();
- Object[] args = new Object[] { m_Config };
+ object[] args = new object[] { m_Config };
// IFriendsService friendsService = ServerUtils.LoadPlugin(m_LocalServiceDll, args)
ServerUtils.LoadPlugin(m_LocalServiceDll, args);
diff --git a/OpenSim/Services/AssetService/AssetService.cs b/OpenSim/Services/AssetService/AssetService.cs
index ee2e56871f..3930a68ce0 100644
--- a/OpenSim/Services/AssetService/AssetService.cs
+++ b/OpenSim/Services/AssetService/AssetService.cs
@@ -40,9 +40,7 @@ namespace OpenSim.Services.AssetService
{
public class AssetService : AssetServiceBase, IAssetService
{
- private static readonly ILog m_log =
- LogManager.GetLogger(
- MethodBase.GetCurrentMethod().DeclaringType);
+ private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
protected static AssetService m_RootInstance;
diff --git a/OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs b/OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs
index 21a3a3495d..68852a1347 100644
--- a/OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs
+++ b/OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs
@@ -43,29 +43,30 @@ namespace OpenSim.Services.Connectors
{
public class AssetServicesConnector : BaseServiceConnector, IAssetService
{
- private static readonly ILog m_log =
- LogManager.GetLogger(
- MethodBase.GetCurrentMethod().DeclaringType);
+ private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
const int MAXSENDRETRIESLEN = 30;
+ public object ConnectorLock = new object();
+
private string m_ServerURI = String.Empty;
private IAssetCache m_Cache = null;
private int m_retryCounter;
private bool m_inRetries;
- private List[] m_sendRetries = new List[MAXSENDRETRIESLEN];
+ private List[] m_sendRetries = new List[MAXSENDRETRIESLEN];
+ private List[] m_sendCachedRetries = new List[MAXSENDRETRIESLEN];
private System.Timers.Timer m_retryTimer;
- private int m_maxAssetRequestConcurrency = 30;
+
+ private int m_maxAssetRequestConcurrency = 8;
private delegate void AssetRetrievedEx(AssetBase asset);
// Keeps track of concurrent requests for the same asset, so that it's only loaded once.
// Maps: Asset ID -> Handlers which will be called when the asset has been loaded
-// private Dictionary m_AssetHandlers = new Dictionary();
private Dictionary> m_AssetHandlers = new Dictionary>();
- private Dictionary m_UriMap = new Dictionary();
+ private Dictionary m_UriMap;
private Thread[] m_fetchThreads;
@@ -84,8 +85,7 @@ namespace OpenSim.Services.Connectors
m_ServerURI = serverURI.TrimEnd('/');
}
- public AssetServicesConnector(IConfigSource source)
- : base(source, "AssetService")
+ public AssetServicesConnector(IConfigSource source)
{
Initialise(source);
}
@@ -103,64 +103,78 @@ namespace OpenSim.Services.Connectors
throw new Exception("Asset connector init error");
}
- m_ServerURI = assetConfig.GetString("AssetServerURI", String.Empty);
- if (m_ServerURI == String.Empty)
+ m_ServerURI = assetConfig.GetString("AssetServerURI", string.Empty);
+ if (string.IsNullOrEmpty(m_ServerURI))
{
- m_log.Error("[ASSET CONNECTOR]: No Server URI named in section AssetService");
+ IConfig netConfig = source.Configs["Network"];
+ m_ServerURI = netConfig.GetString("asset_server_url", string.Empty);
+ }
+ if (string.IsNullOrEmpty(m_ServerURI))
+ {
+ m_log.Error("[ASSET CONNECTOR]: AssetServerURI not defined in section AssetService");
throw new Exception("Asset connector init error");
}
+ OSHHTPHost m_GridAssetsURL = new OSHHTPHost(m_ServerURI, true);
+ if(!m_GridAssetsURL.IsResolvedHost)
+ {
+ m_log.Error("[ASSET CONNECTOR]: Could not parse or resolve AssetServerURI");
+ throw new Exception("Asset connector init error");
+ }
+
+ m_ServerURI = m_GridAssetsURL.URI;
+ bool usemaps = assetConfig.GetBoolean("AssetServerIsCluster", false);
+
+ if(usemaps)
+ {
+ m_UriMap = new Dictionary();
+ for (int i = 0; i < 256; i++)
+ {
+ string prefix = i.ToString("x2");
+ string groupHost = assetConfig.GetString("AssetServerHost_" + prefix, string.Empty);
+ if(string.IsNullOrEmpty(groupHost))
+ m_UriMap[prefix] = m_ServerURI;
+ else
+ {
+ OSHHTPHost other = new OSHHTPHost(groupHost, true);
+ if(!other.IsResolvedHost)
+ {
+ m_log.Error("[ASSET CONNECTOR]: Could not parse or resolve AssetServerHost_" + prefix);
+ throw new Exception("Asset connector init error");
+ }
+ m_UriMap[prefix] = other.URI;
+ }
+ }
+ }
+ else
+ m_UriMap = null;
+
m_retryTimer = new System.Timers.Timer();
m_retryTimer.Elapsed += new ElapsedEventHandler(retryCheck);
m_retryTimer.AutoReset = true;
m_retryTimer.Interval = 60000;
- Uri serverUri = new Uri(m_ServerURI);
+ m_fetchThreads = new Thread[3];
- string groupHost = serverUri.Host;
+ Initialise(source, "AssetService");
- for (int i = 0 ; i < 256 ; i++)
+ for (int i = 0 ; i < m_fetchThreads.Length; i++)
{
- string prefix = i.ToString("x2");
- groupHost = assetConfig.GetString("AssetServerHost_"+prefix, groupHost);
-
- m_UriMap[prefix] = groupHost;
- //m_log.DebugFormat("[ASSET]: Using {0} for prefix {1}", groupHost, prefix);
- }
-
- m_fetchThreads = new Thread[2];
-
- for (int i = 0 ; i < 2 ; i++)
- {
- m_fetchThreads[i] = WorkManager.StartThread(AssetRequestProcessor, String.Format("GetAssetsWorker{0}", i));
+ m_fetchThreads[i] = WorkManager.StartThread(AssetRequestProcessor, string.Format("GetAssetsWorker{0}", i));
}
}
private string MapServer(string id)
{
- if (m_UriMap.Count == 0)
+ if (m_UriMap == null)
return m_ServerURI;
- UriBuilder serverUri = new UriBuilder(m_ServerURI);
-
string prefix = id.Substring(0, 2).ToLower();
- string host;
+ if (m_UriMap.TryGetValue(prefix, out string host))
+ return host;
- // HG URLs will not be valid UUIDS
- if (m_UriMap.ContainsKey(prefix))
- host = m_UriMap[prefix];
- else
- host = m_UriMap["00"];
-
- serverUri.Host = host;
-
- // m_log.DebugFormat("[ASSET]: Using {0} for host name for prefix {1}", host, prefix);
-
- string ret = serverUri.Uri.AbsoluteUri;
- if (ret.EndsWith("/"))
- ret = ret.Substring(0, ret.Length - 1);
- return ret;
+ return m_UriMap["00"];
}
protected void retryCheck(object source, ElapsedEventArgs e)
@@ -179,48 +193,94 @@ namespace OpenSim.Services.Connectors
int inUse = 0;
int nextlevel;
int timefactor;
- List retrylist;
- // we need to go down
- for(int i = MAXSENDRETRIESLEN - 1; i >= 0; i--)
+ if(m_Cache == null)
{
- lock(m_sendRetries)
- retrylist = m_sendRetries[i];
-
- if(retrylist == null)
- continue;
-
- inUse++;
- nextlevel = i + 1;
-
- //We exponentially fall back on frequency until we reach one attempt per hour
- //The net result is that we end up in the queue for roughly 24 hours..
- //24 hours worth of assets could be a lot, so the hope is that the region admin
- //will have gotten the asset connector back online quickly!
- if(i == 0)
- timefactor = 1;
- else
+ List retrylist;
+ // we need to go down
+ for(int i = MAXSENDRETRIESLEN - 1; i >= 0; i--)
{
- timefactor = 1 << nextlevel;
- if (timefactor > 60)
- timefactor = 60;
+ lock(m_sendRetries)
+ retrylist = m_sendRetries[i];
+
+ if(retrylist == null)
+ continue;
+
+ inUse++;
+ nextlevel = i + 1;
+
+ //We exponentially fall back on frequency until we reach one attempt per hour
+ //The net result is that we end up in the queue for roughly 24 hours..
+ //24 hours worth of assets could be a lot, so the hope is that the region admin
+ //will have gotten the asset connector back online quickly!
+ if(i == 0)
+ timefactor = 1;
+ else
+ {
+ timefactor = 1 << nextlevel;
+ if (timefactor > 60)
+ timefactor = 60;
+ }
+
+ if(m_retryCounter < timefactor)
+ continue; // to update inUse;
+
+ if (m_retryCounter % timefactor != 0)
+ continue;
+
+ // a list to retry
+ lock(m_sendRetries)
+ m_sendRetries[i] = null;
+
+ // we are the only ones with a copy of this retrylist now
+ foreach(AssetBase ass in retrylist)
+ retryStore(ass, nextlevel);
}
+ }
+ else
+ {
+ List retrylist;
+ // we need to go down
+ for (int i = MAXSENDRETRIESLEN - 1; i >= 0; i--)
+ {
+ lock (m_sendRetries)
+ retrylist = m_sendCachedRetries[i];
- if(m_retryCounter < timefactor)
- continue; // to update inUse;
+ if (retrylist == null)
+ continue;
- if (m_retryCounter % timefactor != 0)
- continue;
+ inUse++;
+ nextlevel = i + 1;
- // a list to retry
- lock(m_sendRetries)
- m_sendRetries[i] = null;
+ //We exponentially fall back on frequency until we reach one attempt per hour
+ //The net result is that we end up in the queue for roughly 24 hours..
+ //24 hours worth of assets could be a lot, so the hope is that the region admin
+ //will have gotten the asset connector back online quickly!
+ if (i == 0)
+ timefactor = 1;
+ else
+ {
+ timefactor = 1 << nextlevel;
+ if (timefactor > 60)
+ timefactor = 60;
+ }
- // we are the only ones with a copy of this retrylist now
- foreach(AssetBase ass in retrylist)
- retryStore(ass, nextlevel);
+ if (m_retryCounter < timefactor)
+ continue; // to update inUse;
+
+ if (m_retryCounter % timefactor != 0)
+ continue;
+
+ // a list to retry
+ lock (m_sendRetries)
+ m_sendCachedRetries[i] = null;
+
+ // we are the only ones with a copy of this retrylist now
+ foreach (string id in retrylist)
+ retryCachedStore(id, nextlevel);
+ }
}
- lock(m_sendRetries)
+ lock (m_sendRetries)
{
if(inUse == 0 )
m_retryTimer.Stop();
@@ -229,12 +289,23 @@ namespace OpenSim.Services.Connectors
}
}
- protected void SetCache(IAssetCache cache)
+ public void SetCache(IAssetCache cache)
{
m_Cache = cache;
}
- public AssetBase Get(string id)
+ public AssetBase GetCached(string id)
+ {
+ AssetBase asset = null;
+ if (m_Cache != null)
+ {
+ m_Cache.Get(id, out asset);
+ }
+
+ return asset;
+ }
+
+ public virtual AssetBase Get(string id)
{
AssetBase asset = null;
if (m_Cache != null)
@@ -246,6 +317,7 @@ namespace OpenSim.Services.Connectors
if (asset == null || asset.Data == null || asset.Data.Length == 0)
{
string uri = MapServer(id) + "/assets/" + id;
+
asset = SynchronousRestObjectRequester.MakeRequest("GET", uri, 0, m_Auth);
if (m_Cache != null)
{
@@ -258,20 +330,39 @@ namespace OpenSim.Services.Connectors
return asset;
}
- public AssetBase GetCached(string id)
+ public AssetBase GetForeign(string id)
{
-// m_log.DebugFormat("[ASSET SERVICE CONNECTOR]: Cache request for {0}", id);
+ int type = Util.ParseForeignAssetID(id, out string uri, out string uuidstr);
+ if (type < 0)
+ return null;
AssetBase asset = null;
if (m_Cache != null)
{
- m_Cache.Get(id, out asset);
+ if (!m_Cache.Get(uuidstr, out asset))
+ return null;
}
+ if (asset == null || asset.Data == null || asset.Data.Length == 0)
+ {
+ if (type == 0)
+ uri = MapServer(uuidstr) + "/assets/" + uuidstr;
+ else
+ uri = uri + "/assets/" + uuidstr;
+
+ asset = SynchronousRestObjectRequester.MakeRequest("GET", uri, 0, m_Auth);
+ if (m_Cache != null)
+ {
+ if (asset != null)
+ m_Cache.Cache(asset);
+ else
+ m_Cache.CacheNegative(id);
+ }
+ }
return asset;
}
- public AssetMetadata GetMetadata(string id)
+ public virtual AssetMetadata GetMetadata(string id)
{
if (m_Cache != null)
{
@@ -289,40 +380,87 @@ namespace OpenSim.Services.Connectors
return asset;
}
- public byte[] GetData(string id)
+ public AssetMetadata GetForeignMetadata(string id)
{
+ int type = Util.ParseForeignAssetID(id, out string uri, out string uuidstr);
+ if (type < 0)
+ return null;
+
if (m_Cache != null)
{
AssetBase fullAsset;
- if (!m_Cache.Get(id, out fullAsset))
+ if (!m_Cache.Get(uuidstr, out fullAsset))
+ return null;
+
+ if (fullAsset != null)
+ return fullAsset.Metadata;
+ }
+
+ if (type == 0)
+ uri = MapServer(uuidstr) + "/assets/" + uuidstr + "/metadata";
+ else
+ uri = uri + "/assets/" + uuidstr + "/metadata";
+
+ AssetMetadata asset = SynchronousRestObjectRequester.MakeRequest("GET", uri, 0, m_Auth);
+ return asset;
+ }
+
+ public virtual byte[] GetData(string id)
+ {
+ if (m_Cache != null)
+ {
+ if (!m_Cache.Get(id, out AssetBase fullAsset))
return null;
if (fullAsset != null)
return fullAsset.Data;
}
- using (RestClient rc = new RestClient(MapServer(id)))
- {
- rc.AddResourcePath("assets");
- rc.AddResourcePath(id);
- rc.AddResourcePath("data");
+ string uri = MapServer(id);
+ using (RestClient rc = new RestClient(uri))
+ {
+ rc.AddResourcePath("assets/" + id + "/Data");
rc.RequestMethod = "GET";
- using (Stream s = rc.Request(m_Auth))
+ using (MemoryStream s = rc.Request(m_Auth))
{
- if (s == null)
+ if (s == null || s.Length == 0)
return null;
-
- if (s.Length > 0)
- {
- byte[] ret = new byte[s.Length];
- s.Read(ret, 0, (int)s.Length);
-
- return ret;
- }
+ return s.ToArray();
}
+ }
+ }
+
+ public byte[] GetForeignData(string id)
+ {
+ int type = Util.ParseForeignAssetID(id, out string uri, out string uuidstr);
+ if (type < 0)
return null;
+
+ if (m_Cache != null)
+ {
+ if (!m_Cache.Get(uuidstr, out AssetBase fullAsset))
+ return null;
+
+ if (fullAsset != null)
+ return fullAsset.Data;
+ }
+
+ if (type == 0)
+ uri = MapServer(uuidstr);
+
+ using (RestClient rc = new RestClient(uri))
+ {
+ rc.AddResourcePath("assets/" + id + "/Data");
+ rc.RequestMethod = "GET";
+
+ using (MemoryStream s = rc.Request(m_Auth))
+ {
+ if (s == null || s.Length == 0)
+ return null;
+ return s.ToArray();
+ }
}
}
@@ -332,8 +470,106 @@ namespace OpenSim.Services.Connectors
public string id;
}
- private BlockingCollection m_requestQueue = new BlockingCollection();
+ public virtual bool Get(string id, object sender, AssetRetrieved handler)
+ {
+ AssetBase asset = null;
+ if (m_Cache != null)
+ {
+ if (!m_Cache.Get(id, out asset))
+ return false;
+ }
+ if (asset == null)
+ {
+ string uri = MapServer(id) + "/assets/" + id;
+
+ lock (m_AssetHandlers)
+ {
+ AssetRetrievedEx handlerEx = new AssetRetrievedEx(delegate (AssetBase _asset) { handler(id, sender, _asset); });
+
+ List handlers;
+ if (m_AssetHandlers.TryGetValue(id, out handlers))
+ {
+ // Someone else is already loading this asset. It will notify our handler when done.
+ handlers.Add(handlerEx);
+ return true;
+ }
+
+ handlers = new List();
+ handlers.Add(handlerEx);
+
+ m_AssetHandlers.Add(id, handlers);
+
+ QueuedAssetRequest request = new QueuedAssetRequest();
+ request.id = id;
+ request.uri = uri;
+ m_requestQueue.Add(request);
+ }
+ }
+ else
+ {
+ if (asset != null && (asset.Data == null || asset.Data.Length == 0))
+ asset = null;
+ handler(id, sender, asset);
+ }
+
+ return true;
+ }
+
+ public bool GetForeign(string id, object sender, AssetRetrieved handler)
+ {
+ int type = Util.ParseForeignAssetID(id, out string uri, out string uuidstr);
+ if (type < 0)
+ return false;
+
+ AssetBase asset = null;
+ if (m_Cache != null)
+ {
+ if (!m_Cache.Get(uuidstr, out asset))
+ return false;
+ }
+
+ if (asset == null)
+ {
+ if (type == 0)
+ uri = MapServer(uuidstr) + "/assets/" + uuidstr;
+ else
+ uri = uri + "/assets/" + uuidstr;
+
+ lock (m_AssetHandlers)
+ {
+ AssetRetrievedEx handlerEx = new AssetRetrievedEx(delegate (AssetBase _asset) { handler(id, sender, _asset); });
+
+ List handlers;
+ if (m_AssetHandlers.TryGetValue(id, out handlers))
+ {
+ // Someone else is already loading this asset. It will notify our handler when done.
+ handlers.Add(handlerEx);
+ return true;
+ }
+
+ handlers = new List();
+ handlers.Add(handlerEx);
+
+ m_AssetHandlers.Add(id, handlers);
+
+ QueuedAssetRequest request = new QueuedAssetRequest();
+ request.id = id;
+ request.uri = uri;
+ m_requestQueue.Add(request);
+ }
+ }
+ else
+ {
+ if (asset != null && (asset.Data == null || asset.Data.Length == 0))
+ asset = null;
+ handler(id, sender, asset);
+ }
+
+ return true;
+ }
+
+ private BlockingCollection m_requestQueue = new BlockingCollection();
private void AssetRequestProcessor()
{
QueuedAssetRequest r;
@@ -347,12 +583,11 @@ namespace OpenSim.Services.Connectors
}
Watchdog.UpdateThread();
- string uri = r.uri;
string id = r.id;
try
{
- AssetBase a = SynchronousRestObjectRequester.MakeRequest("GET", uri, 0, 30000, m_Auth);
+ AssetBase a = SynchronousRestObjectRequester.MakeRequest("GET", r.uri, 0, 30000, m_Auth);
if (a != null && m_Cache != null)
m_Cache.Cache(a);
@@ -381,50 +616,6 @@ namespace OpenSim.Services.Connectors
}
}
- public bool Get(string id, Object sender, AssetRetrieved handler)
- {
- string uri = MapServer(id) + "/assets/" + id;
-
- AssetBase asset = null;
- if (m_Cache != null)
- {
- if (!m_Cache.Get(id, out asset))
- return false;
- }
-
- if (asset == null || asset.Data == null || asset.Data.Length == 0)
- {
- lock (m_AssetHandlers)
- {
- AssetRetrievedEx handlerEx = new AssetRetrievedEx(delegate(AssetBase _asset) { handler(id, sender, _asset); });
-
- List handlers;
- if (m_AssetHandlers.TryGetValue(id, out handlers))
- {
- // Someone else is already loading this asset. It will notify our handler when done.
- handlers.Add(handlerEx);
- return true;
- }
-
- handlers = new List();
- handlers.Add(handlerEx);
-
- m_AssetHandlers.Add(id, handlers);
-
- QueuedAssetRequest request = new QueuedAssetRequest();
- request.id = id;
- request.uri = uri;
- m_requestQueue.Add(request);
- }
- }
- else
- {
- handler(id, sender, asset);
- }
-
- return true;
- }
-
public virtual bool[] AssetsExist(string[] ids)
{
string uri = m_ServerURI + "/get_assets_exist";
@@ -446,9 +637,80 @@ namespace OpenSim.Services.Connectors
return exist;
}
+ private struct AssetAndIndex
+ {
+ public string assetID;
+ public int index;
+
+ public AssetAndIndex(string assetID, int index)
+ {
+ this.assetID = assetID;
+ this.index = index;
+ }
+ }
+
+ public bool[] ForeignAssetsExist(string[] ids)
+ {
+ bool[] exist = new bool[ids.Length];
+
+ var url2assets = new Dictionary>();
+
+ for (int i = 0; i < ids.Length; i++)
+ {
+ int ltype = Util.ParseForeignAssetID(ids[i], out string lurl, out string luuidstr);
+ if (ltype > 0)
+ {
+ if (ltype == 0)
+ lurl = m_ServerURI;
+
+ List < AssetAndIndex > lst;
+ if (!url2assets.TryGetValue(lurl, out lst))
+ {
+ lst = new List();
+ url2assets.Add(lurl, lst);
+ }
+ lst.Add(new AssetAndIndex(luuidstr, i));
+ }
+ }
+
+ // Query each of the servers in turn
+ foreach (KeyValuePair> kvp in url2assets)
+ {
+ List curAssets = kvp.Value;
+ string[] assetIDs = new string[curAssets.Count];
+ for(int i = 0; i < assetIDs.Length;++i)
+ assetIDs[i] = curAssets[i].assetID;
+
+ string uri = kvp.Key + "/get_assets_exist";
+
+ bool[] curExist = null;
+ try
+ {
+ curExist = SynchronousRestObjectRequester.MakeRequest("POST", uri, assetIDs, m_Auth);
+ }
+ catch (Exception)
+ {
+ // This is most likely to happen because the server doesn't support this function,
+ // so just silently return "doesn't exist" for all the assets.
+ }
+
+ if(curExist != null)
+ {
+ int i = 0;
+ foreach (AssetAndIndex ai in curAssets)
+ {
+ exist[ai.index] = curExist[i];
+ ++i;
+ }
+ }
+ }
+
+ return exist;
+ }
+
string stringUUIDZero = UUID.Zero.ToString();
- public string Store(AssetBase asset)
+ public virtual string Store(AssetBase asset)
{
// Have to assign the asset ID here. This isn't likely to
// trigger since current callers don't pass emtpy IDs
@@ -492,23 +754,31 @@ namespace OpenSim.Services.Connectors
string newID = null;
try
{
- newID = SynchronousRestObjectRequester.
- MakeRequest("POST", uri, asset, 10000, m_Auth);
+ newID = SynchronousRestObjectRequester.MakeRequest("POST", uri, asset, 10000, m_Auth);
}
catch
{
newID = null;
}
- if (newID == null || newID == String.Empty || newID == stringUUIDZero)
+ if (string.IsNullOrEmpty(newID) || newID == stringUUIDZero)
{
//The asset upload failed, try later
lock(m_sendRetries)
{
- if (m_sendRetries[0] == null)
- m_sendRetries[0] = new List();
- List m_queue = m_sendRetries[0];
- m_queue.Add(asset);
+ if(m_Cache == null)
+ {
+ if (m_sendRetries[0] == null)
+ m_sendRetries[0] = new List();
+ m_sendRetries[0].Add(asset);
+ }
+ else
+ {
+ if (m_sendCachedRetries[0] == null)
+ m_sendCachedRetries[0] = new List();
+ m_sendCachedRetries[0].Add(asset.ID);
+ }
+
m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later",
asset.ID.ToString(), asset.Type.ToString());
m_retryTimer.Start();
@@ -520,26 +790,138 @@ namespace OpenSim.Services.Connectors
{
// Placing this here, so that this work with old asset servers that don't send any reply back
// SynchronousRestObjectRequester returns somethins that is not an empty string
-
asset.ID = newID;
+ if (m_Cache != null)
+ m_Cache.Cache(asset);
}
}
+
+ return asset.ID;
+ }
+
+ public virtual string StoreForeign(AssetBase asset)
+ {
+ int type = Util.ParseForeignAssetID(asset.ID, out string uri, out string uuidstr);
+ if(type < 0)
+ return string.Empty;
+
+ if(type != 0)
+ asset.ID = uuidstr;
+
if (m_Cache != null)
m_Cache.Cache(asset);
+ if (asset.Temporary || asset.Local)
+ {
+ return asset.ID;
+ }
+
+ if(type == 0)
+ uri = MapServer(uuidstr) + "/assets/";
+ else
+ uri += "/assets/";
+
+ string newID = null;
+ try
+ {
+ newID = SynchronousRestObjectRequester.MakeRequest("POST", uri, asset, 10000, m_Auth);
+ }
+ catch
+ {
+ newID = null;
+ }
+
+ if (string.IsNullOrEmpty(newID) || newID == stringUUIDZero)
+ {
+ //The asset upload failed, try later
+ lock (m_sendRetries)
+ {
+ if (m_Cache == null)
+ {
+ if (m_sendRetries[0] == null)
+ m_sendRetries[0] = new List();
+ m_sendRetries[0].Add(asset);
+ }
+ else
+ {
+ if (m_sendCachedRetries[0] == null)
+ m_sendCachedRetries[0] = new List();
+ m_sendCachedRetries[0].Add(asset.ID);
+ }
+
+ m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later",
+ asset.ID.ToString(), asset.Type.ToString());
+ m_retryTimer.Start();
+ }
+ }
+ else
+ {
+ if (newID != asset.ID)
+ {
+ // Placing this here, so that this work with old asset servers that don't send any reply back
+ // SynchronousRestObjectRequester returns somethins that is not an empty string
+ asset.ID = newID;
+ if (m_Cache != null)
+ m_Cache.Cache(asset);
+ }
+ }
+
return asset.ID;
}
public void retryStore(AssetBase asset, int nextRetryLevel)
{
-/* this may be bad, so excluding
- if (m_Cache != null && !m_Cache.Check(asset.ID))
+ string uri = MapServer(asset.FullID.ToString()) + "/assets/";
+
+ string newID = null;
+ try
{
- m_log.WarnFormat("[Assets] Upload giveup asset bc no longer in local cache: {0}",
- asset.ID.ToString();
- return; // if no longer in cache, it was deleted or expired
+ newID = SynchronousRestObjectRequester.MakeRequest("POST", uri, asset, 100000, m_Auth);
}
-*/
+ catch
+ {
+ newID = null;
+ }
+
+ if (string.IsNullOrEmpty(newID) || newID == stringUUIDZero)
+ {
+ if (nextRetryLevel >= MAXSENDRETRIESLEN)
+ m_log.WarnFormat("[Assets] Giving up on uploading after {2} retries id: {0} type {1}",
+ asset.ID.ToString(), asset.Type.ToString(), MAXSENDRETRIESLEN);
+ else
+ {
+ lock (m_sendRetries)
+ {
+ if (m_sendRetries[nextRetryLevel] == null)
+ m_sendRetries[nextRetryLevel] = new List();
+
+ List m_queue = m_sendRetries[nextRetryLevel];
+ m_queue.Add(asset);
+ m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later",
+ asset.ID.ToString(), asset.Type.ToString());
+ }
+ }
+ }
+ else
+ {
+ m_log.InfoFormat("[Assets] Upload of {0} succeeded after {1} failed attempts", asset.ID.ToString(), nextRetryLevel.ToString());
+ if (newID != asset.ID)
+ {
+ asset.ID = newID;
+ m_Cache?.Cache(asset);
+ }
+ }
+ }
+
+ public void retryCachedStore(string assetID, int nextRetryLevel)
+ {
+ m_Cache.Get(assetID,out AssetBase asset);
+ if(asset == null)
+ {
+ m_log.WarnFormat("[Assets] asset not in cache on uploading after {2} retries id: {0}",
+ assetID, MAXSENDRETRIESLEN);
+ }
+
string uri = MapServer(asset.FullID.ToString()) + "/assets/";
string newID = null;
@@ -553,23 +935,21 @@ namespace OpenSim.Services.Connectors
newID = null;
}
- if (newID == null || newID == String.Empty || newID == stringUUIDZero)
+ if (string.IsNullOrEmpty(newID) || newID == stringUUIDZero)
{
- if(nextRetryLevel >= MAXSENDRETRIESLEN)
+ if (nextRetryLevel >= MAXSENDRETRIESLEN)
m_log.WarnFormat("[Assets] Giving up on uploading after {2} retries id: {0} type {1}",
asset.ID.ToString(), asset.Type.ToString(), MAXSENDRETRIESLEN);
else
{
- lock(m_sendRetries)
+ lock (m_sendRetries)
{
- if (m_sendRetries[nextRetryLevel] == null)
- {
- m_sendRetries[nextRetryLevel] = new List();
- }
- List m_queue = m_sendRetries[nextRetryLevel];
- m_queue.Add(asset);
- m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later",
- asset.ID.ToString(), asset.Type.ToString());
+ if (m_sendCachedRetries[nextRetryLevel] == null)
+ m_sendCachedRetries[nextRetryLevel] = new List();
+
+ m_sendCachedRetries[nextRetryLevel].Add(assetID);
+ m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later",
+ asset.ID.ToString(), asset.Type.ToString());
}
}
}
@@ -578,14 +958,13 @@ namespace OpenSim.Services.Connectors
m_log.InfoFormat("[Assets] Upload of {0} succeeded after {1} failed attempts", asset.ID.ToString(), nextRetryLevel.ToString());
if (newID != asset.ID)
{
- asset.ID = newID;
+ asset.ID = newID;
+ m_Cache?.Cache(asset);
}
}
-
- m_Cache?.Cache(asset);
}
- public bool UpdateContent(string id, byte[] data)
+ public virtual bool UpdateContent(string id, byte[] data)
{
AssetBase asset = null;
@@ -612,8 +991,7 @@ namespace OpenSim.Services.Connectors
return false;
}
-
- public bool Delete(string id)
+ public virtual bool Delete(string id)
{
string uri = MapServer(id) + "/assets/" + id;
diff --git a/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs b/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs
index 7fa80b048a..e63401c4a3 100644
--- a/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs
+++ b/OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs
@@ -41,29 +41,8 @@ namespace OpenSim.Services.Connectors
{
public class HGAssetServiceConnector : IAssetService
{
- private static readonly ILog m_log =
- LogManager.GetLogger(
- MethodBase.GetCurrentMethod().DeclaringType);
-
- private Dictionary m_endpointSerializer = new Dictionary();
- private object EndPointLock(IAssetService connector)
- {
- lock (m_endpointSerializer)
- {
- object eplock = null;
-
- if (! m_endpointSerializer.TryGetValue(connector, out eplock))
- {
- eplock = new object();
- m_endpointSerializer.Add(connector, eplock);
- // m_log.WarnFormat("[WEB UTIL] add a new host to end point serializer {0}",endpoint);
- }
-
- return eplock;
- }
- }
-
- private Dictionary m_connectors = new Dictionary();
+ private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
+ private ExpiringCacheOS m_connectors = new ExpiringCacheOS(120000);
public HGAssetServiceConnector(IConfigSource source)
{
@@ -83,16 +62,12 @@ namespace OpenSim.Services.Connectors
}
}
- private IAssetService GetConnector(string url)
+ private AssetServicesConnector GetConnector(string url)
{
- IAssetService connector = null;
+ AssetServicesConnector connector = null;
lock (m_connectors)
{
- if (m_connectors.ContainsKey(url))
- {
- connector = m_connectors[url];
- }
- else
+ if (!m_connectors.TryGetValue(url, 120000, out connector))
{
// Still not as flexible as I would like this to be,
// but good enough for now
@@ -108,7 +83,7 @@ namespace OpenSim.Services.Connectors
string url = string.Empty;
string assetID = string.Empty;
- if (Util.ParseForeignAssetID(id, out url, out assetID))
+ if (Util.ParseForeignAssetID(id, out url, out assetID) > 0)
{
IAssetService connector = GetConnector(url);
return connector.Get(assetID);
@@ -122,7 +97,7 @@ namespace OpenSim.Services.Connectors
string url = string.Empty;
string assetID = string.Empty;
- if (Util.ParseForeignAssetID(id, out url, out assetID))
+ if (Util.ParseForeignAssetID(id, out url, out assetID) > 0)
{
IAssetService connector = GetConnector(url);
return connector.GetCached(assetID);
@@ -136,7 +111,7 @@ namespace OpenSim.Services.Connectors
string url = string.Empty;
string assetID = string.Empty;
- if (Util.ParseForeignAssetID(id, out url, out assetID))
+ if (Util.ParseForeignAssetID(id, out url, out assetID) > 0)
{
IAssetService connector = GetConnector(url);
return connector.GetMetadata(assetID);
@@ -150,12 +125,12 @@ namespace OpenSim.Services.Connectors
return null;
}
- public bool Get(string id, Object sender, AssetRetrieved handler)
+ public bool Get(string id, object sender, AssetRetrieved handler)
{
string url = string.Empty;
string assetID = string.Empty;
- if (Util.ParseForeignAssetID(id, out url, out assetID))
+ if (Util.ParseForeignAssetID(id, out url, out assetID) > 0)
{
IAssetService connector = GetConnector(url);
return connector.Get(assetID, sender, handler);
@@ -164,7 +139,6 @@ namespace OpenSim.Services.Connectors
return false;
}
-
private struct AssetAndIndex
{
public UUID assetID;
@@ -191,7 +165,7 @@ namespace OpenSim.Services.Connectors
string url = string.Empty;
string assetID = string.Empty;
- if (Util.ParseForeignAssetID(ids[i], out url, out assetID))
+ if (Util.ParseForeignAssetID(ids[i], out url, out assetID) > 0)
{
if (!url2assets.ContainsKey(url))
url2assets.Add(url, new List());
@@ -205,8 +179,8 @@ namespace OpenSim.Services.Connectors
foreach (string url in url2assets.Keys)
{
- IAssetService connector = GetConnector(url);
- lock (EndPointLock(connector))
+ AssetServicesConnector connector = GetConnector(url);
+ lock (connector.ConnectorLock)
{
List curAssets = url2assets[url];
string[] assetIDs = curAssets.ConvertAll(a => a.assetID.ToString()).ToArray();
@@ -229,16 +203,16 @@ namespace OpenSim.Services.Connectors
string url = string.Empty;
string assetID = string.Empty;
- if (Util.ParseForeignAssetID(asset.ID, out url, out assetID))
+ if (Util.ParseForeignAssetID(asset.ID, out url, out assetID) > 0)
{
- IAssetService connector = GetConnector(url);
+ AssetServicesConnector connector = GetConnector(url);
// Restore the assetID to a simple UUID
asset.ID = assetID;
- lock (EndPointLock(connector))
+ lock ((connector.ConnectorLock))
return connector.Store(asset);
}
- return String.Empty;
+ return string.Empty;
}
public bool UpdateContent(string id, byte[] data)