ogb912/opensim
1
0
Fork 0
mirror of https://github.com/opensim/opensim.git synced 2026-08-12 20:55:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

make AssetServicesConnector do HG work (HGAssetServicesConnector should no not be in used now); change HGAssetBroker to be a extended derivative of it. WARNING TESTgit push origin master possible still very broken

Browse Source
This commit is contained in:
UbitUmarov
2020-10-23 19:43:31 +01:00
parent 56d3e1cdc9
commit b3893b9dc5
8 changed files with 616 additions and 255 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
OpenSim/Framework/GridInfo.cs
View File

@@ -50,8 +50,9 @@ namespace OpenSim.Framework
public int Port;
public IPAddress IP;
public readonly string Host;
public readonly string URI;
public readonly string URL;
public readonly string Path;
public readonly string URI;
public OSHTTPURI(string uri, bool withDNSResolve = false)
{
@@ -60,6 +61,7 @@ namespace OpenSim.Framework
IP = null;
Host = string.Empty;
URI = string.Empty;
URL = string.Empty;
Path = string.Empty;
if (string.IsNullOrEmpty(uri))
@@ -80,7 +82,8 @@ namespace OpenSim.Framework
if (Path[Path.Length - 1] == '/')
Path = Path.Substring(0, Path.Length - 1);
URI = m_checkuri.Scheme + "://" + Host + ":" + Port + Path;
URL = m_checkuri.Scheme + "://" + Host + ":" + Port;
URI = URL + Path;
if (withDNSResolve)
{

49
OpenSim/Framework/Util.cs
View File

@@ -1485,30 +1485,41 @@ namespace OpenSim.Framework
/// <param name="url">The URL: http://grid.example.com:8002</param>
/// <param name="assetID">The asset ID: 00000000-0000-0000-0000-000000000000. Returned even if 'id' isn't foreign.</param>
/// <returns>True: this is a foreign asset ID; False: it isn't</returns>
public static bool ParseForeignAssetID(string id, out string url, out string assetID)
public static int ParseForeignAssetID(string id, out string url, out string assetID)
{
url = String.Empty;
assetID = String.Empty;
url = string.Empty;
assetID = string.Empty;
if (UUID.TryParse(id, out UUID uuid))
if (id.Length == 0)
return -1;
if(id[0] != 'h' && id[0] != 'H')
{
assetID = uuid.ToString();
return false;
if (UUID.TryParse(id, out UUID luuid))
{
assetID = id;
return 0;
}
return -1;
}
if ((id.Length == 0) || (id[0] != 'h' && id[0] != 'H'))
return false;
if (!Uri.TryCreate(id, UriKind.Absolute, out Uri assetUri) || assetUri.Scheme != Uri.UriSchemeHttp)
return false;
url = "http://" + assetUri.Authority;
assetID = assetUri.LocalPath.Trim(new char[] { '/' });
if (!UUID.TryParse(assetID, out uuid))
return false;
return true;
OSHTTPURI uri = new OSHTTPURI(id, true);
if(uri.IsResolvedHost)
{
url = uri.URL;
string tmp = uri.Path;
if(tmp.Length < 36)
return -3;
if(tmp[0] =='/')
tmp = tmp.Substring(1);
if (UUID.TryParse(tmp, out UUID uuid))
{
assetID = tmp;
return 1;
}
return -1;
}
return -2;
}
/// <summary>

3
OpenSim/Region/CoreModules/Framework/InventoryAccess/HGAssetMapper.cs
View File

@@ -147,7 +147,7 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess
asset1.Data = asset.Data;
string id = m_scene.AssetService.Store(asset1);
if (String.IsNullOrEmpty(id))
if (string.IsNullOrEmpty(id))
{
if (verbose)
m_log.DebugFormat("[HG ASSET MAPPER]: Asset server {0} did not accept {1}", url, asset.ID);
@@ -169,7 +169,6 @@ namespace OpenSim.Region.CoreModules.Framework.InventoryAccess
to.Name = from.Name;
to.Temporary = from.Temporary;
to.Type = from.Type;
}
private void AdjustIdentifiers(AssetMetadata meta)

2
OpenSim/Region/CoreModules/ServiceConnectorsIn/Asset/AssetServiceInConnectorModule.cs
View File

@@ -96,7 +96,7 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsIn.Asset
m_log.Info("[HGAssetService]: Starting...");
Object[] args = new Object[] { m_Config, MainServer.Instance, "HGAssetService" };
object[] args = new object[] { m_Config, MainServer.Instance, "HGAssetService" };
ServerUtils.LoadPlugin<IServiceConnector>("OpenSim.Server.Handlers.dll:AssetServiceConnector", args);
}

8
OpenSim/Region/CoreModules/ServiceConnectorsIn/Hypergrid/HypergridServiceInConnectorModule.cs
View File

@@ -51,7 +51,7 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsIn.Hypergrid
private IConfigSource m_Config;
private bool m_Registered = false;
private string m_LocalServiceDll = String.Empty;
private string m_LocalServiceDll = string.Empty;
private GatekeeperServiceInConnector m_HypergridHandler;
private UserAgentServerConnector m_UASHandler;
@@ -71,13 +71,11 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsIn.Hypergrid
if (fconfig != null)
{
m_LocalServiceDll = fconfig.GetString("LocalServiceModule", m_LocalServiceDll);
if (m_LocalServiceDll == String.Empty)
if (m_LocalServiceDll == string.Empty)
m_log.WarnFormat("[HGGRID IN CONNECTOR]: Friends LocalServiceModule config missing");
}
}
}
}
public void PostInitialise()
@@ -123,7 +121,7 @@ namespace OpenSim.Region.CoreModules.ServiceConnectorsIn.Hypergrid
ISimulationService simService = scene.RequestModuleInterface<ISimulationService>();
IFriendsSimConnector friendsConn = scene.RequestModuleInterface<IFriendsSimConnector>();
Object[] args = new Object[] { m_Config };
object[] args = new object[] { m_Config };
// IFriendsService friendsService = ServerUtils.LoadPlugin<IFriendsService>(m_LocalServiceDll, args)
ServerUtils.LoadPlugin<IFriendsService>(m_LocalServiceDll, args);

4
OpenSim/Services/AssetService/AssetService.cs
View File

@@ -40,9 +40,7 @@ namespace OpenSim.Services.AssetService
{
public class AssetService : AssetServiceBase, IAssetService
{
private static readonly ILog m_log =
LogManager.GetLogger(
MethodBase.GetCurrentMethod().DeclaringType);
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
protected static AssetService m_RootInstance;

738
OpenSim/Services/Connectors/Asset/AssetServicesConnector.cs
View File

@@ -43,29 +43,30 @@ namespace OpenSim.Services.Connectors
{
public class AssetServicesConnector : BaseServiceConnector, IAssetService
{
private static readonly ILog m_log =
LogManager.GetLogger(
MethodBase.GetCurrentMethod().DeclaringType);
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
const int MAXSENDRETRIESLEN = 30;
public object ConnectorLock = new object();
private string m_ServerURI = String.Empty;
private IAssetCache m_Cache = null;
private int m_retryCounter;
private bool m_inRetries;
private List<AssetBase>[] m_sendRetries = new List<AssetBase>[MAXSENDRETRIESLEN];
private List<AssetBase>[] m_sendRetries = new List<AssetBase>[MAXSENDRETRIESLEN];
private List<string>[] m_sendCachedRetries = new List<string>[MAXSENDRETRIESLEN];
private System.Timers.Timer m_retryTimer;
private int m_maxAssetRequestConcurrency = 30;
private int m_maxAssetRequestConcurrency = 8;
private delegate void AssetRetrievedEx(AssetBase asset);
// Keeps track of concurrent requests for the same asset, so that it's only loaded once.
// Maps: Asset ID -> Handlers which will be called when the asset has been loaded
// private Dictionary<string, AssetRetrievedEx> m_AssetHandlers = new Dictionary<string, AssetRetrievedEx>();
private Dictionary<string, List<AssetRetrievedEx>> m_AssetHandlers = new Dictionary<string, List<AssetRetrievedEx>>();
private Dictionary<string, string> m_UriMap = new Dictionary<string, string>();
private Dictionary<string, string> m_UriMap;
private Thread[] m_fetchThreads;
@@ -84,8 +85,7 @@ namespace OpenSim.Services.Connectors
m_ServerURI = serverURI.TrimEnd('/');
}
public AssetServicesConnector(IConfigSource source)
: base(source, "AssetService")
public AssetServicesConnector(IConfigSource source)
{
Initialise(source);
}
@@ -103,64 +103,78 @@ namespace OpenSim.Services.Connectors
throw new Exception("Asset connector init error");
}
m_ServerURI = assetConfig.GetString("AssetServerURI", String.Empty);
if (m_ServerURI == String.Empty)
m_ServerURI = assetConfig.GetString("AssetServerURI", string.Empty);
if (string.IsNullOrEmpty(m_ServerURI))
{
m_log.Error("[ASSET CONNECTOR]: No Server URI named in section AssetService");
IConfig netConfig = source.Configs["Network"];
m_ServerURI = netConfig.GetString("asset_server_url", string.Empty);
}
if (string.IsNullOrEmpty(m_ServerURI))
{
m_log.Error("[ASSET CONNECTOR]: AssetServerURI not defined in section AssetService");
throw new Exception("Asset connector init error");
}
OSHHTPHost m_GridAssetsURL = new OSHHTPHost(m_ServerURI, true);
if(!m_GridAssetsURL.IsResolvedHost)
{
m_log.Error("[ASSET CONNECTOR]: Could not parse or resolve AssetServerURI");
throw new Exception("Asset connector init error");
}
m_ServerURI = m_GridAssetsURL.URI;
bool usemaps = assetConfig.GetBoolean("AssetServerIsCluster", false);
if(usemaps)
{
m_UriMap = new Dictionary<string, string>();
for (int i = 0; i < 256; i++)
{
string prefix = i.ToString("x2");
string groupHost = assetConfig.GetString("AssetServerHost_" + prefix, string.Empty);
if(string.IsNullOrEmpty(groupHost))
m_UriMap[prefix] = m_ServerURI;
else
{
OSHHTPHost other = new OSHHTPHost(groupHost, true);
if(!other.IsResolvedHost)
{
m_log.Error("[ASSET CONNECTOR]: Could not parse or resolve AssetServerHost_" + prefix);
throw new Exception("Asset connector init error");
}
m_UriMap[prefix] = other.URI;
}
}
}
else
m_UriMap = null;
m_retryTimer = new System.Timers.Timer();
m_retryTimer.Elapsed += new ElapsedEventHandler(retryCheck);
m_retryTimer.AutoReset = true;
m_retryTimer.Interval = 60000;
Uri serverUri = new Uri(m_ServerURI);
m_fetchThreads = new Thread[3];
string groupHost = serverUri.Host;
Initialise(source, "AssetService");
for (int i = 0 ; i < 256 ; i++)
for (int i = 0 ; i < m_fetchThreads.Length; i++)
{
string prefix = i.ToString("x2");
groupHost = assetConfig.GetString("AssetServerHost_"+prefix, groupHost);
m_UriMap[prefix] = groupHost;
//m_log.DebugFormat("[ASSET]: Using {0} for prefix {1}", groupHost, prefix);
}
m_fetchThreads = new Thread[2];
for (int i = 0 ; i < 2 ; i++)
{
m_fetchThreads[i] = WorkManager.StartThread(AssetRequestProcessor, String.Format("GetAssetsWorker{0}", i));
m_fetchThreads[i] = WorkManager.StartThread(AssetRequestProcessor, string.Format("GetAssetsWorker{0}", i));
}
}
private string MapServer(string id)
{
if (m_UriMap.Count == 0)
if (m_UriMap == null)
return m_ServerURI;
UriBuilder serverUri = new UriBuilder(m_ServerURI);
string prefix = id.Substring(0, 2).ToLower();
string host;
if (m_UriMap.TryGetValue(prefix, out string host))
return host;
// HG URLs will not be valid UUIDS
if (m_UriMap.ContainsKey(prefix))
host = m_UriMap[prefix];
else
host = m_UriMap["00"];
serverUri.Host = host;
// m_log.DebugFormat("[ASSET]: Using {0} for host name for prefix {1}", host, prefix);
string ret = serverUri.Uri.AbsoluteUri;
if (ret.EndsWith("/"))
ret = ret.Substring(0, ret.Length - 1);
return ret;
return m_UriMap["00"];
}
protected void retryCheck(object source, ElapsedEventArgs e)
@@ -179,48 +193,94 @@ namespace OpenSim.Services.Connectors
int inUse = 0;
int nextlevel;
int timefactor;
List<AssetBase> retrylist;
// we need to go down
for(int i = MAXSENDRETRIESLEN - 1; i >= 0; i--)
if(m_Cache == null)
{
lock(m_sendRetries)
retrylist = m_sendRetries[i];
if(retrylist == null)
continue;
inUse++;
nextlevel = i + 1;
//We exponentially fall back on frequency until we reach one attempt per hour
//The net result is that we end up in the queue for roughly 24 hours..
//24 hours worth of assets could be a lot, so the hope is that the region admin
//will have gotten the asset connector back online quickly!
if(i == 0)
timefactor = 1;
else
List<AssetBase> retrylist;
// we need to go down
for(int i = MAXSENDRETRIESLEN - 1; i >= 0; i--)
{
timefactor = 1 << nextlevel;
if (timefactor > 60)
timefactor = 60;
lock(m_sendRetries)
retrylist = m_sendRetries[i];
if(retrylist == null)
continue;
inUse++;
nextlevel = i + 1;
//We exponentially fall back on frequency until we reach one attempt per hour
//The net result is that we end up in the queue for roughly 24 hours..
//24 hours worth of assets could be a lot, so the hope is that the region admin
//will have gotten the asset connector back online quickly!
if(i == 0)
timefactor = 1;
else
{
timefactor = 1 << nextlevel;
if (timefactor > 60)
timefactor = 60;
}
if(m_retryCounter < timefactor)
continue; // to update inUse;
if (m_retryCounter % timefactor != 0)
continue;
// a list to retry
lock(m_sendRetries)
m_sendRetries[i] = null;
// we are the only ones with a copy of this retrylist now
foreach(AssetBase ass in retrylist)
retryStore(ass, nextlevel);
}
}
else
{
List<string> retrylist;
// we need to go down
for (int i = MAXSENDRETRIESLEN - 1; i >= 0; i--)
{
lock (m_sendRetries)
retrylist = m_sendCachedRetries[i];
if(m_retryCounter < timefactor)
continue; // to update inUse;
if (retrylist == null)
continue;
if (m_retryCounter % timefactor != 0)
continue;
inUse++;
nextlevel = i + 1;
// a list to retry
lock(m_sendRetries)
m_sendRetries[i] = null;
//We exponentially fall back on frequency until we reach one attempt per hour
//The net result is that we end up in the queue for roughly 24 hours..
//24 hours worth of assets could be a lot, so the hope is that the region admin
//will have gotten the asset connector back online quickly!
if (i == 0)
timefactor = 1;
else
{
timefactor = 1 << nextlevel;
if (timefactor > 60)
timefactor = 60;
}
// we are the only ones with a copy of this retrylist now
foreach(AssetBase ass in retrylist)
retryStore(ass, nextlevel);
if (m_retryCounter < timefactor)
continue; // to update inUse;
if (m_retryCounter % timefactor != 0)
continue;
// a list to retry
lock (m_sendRetries)
m_sendCachedRetries[i] = null;
// we are the only ones with a copy of this retrylist now
foreach (string id in retrylist)
retryCachedStore(id, nextlevel);
}
}
lock(m_sendRetries)
lock (m_sendRetries)
{
if(inUse == 0 )
m_retryTimer.Stop();
@@ -229,12 +289,23 @@ namespace OpenSim.Services.Connectors
}
}
protected void SetCache(IAssetCache cache)
public void SetCache(IAssetCache cache)
{
m_Cache = cache;
}
public AssetBase Get(string id)
public AssetBase GetCached(string id)
{
AssetBase asset = null;
if (m_Cache != null)
{
m_Cache.Get(id, out asset);
}
return asset;
}
public virtual AssetBase Get(string id)
{
AssetBase asset = null;
if (m_Cache != null)
@@ -246,6 +317,7 @@ namespace OpenSim.Services.Connectors
if (asset == null || asset.Data == null || asset.Data.Length == 0)
{
string uri = MapServer(id) + "/assets/" + id;
asset = SynchronousRestObjectRequester.MakeRequest<int, AssetBase>("GET", uri, 0, m_Auth);
if (m_Cache != null)
{
@@ -258,20 +330,39 @@ namespace OpenSim.Services.Connectors
return asset;
}
public AssetBase GetCached(string id)
public AssetBase GetForeign(string id)
{
// m_log.DebugFormat("[ASSET SERVICE CONNECTOR]: Cache request for {0}", id);
int type = Util.ParseForeignAssetID(id, out string uri, out string uuidstr);
if (type < 0)
return null;
AssetBase asset = null;
if (m_Cache != null)
{
m_Cache.Get(id, out asset);
if (!m_Cache.Get(uuidstr, out asset))
return null;
}
if (asset == null || asset.Data == null || asset.Data.Length == 0)
{
if (type == 0)
uri = MapServer(uuidstr) + "/assets/" + uuidstr;
else
uri = uri + "/assets/" + uuidstr;
asset = SynchronousRestObjectRequester.MakeRequest<int, AssetBase>("GET", uri, 0, m_Auth);
if (m_Cache != null)
{
if (asset != null)
m_Cache.Cache(asset);
else
m_Cache.CacheNegative(id);
}
}
return asset;
}
public AssetMetadata GetMetadata(string id)
public virtual AssetMetadata GetMetadata(string id)
{
if (m_Cache != null)
{
@@ -289,40 +380,87 @@ namespace OpenSim.Services.Connectors
return asset;
}
public byte[] GetData(string id)
public AssetMetadata GetForeignMetadata(string id)
{
int type = Util.ParseForeignAssetID(id, out string uri, out string uuidstr);
if (type < 0)
return null;
if (m_Cache != null)
{
AssetBase fullAsset;
if (!m_Cache.Get(id, out fullAsset))
if (!m_Cache.Get(uuidstr, out fullAsset))
return null;
if (fullAsset != null)
return fullAsset.Metadata;
}
if (type == 0)
uri = MapServer(uuidstr) + "/assets/" + uuidstr + "/metadata";
else
uri = uri + "/assets/" + uuidstr + "/metadata";
AssetMetadata asset = SynchronousRestObjectRequester.MakeRequest<int, AssetMetadata>("GET", uri, 0, m_Auth);
return asset;
}
public virtual byte[] GetData(string id)
{
if (m_Cache != null)
{
if (!m_Cache.Get(id, out AssetBase fullAsset))
return null;
if (fullAsset != null)
return fullAsset.Data;
}
using (RestClient rc = new RestClient(MapServer(id)))
{
rc.AddResourcePath("assets");
rc.AddResourcePath(id);
rc.AddResourcePath("data");
string uri = MapServer(id);
using (RestClient rc = new RestClient(uri))
{
rc.AddResourcePath("assets/" + id + "/Data");
rc.RequestMethod = "GET";
using (Stream s = rc.Request(m_Auth))
using (MemoryStream s = rc.Request(m_Auth))
{
if (s == null)
if (s == null || s.Length == 0)
return null;
if (s.Length > 0)
{
byte[] ret = new byte[s.Length];
s.Read(ret, 0, (int)s.Length);
return ret;
}
return s.ToArray();
}
}
}
public byte[] GetForeignData(string id)
{
int type = Util.ParseForeignAssetID(id, out string uri, out string uuidstr);
if (type < 0)
return null;
if (m_Cache != null)
{
if (!m_Cache.Get(uuidstr, out AssetBase fullAsset))
return null;
if (fullAsset != null)
return fullAsset.Data;
}
if (type == 0)
uri = MapServer(uuidstr);
using (RestClient rc = new RestClient(uri))
{
rc.AddResourcePath("assets/" + id + "/Data");
rc.RequestMethod = "GET";
using (MemoryStream s = rc.Request(m_Auth))
{
if (s == null || s.Length == 0)
return null;
return s.ToArray();
}
}
}
@@ -332,8 +470,106 @@ namespace OpenSim.Services.Connectors
public string id;
}
private BlockingCollection<QueuedAssetRequest> m_requestQueue = new BlockingCollection<QueuedAssetRequest>();
public virtual bool Get(string id, object sender, AssetRetrieved handler)
{
AssetBase asset = null;
if (m_Cache != null)
{
if (!m_Cache.Get(id, out asset))
return false;
}
if (asset == null)
{
string uri = MapServer(id) + "/assets/" + id;
lock (m_AssetHandlers)
{
AssetRetrievedEx handlerEx = new AssetRetrievedEx(delegate (AssetBase _asset) { handler(id, sender, _asset); });
List<AssetRetrievedEx> handlers;
if (m_AssetHandlers.TryGetValue(id, out handlers))
{
// Someone else is already loading this asset. It will notify our handler when done.
handlers.Add(handlerEx);
return true;
}
handlers = new List<AssetRetrievedEx>();
handlers.Add(handlerEx);
m_AssetHandlers.Add(id, handlers);
QueuedAssetRequest request = new QueuedAssetRequest();
request.id = id;
request.uri = uri;
m_requestQueue.Add(request);
}
}
else
{
if (asset != null && (asset.Data == null || asset.Data.Length == 0))
asset = null;
handler(id, sender, asset);
}
return true;
}
public bool GetForeign(string id, object sender, AssetRetrieved handler)
{
int type = Util.ParseForeignAssetID(id, out string uri, out string uuidstr);
if (type < 0)
return false;
AssetBase asset = null;
if (m_Cache != null)
{
if (!m_Cache.Get(uuidstr, out asset))
return false;
}
if (asset == null)
{
if (type == 0)
uri = MapServer(uuidstr) + "/assets/" + uuidstr;
else
uri = uri + "/assets/" + uuidstr;
lock (m_AssetHandlers)
{
AssetRetrievedEx handlerEx = new AssetRetrievedEx(delegate (AssetBase _asset) { handler(id, sender, _asset); });
List<AssetRetrievedEx> handlers;
if (m_AssetHandlers.TryGetValue(id, out handlers))
{
// Someone else is already loading this asset. It will notify our handler when done.
handlers.Add(handlerEx);
return true;
}
handlers = new List<AssetRetrievedEx>();
handlers.Add(handlerEx);
m_AssetHandlers.Add(id, handlers);
QueuedAssetRequest request = new QueuedAssetRequest();
request.id = id;
request.uri = uri;
m_requestQueue.Add(request);
}
}
else
{
if (asset != null && (asset.Data == null || asset.Data.Length == 0))
asset = null;
handler(id, sender, asset);
}
return true;
}
private BlockingCollection<QueuedAssetRequest> m_requestQueue = new BlockingCollection<QueuedAssetRequest>();
private void AssetRequestProcessor()
{
QueuedAssetRequest r;
@@ -347,12 +583,11 @@ namespace OpenSim.Services.Connectors
}
Watchdog.UpdateThread();
string uri = r.uri;
string id = r.id;
try
{
AssetBase a = SynchronousRestObjectRequester.MakeRequest<int, AssetBase>("GET", uri, 0, 30000, m_Auth);
AssetBase a = SynchronousRestObjectRequester.MakeRequest<int, AssetBase>("GET", r.uri, 0, 30000, m_Auth);
if (a != null && m_Cache != null)
m_Cache.Cache(a);
@@ -381,50 +616,6 @@ namespace OpenSim.Services.Connectors
}
}
public bool Get(string id, Object sender, AssetRetrieved handler)
{
string uri = MapServer(id) + "/assets/" + id;
AssetBase asset = null;
if (m_Cache != null)
{
if (!m_Cache.Get(id, out asset))
return false;
}
if (asset == null || asset.Data == null || asset.Data.Length == 0)
{
lock (m_AssetHandlers)
{
AssetRetrievedEx handlerEx = new AssetRetrievedEx(delegate(AssetBase _asset) { handler(id, sender, _asset); });
List<AssetRetrievedEx> handlers;
if (m_AssetHandlers.TryGetValue(id, out handlers))
{
// Someone else is already loading this asset. It will notify our handler when done.
handlers.Add(handlerEx);
return true;
}
handlers = new List<AssetRetrievedEx>();
handlers.Add(handlerEx);
m_AssetHandlers.Add(id, handlers);
QueuedAssetRequest request = new QueuedAssetRequest();
request.id = id;
request.uri = uri;
m_requestQueue.Add(request);
}
}
else
{
handler(id, sender, asset);
}
return true;
}
public virtual bool[] AssetsExist(string[] ids)
{
string uri = m_ServerURI + "/get_assets_exist";
@@ -446,9 +637,80 @@ namespace OpenSim.Services.Connectors
return exist;
}
private struct AssetAndIndex
{
public string assetID;
public int index;
public AssetAndIndex(string assetID, int index)
{
this.assetID = assetID;
this.index = index;
}
}
public bool[] ForeignAssetsExist(string[] ids)
{
bool[] exist = new bool[ids.Length];
var url2assets = new Dictionary<string, List<AssetAndIndex>>();
for (int i = 0; i < ids.Length; i++)
{
int ltype = Util.ParseForeignAssetID(ids[i], out string lurl, out string luuidstr);
if (ltype > 0)
{
if (ltype == 0)
lurl = m_ServerURI;
List < AssetAndIndex > lst;
if (!url2assets.TryGetValue(lurl, out lst))
{
lst = new List<AssetAndIndex>();
url2assets.Add(lurl, lst);
}
lst.Add(new AssetAndIndex(luuidstr, i));
}
}
// Query each of the servers in turn
foreach (KeyValuePair<string, List<AssetAndIndex>> kvp in url2assets)
{
List<AssetAndIndex> curAssets = kvp.Value;
string[] assetIDs = new string[curAssets.Count];
for(int i = 0; i < assetIDs.Length;++i)
assetIDs[i] = curAssets[i].assetID;
string uri = kvp.Key + "/get_assets_exist";
bool[] curExist = null;
try
{
curExist = SynchronousRestObjectRequester.MakeRequest<string[], bool[]>("POST", uri, assetIDs, m_Auth);
}
catch (Exception)
{
// This is most likely to happen because the server doesn't support this function,
// so just silently return "doesn't exist" for all the assets.
}
if(curExist != null)
{
int i = 0;
foreach (AssetAndIndex ai in curAssets)
{
exist[ai.index] = curExist[i];
++i;
}
}
}
return exist;
}
string stringUUIDZero = UUID.Zero.ToString();
public string Store(AssetBase asset)
public virtual string Store(AssetBase asset)
{
// Have to assign the asset ID here. This isn't likely to
// trigger since current callers don't pass emtpy IDs
@@ -492,23 +754,31 @@ namespace OpenSim.Services.Connectors
string newID = null;
try
{
newID = SynchronousRestObjectRequester.
MakeRequest<AssetBase, string>("POST", uri, asset, 10000, m_Auth);
newID = SynchronousRestObjectRequester.MakeRequest<AssetBase, string>("POST", uri, asset, 10000, m_Auth);
}
catch
{
newID = null;
}
if (newID == null || newID == String.Empty || newID == stringUUIDZero)
if (string.IsNullOrEmpty(newID) || newID == stringUUIDZero)
{
//The asset upload failed, try later
lock(m_sendRetries)
{
if (m_sendRetries[0] == null)
m_sendRetries[0] = new List<AssetBase>();
List<AssetBase> m_queue = m_sendRetries[0];
m_queue.Add(asset);
if(m_Cache == null)
{
if (m_sendRetries[0] == null)
m_sendRetries[0] = new List<AssetBase>();
m_sendRetries[0].Add(asset);
}
else
{
if (m_sendCachedRetries[0] == null)
m_sendCachedRetries[0] = new List<string>();
m_sendCachedRetries[0].Add(asset.ID);
}
m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later",
asset.ID.ToString(), asset.Type.ToString());
m_retryTimer.Start();
@@ -520,26 +790,138 @@ namespace OpenSim.Services.Connectors
{
// Placing this here, so that this work with old asset servers that don't send any reply back
// SynchronousRestObjectRequester returns somethins that is not an empty string
asset.ID = newID;
if (m_Cache != null)
m_Cache.Cache(asset);
}
}
return asset.ID;
}
public virtual string StoreForeign(AssetBase asset)
{
int type = Util.ParseForeignAssetID(asset.ID, out string uri, out string uuidstr);
if(type < 0)
return string.Empty;
if(type != 0)
asset.ID = uuidstr;
if (m_Cache != null)
m_Cache.Cache(asset);
if (asset.Temporary || asset.Local)
{
return asset.ID;
}
if(type == 0)
uri = MapServer(uuidstr) + "/assets/";
else
uri += "/assets/";
string newID = null;
try
{
newID = SynchronousRestObjectRequester.MakeRequest<AssetBase, string>("POST", uri, asset, 10000, m_Auth);
}
catch
{
newID = null;
}
if (string.IsNullOrEmpty(newID) || newID == stringUUIDZero)
{
//The asset upload failed, try later
lock (m_sendRetries)
{
if (m_Cache == null)
{
if (m_sendRetries[0] == null)
m_sendRetries[0] = new List<AssetBase>();
m_sendRetries[0].Add(asset);
}
else
{
if (m_sendCachedRetries[0] == null)
m_sendCachedRetries[0] = new List<string>();
m_sendCachedRetries[0].Add(asset.ID);
}
m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later",
asset.ID.ToString(), asset.Type.ToString());
m_retryTimer.Start();
}
}
else
{
if (newID != asset.ID)
{
// Placing this here, so that this work with old asset servers that don't send any reply back
// SynchronousRestObjectRequester returns somethins that is not an empty string
asset.ID = newID;
if (m_Cache != null)
m_Cache.Cache(asset);
}
}
return asset.ID;
}
public void retryStore(AssetBase asset, int nextRetryLevel)
{
/* this may be bad, so excluding
if (m_Cache != null && !m_Cache.Check(asset.ID))
string uri = MapServer(asset.FullID.ToString()) + "/assets/";
string newID = null;
try
{
m_log.WarnFormat("[Assets] Upload giveup asset bc no longer in local cache: {0}",
asset.ID.ToString();
return; // if no longer in cache, it was deleted or expired
newID = SynchronousRestObjectRequester.MakeRequest<AssetBase, string>("POST", uri, asset, 100000, m_Auth);
}
*/
catch
{
newID = null;
}
if (string.IsNullOrEmpty(newID) || newID == stringUUIDZero)
{
if (nextRetryLevel >= MAXSENDRETRIESLEN)
m_log.WarnFormat("[Assets] Giving up on uploading after {2} retries id: {0} type {1}",
asset.ID.ToString(), asset.Type.ToString(), MAXSENDRETRIESLEN);
else
{
lock (m_sendRetries)
{
if (m_sendRetries[nextRetryLevel] == null)
m_sendRetries[nextRetryLevel] = new List<AssetBase>();
List<AssetBase> m_queue = m_sendRetries[nextRetryLevel];
m_queue.Add(asset);
m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later",
asset.ID.ToString(), asset.Type.ToString());
}
}
}
else
{
m_log.InfoFormat("[Assets] Upload of {0} succeeded after {1} failed attempts", asset.ID.ToString(), nextRetryLevel.ToString());
if (newID != asset.ID)
{
asset.ID = newID;
m_Cache?.Cache(asset);
}
}
}
public void retryCachedStore(string assetID, int nextRetryLevel)
{
m_Cache.Get(assetID,out AssetBase asset);
if(asset == null)
{
m_log.WarnFormat("[Assets] asset not in cache on uploading after {2} retries id: {0}",
assetID, MAXSENDRETRIESLEN);
}
string uri = MapServer(asset.FullID.ToString()) + "/assets/";
string newID = null;
@@ -553,23 +935,21 @@ namespace OpenSim.Services.Connectors
newID = null;
}
if (newID == null || newID == String.Empty || newID == stringUUIDZero)
if (string.IsNullOrEmpty(newID) || newID == stringUUIDZero)
{
if(nextRetryLevel >= MAXSENDRETRIESLEN)
if (nextRetryLevel >= MAXSENDRETRIESLEN)
m_log.WarnFormat("[Assets] Giving up on uploading after {2} retries id: {0} type {1}",
asset.ID.ToString(), asset.Type.ToString(), MAXSENDRETRIESLEN);
else
{
lock(m_sendRetries)
lock (m_sendRetries)
{
if (m_sendRetries[nextRetryLevel] == null)
{
m_sendRetries[nextRetryLevel] = new List<AssetBase>();
}
List<AssetBase> m_queue = m_sendRetries[nextRetryLevel];
m_queue.Add(asset);
m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later",
asset.ID.ToString(), asset.Type.ToString());
if (m_sendCachedRetries[nextRetryLevel] == null)
m_sendCachedRetries[nextRetryLevel] = new List<string>();
m_sendCachedRetries[nextRetryLevel].Add(assetID);
m_log.WarnFormat("[Assets] Upload failed: {0} type {1} will retry later",
asset.ID.ToString(), asset.Type.ToString());
}
}
}
@@ -578,14 +958,13 @@ namespace OpenSim.Services.Connectors
m_log.InfoFormat("[Assets] Upload of {0} succeeded after {1} failed attempts", asset.ID.ToString(), nextRetryLevel.ToString());
if (newID != asset.ID)
{
asset.ID = newID;
asset.ID = newID;
m_Cache?.Cache(asset);
}
}
m_Cache?.Cache(asset);
}
public bool UpdateContent(string id, byte[] data)
public virtual bool UpdateContent(string id, byte[] data)
{
AssetBase asset = null;
@@ -612,8 +991,7 @@ namespace OpenSim.Services.Connectors
return false;
}
public bool Delete(string id)
public virtual bool Delete(string id)
{
string uri = MapServer(id) + "/assets/" + id;

60
OpenSim/Services/Connectors/Asset/HGAssetServiceConnector.cs
View File

@@ -41,29 +41,8 @@ namespace OpenSim.Services.Connectors
{
public class HGAssetServiceConnector : IAssetService
{
private static readonly ILog m_log =
LogManager.GetLogger(
MethodBase.GetCurrentMethod().DeclaringType);
private Dictionary<IAssetService, object> m_endpointSerializer = new Dictionary<IAssetService, object>();
private object EndPointLock(IAssetService connector)
{
lock (m_endpointSerializer)
{
object eplock = null;
if (! m_endpointSerializer.TryGetValue(connector, out eplock))
{
eplock = new object();
m_endpointSerializer.Add(connector, eplock);
// m_log.WarnFormat("[WEB UTIL] add a new host to end point serializer {0}",endpoint);
}
return eplock;
}
}
private Dictionary<string, IAssetService> m_connectors = new Dictionary<string, IAssetService>();
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
private ExpiringCacheOS<string, AssetServicesConnector> m_connectors = new ExpiringCacheOS<string, AssetServicesConnector>(120000);
public HGAssetServiceConnector(IConfigSource source)
{
@@ -83,16 +62,12 @@ namespace OpenSim.Services.Connectors
}
}
private IAssetService GetConnector(string url)
private AssetServicesConnector GetConnector(string url)
{
IAssetService connector = null;
AssetServicesConnector connector = null;
lock (m_connectors)
{
if (m_connectors.ContainsKey(url))
{
connector = m_connectors[url];
}
else
if (!m_connectors.TryGetValue(url, 120000, out connector))
{
// Still not as flexible as I would like this to be,
// but good enough for now
@@ -108,7 +83,7 @@ namespace OpenSim.Services.Connectors
string url = string.Empty;
string assetID = string.Empty;
if (Util.ParseForeignAssetID(id, out url, out assetID))
if (Util.ParseForeignAssetID(id, out url, out assetID) > 0)
{
IAssetService connector = GetConnector(url);
return connector.Get(assetID);
@@ -122,7 +97,7 @@ namespace OpenSim.Services.Connectors
string url = string.Empty;
string assetID = string.Empty;
if (Util.ParseForeignAssetID(id, out url, out assetID))
if (Util.ParseForeignAssetID(id, out url, out assetID) > 0)
{
IAssetService connector = GetConnector(url);
return connector.GetCached(assetID);
@@ -136,7 +111,7 @@ namespace OpenSim.Services.Connectors
string url = string.Empty;
string assetID = string.Empty;
if (Util.ParseForeignAssetID(id, out url, out assetID))
if (Util.ParseForeignAssetID(id, out url, out assetID) > 0)
{
IAssetService connector = GetConnector(url);
return connector.GetMetadata(assetID);
@@ -150,12 +125,12 @@ namespace OpenSim.Services.Connectors
return null;
}
public bool Get(string id, Object sender, AssetRetrieved handler)
public bool Get(string id, object sender, AssetRetrieved handler)
{
string url = string.Empty;
string assetID = string.Empty;
if (Util.ParseForeignAssetID(id, out url, out assetID))
if (Util.ParseForeignAssetID(id, out url, out assetID) > 0)
{
IAssetService connector = GetConnector(url);
return connector.Get(assetID, sender, handler);
@@ -164,7 +139,6 @@ namespace OpenSim.Services.Connectors
return false;
}
private struct AssetAndIndex
{
public UUID assetID;
@@ -191,7 +165,7 @@ namespace OpenSim.Services.Connectors
string url = string.Empty;
string assetID = string.Empty;
if (Util.ParseForeignAssetID(ids[i], out url, out assetID))
if (Util.ParseForeignAssetID(ids[i], out url, out assetID) > 0)
{
if (!url2assets.ContainsKey(url))
url2assets.Add(url, new List<AssetAndIndex>());
@@ -205,8 +179,8 @@ namespace OpenSim.Services.Connectors
foreach (string url in url2assets.Keys)
{
IAssetService connector = GetConnector(url);
lock (EndPointLock(connector))
AssetServicesConnector connector = GetConnector(url);
lock (connector.ConnectorLock)
{
List<AssetAndIndex> curAssets = url2assets[url];
string[] assetIDs = curAssets.ConvertAll(a => a.assetID.ToString()).ToArray();
@@ -229,16 +203,16 @@ namespace OpenSim.Services.Connectors
string url = string.Empty;
string assetID = string.Empty;
if (Util.ParseForeignAssetID(asset.ID, out url, out assetID))
if (Util.ParseForeignAssetID(asset.ID, out url, out assetID) > 0)
{
IAssetService connector = GetConnector(url);
AssetServicesConnector connector = GetConnector(url);
// Restore the assetID to a simple UUID
asset.ID = assetID;
lock (EndPointLock(connector))
lock ((connector.ConnectorLock))
return connector.Store(asset);
}
return String.Empty;
return string.Empty;
}
public bool UpdateContent(string id, byte[] data)