ogb912/opensim
1
0
Fork 0
mirror of https://github.com/opensim/opensim.git synced 2026-08-06 01:06:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

oar load: do a block check for assets presence on grid services, instead of one by one

Browse Source
This commit is contained in:
UbitUmarov
2021-08-10 17:02:27 +01:00
parent 554ef1b3fd
commit cc86bdddcf
1 changed files with 124 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

211
OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs
View File

@@ -285,6 +285,91 @@ namespace OpenSim.Region.CoreModules.World.Archiver
m_assetService = m_rootScene.AssetService;
}
public struct assetfileInfo
{
public string path;
public byte[] data;
}
private void loadNeededAssets(assetfileInfo[] assetsFiles, int count, ref int successfulAssetRestores,
ref int failedAssetRestores, ref int skipedAssetRestores)
{
List<string> ids = new List<string>(count);
List<UUID> uuids = new List<UUID>(count);
List<sbyte> types = new List<sbyte>(count);
List<byte[]> datas = new List<byte[]>(count);
for (int i = 0; i < count; ++i)
{
string assetPath = assetsFiles[i].path;
byte[] data = assetsFiles[i].data;
assetsFiles[i].path = null;
assetsFiles[i].data = null;
string filename = assetPath.Substring(ArchiveConstants.ASSETS_PATH.Length);
int indx = filename.LastIndexOf(ArchiveConstants.ASSET_EXTENSION_SEPARATOR);
if (indx < 0)
{
m_log.ErrorFormat(
"[ARCHIVER]: Could not find extension information in asset path {0} since it's missing the separator {1}. Skipping",
assetPath, ArchiveConstants.ASSET_EXTENSION_SEPARATOR);
failedAssetRestores++;
continue;
}
string extension = filename.Substring(indx);
if (ArchiveConstants.EXTENSION_TO_ASSET_TYPE.ContainsKey(extension))
{
string id = filename.Remove(indx);
if (UUID.TryParse(id, out UUID uuid))
{
ids.Add(id);
uuids.Add(uuid);
datas.Add(data);
types.Add(ArchiveConstants.EXTENSION_TO_ASSET_TYPE[extension]);
continue;
}
}
failedAssetRestores++;
}
bool[] exits = m_assetService.AssetsExist(ids.ToArray());
ids.Clear();
if (exits == null)
{
m_log.Error("[ARCHIVER]: asset service AssetsExists failed");
failedAssetRestores += uuids.Count;
return;
}
if (exits.Length != uuids.Count)
{
m_log.Error("[ARCHIVER]: asset service AssetsExists return size mismatch");
failedAssetRestores += uuids.Count;
return;
}
for (int i = 0; i < uuids.Count; ++i)
{
if (exits[i])
{
++skipedAssetRestores;
}
else
{
if (TryUploadAsset(uuids[i],types[i], datas[i]))
successfulAssetRestores++;
else
failedAssetRestores++;
}
int tot = successfulAssetRestores + failedAssetRestores + skipedAssetRestores;
if (tot % 250 == 0)
m_log.Debug("[ARCHIVER]: done " + tot + "; uploaded: " + successfulAssetRestores + " skipped: " + skipedAssetRestores + " failed: "+ failedAssetRestores + " assets...");
}
}
/// <summary>
/// Dearchive the region embodied in this request.
/// </summary>
@@ -296,6 +381,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
public void DearchiveRegion(bool shouldStartScripts)
{
int successfulAssetRestores = 0;
int skippedAssetRestores = 0;
int failedAssetRestores = 0;
DearchiveScenesInfo dearchivedScenes;
@@ -308,7 +394,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver
TarArchiveReader archive = null;
byte[] data;
TarArchiveReader.TarEntryType entryType;
assetfileInfo[] assetsFiles = new assetfileInfo[32];
int assetsFilesCount = 0;
try
{
FindAndLoadControlFile(out archive, out dearchivedScenes);
@@ -321,7 +408,6 @@ namespace OpenSim.Region.CoreModules.World.Archiver
if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType)
continue;
// Find the scene that this file belongs to
Scene scene;
@@ -339,7 +425,6 @@ namespace OpenSim.Region.CoreModules.World.Archiver
}
}
// Process the file
if (filePath.StartsWith(ArchiveConstants.OBJECTS_PATH) && !m_noObjects)
@@ -348,13 +433,18 @@ namespace OpenSim.Region.CoreModules.World.Archiver
}
else if (filePath.StartsWith(ArchiveConstants.ASSETS_PATH) && !m_skipAssets)
{
if (LoadAsset(filePath, data))
successfulAssetRestores++;
else
failedAssetRestores++;
assetfileInfo asf = new assetfileInfo
{
path = filePath,
data = data
};
if ((successfulAssetRestores + failedAssetRestores) % 250 == 0)
m_log.Debug("[ARCHIVER]: Loaded " + successfulAssetRestores + " assets and failed to load " + failedAssetRestores + " assets...");
assetsFiles[assetsFilesCount++] = asf;
if (assetsFilesCount == 32)
{
loadNeededAssets(assetsFiles, assetsFilesCount, ref successfulAssetRestores, ref failedAssetRestores, ref skippedAssetRestores);
assetsFilesCount = 0;
}
}
else if (filePath.StartsWith(ArchiveConstants.TERRAINS_PATH) && (!m_merge || m_mergeTerrain))
{
@@ -374,6 +464,9 @@ namespace OpenSim.Region.CoreModules.World.Archiver
}
}
if (assetsFilesCount > 0)
loadNeededAssets(assetsFiles, assetsFilesCount, ref successfulAssetRestores, ref failedAssetRestores, ref skippedAssetRestores);
//m_log.Debug("[ARCHIVER]: Reached end of archive");
}
catch (Exception e)
@@ -392,11 +485,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver
if (!m_skipAssets)
{
m_log.InfoFormat("[ARCHIVER]: Restored {0} assets", successfulAssetRestores);
m_log.InfoFormat("[ARCHIVER]: Restored {0} assets", successfulAssetRestores + skippedAssetRestores);
m_log.InfoFormat("[ARCHIVER]: Skipped {0} asset uploads", skippedAssetRestores);
if (failedAssetRestores > 0)
{
m_log.ErrorFormat("[ARCHIVER]: Failed to load {0} assets", failedAssetRestores);
m_log.ErrorFormat("[ARCHIVER]: Failed to load {0} assets", failedAssetRestores);
m_errorMessage += String.Format("Failed to load {0} assets", failedAssetRestores);
}
}
@@ -944,88 +1038,31 @@ namespace OpenSim.Region.CoreModules.World.Archiver
}
}
/// Load an asset
/// </summary>
/// <param name="assetFilename"></param>
/// <param name="data"></param>
/// <returns>true if asset was successfully loaded, false otherwise</returns>
private bool LoadAsset(string assetPath, byte[] data)
private bool TryUploadAsset(UUID assetID, sbyte assetType, byte[] data)
{
// Right now we're nastily obtaining the UUID from the filename
string filename = assetPath.Remove(0, ArchiveConstants.ASSETS_PATH.Length);
int i = filename.LastIndexOf(ArchiveConstants.ASSET_EXTENSION_SEPARATOR);
if (i == -1)
if (assetType == (sbyte)AssetType.Unknown)
{
m_log.ErrorFormat(
"[ARCHIVER]: Could not find extension information in asset path {0} since it's missing the separator {1}. Skipping",
assetPath, ArchiveConstants.ASSET_EXTENSION_SEPARATOR);
m_log.WarnFormat("[ARCHIVER]: Importing {0} byte asset {1} with unknown type", data.Length, assetID.ToString());
}
else if (assetType == (sbyte)AssetType.Object)
{
data = SceneObjectSerializer.ModifySerializedObject(assetID, data,
sog =>
{
ModifySceneObject(m_rootScene, sog);
return true;
});
return false;
if (data == null)
return false;
}
string extension = filename.Substring(i);
string uuid = filename.Remove(filename.Length - extension.Length);
//m_log.DebugFormat("[ARCHIVER]: Importing asset {0}, type {1}", uuid, assetType);
AssetBase asset = new AssetBase(assetID, string.Empty, assetType, UUID.Zero.ToString());
asset.Data = data;
if (m_assetService.GetMetadata(uuid) != null)
{
sbyte asype = ArchiveConstants.EXTENSION_TO_ASSET_TYPE[extension];
// m_log.DebugFormat("[ARCHIVER]: found existing asset {0}",uuid);
return true;
}
if (ArchiveConstants.EXTENSION_TO_ASSET_TYPE.ContainsKey(extension))
{
sbyte assetType = ArchiveConstants.EXTENSION_TO_ASSET_TYPE[extension];
if (assetType == (sbyte)AssetType.Unknown)
{
m_log.WarnFormat("[ARCHIVER]: Importing {0} byte asset {1} with unknown type", data.Length, uuid);
}
else if (assetType == (sbyte)AssetType.Object)
{
data = SceneObjectSerializer.ModifySerializedObject(UUID.Parse(uuid), data,
sog =>
{
ModifySceneObject(m_rootScene, sog);
return true;
});
if (data == null)
return false;
}
//m_log.DebugFormat("[ARCHIVER]: Importing asset {0}, type {1}", uuid, assetType);
AssetBase asset = new AssetBase(new UUID(uuid), String.Empty, assetType, UUID.Zero.ToString());
asset.Data = data;
// We're relying on the asset service to do the sensible thing and not store the asset if it already
// exists.
m_assetService.Store(asset);
/**
* Create layers on decode for image assets. This is likely to significantly increase the time to load archives so
* it might be best done when dearchive takes place on a separate thread
if (asset.Type=AssetType.Texture)
{
IJ2KDecoder cacheLayerDecode = scene.RequestModuleInterface<IJ2KDecoder>();
if (cacheLayerDecode != null)
cacheLayerDecode.syncdecode(asset.FullID, asset.Data);
}
*/
return true;
}
else
{
m_log.ErrorFormat(
"[ARCHIVER]: Tried to dearchive data with path {0} with an unknown type extension {1}",
assetPath, extension);
return false;
}
m_assetService.Store(asset);
return true; // not right
}
/// <summary>