From cc86bdddcfd5915a2e2444074365956d4a87c59e Mon Sep 17 00:00:00 2001 From: UbitUmarov Date: Tue, 10 Aug 2021 17:02:27 +0100 Subject: [PATCH] oar load: do a block check for assets presence on grid services, instead of one by one --- .../World/Archiver/ArchiveReadRequest.cs | 211 ++++++++++-------- 1 file changed, 124 insertions(+), 87 deletions(-) diff --git a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs index b69f1dd956..a6f7dd6d19 100644 --- a/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs +++ b/OpenSim/Region/CoreModules/World/Archiver/ArchiveReadRequest.cs @@ -285,6 +285,91 @@ namespace OpenSim.Region.CoreModules.World.Archiver m_assetService = m_rootScene.AssetService; } + public struct assetfileInfo + { + public string path; + public byte[] data; + } + + private void loadNeededAssets(assetfileInfo[] assetsFiles, int count, ref int successfulAssetRestores, + ref int failedAssetRestores, ref int skipedAssetRestores) + { + List ids = new List(count); + List uuids = new List(count); + List types = new List(count); + List datas = new List(count); + + for (int i = 0; i < count; ++i) + { + string assetPath = assetsFiles[i].path; + byte[] data = assetsFiles[i].data; + + assetsFiles[i].path = null; + assetsFiles[i].data = null; + + string filename = assetPath.Substring(ArchiveConstants.ASSETS_PATH.Length); + int indx = filename.LastIndexOf(ArchiveConstants.ASSET_EXTENSION_SEPARATOR); + if (indx < 0) + { + m_log.ErrorFormat( + "[ARCHIVER]: Could not find extension information in asset path {0} since it's missing the separator {1}. Skipping", + assetPath, ArchiveConstants.ASSET_EXTENSION_SEPARATOR); + failedAssetRestores++; + continue; + } + string extension = filename.Substring(indx); + if (ArchiveConstants.EXTENSION_TO_ASSET_TYPE.ContainsKey(extension)) + { + string id = filename.Remove(indx); + if (UUID.TryParse(id, out UUID uuid)) + { + ids.Add(id); + uuids.Add(uuid); + datas.Add(data); + types.Add(ArchiveConstants.EXTENSION_TO_ASSET_TYPE[extension]); + continue; + } + } + failedAssetRestores++; + } + + bool[] exits = m_assetService.AssetsExist(ids.ToArray()); + ids.Clear(); + + if (exits == null) + { + m_log.Error("[ARCHIVER]: asset service AssetsExists failed"); + failedAssetRestores += uuids.Count; + return; + } + + if (exits.Length != uuids.Count) + { + m_log.Error("[ARCHIVER]: asset service AssetsExists return size mismatch"); + failedAssetRestores += uuids.Count; + return; + } + + for (int i = 0; i < uuids.Count; ++i) + { + if (exits[i]) + { + ++skipedAssetRestores; + } + else + { + if (TryUploadAsset(uuids[i],types[i], datas[i])) + successfulAssetRestores++; + else + failedAssetRestores++; + } + + int tot = successfulAssetRestores + failedAssetRestores + skipedAssetRestores; + if (tot % 250 == 0) + m_log.Debug("[ARCHIVER]: done " + tot + "; uploaded: " + successfulAssetRestores + " skipped: " + skipedAssetRestores + " failed: "+ failedAssetRestores + " assets..."); + } + } + /// /// Dearchive the region embodied in this request. /// @@ -296,6 +381,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver public void DearchiveRegion(bool shouldStartScripts) { int successfulAssetRestores = 0; + int skippedAssetRestores = 0; int failedAssetRestores = 0; DearchiveScenesInfo dearchivedScenes; @@ -308,7 +394,8 @@ namespace OpenSim.Region.CoreModules.World.Archiver TarArchiveReader archive = null; byte[] data; TarArchiveReader.TarEntryType entryType; - + assetfileInfo[] assetsFiles = new assetfileInfo[32]; + int assetsFilesCount = 0; try { FindAndLoadControlFile(out archive, out dearchivedScenes); @@ -321,7 +408,6 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (TarArchiveReader.TarEntryType.TYPE_DIRECTORY == entryType) continue; - // Find the scene that this file belongs to Scene scene; @@ -339,7 +425,6 @@ namespace OpenSim.Region.CoreModules.World.Archiver } } - // Process the file if (filePath.StartsWith(ArchiveConstants.OBJECTS_PATH) && !m_noObjects) @@ -348,13 +433,18 @@ namespace OpenSim.Region.CoreModules.World.Archiver } else if (filePath.StartsWith(ArchiveConstants.ASSETS_PATH) && !m_skipAssets) { - if (LoadAsset(filePath, data)) - successfulAssetRestores++; - else - failedAssetRestores++; + assetfileInfo asf = new assetfileInfo + { + path = filePath, + data = data + }; - if ((successfulAssetRestores + failedAssetRestores) % 250 == 0) - m_log.Debug("[ARCHIVER]: Loaded " + successfulAssetRestores + " assets and failed to load " + failedAssetRestores + " assets..."); + assetsFiles[assetsFilesCount++] = asf; + if (assetsFilesCount == 32) + { + loadNeededAssets(assetsFiles, assetsFilesCount, ref successfulAssetRestores, ref failedAssetRestores, ref skippedAssetRestores); + assetsFilesCount = 0; + } } else if (filePath.StartsWith(ArchiveConstants.TERRAINS_PATH) && (!m_merge || m_mergeTerrain)) { @@ -374,6 +464,9 @@ namespace OpenSim.Region.CoreModules.World.Archiver } } + if (assetsFilesCount > 0) + loadNeededAssets(assetsFiles, assetsFilesCount, ref successfulAssetRestores, ref failedAssetRestores, ref skippedAssetRestores); + //m_log.Debug("[ARCHIVER]: Reached end of archive"); } catch (Exception e) @@ -392,11 +485,12 @@ namespace OpenSim.Region.CoreModules.World.Archiver if (!m_skipAssets) { - m_log.InfoFormat("[ARCHIVER]: Restored {0} assets", successfulAssetRestores); + m_log.InfoFormat("[ARCHIVER]: Restored {0} assets", successfulAssetRestores + skippedAssetRestores); + m_log.InfoFormat("[ARCHIVER]: Skipped {0} asset uploads", skippedAssetRestores); if (failedAssetRestores > 0) { - m_log.ErrorFormat("[ARCHIVER]: Failed to load {0} assets", failedAssetRestores); + m_log.ErrorFormat("[ARCHIVER]: Failed to load {0} assets", failedAssetRestores); m_errorMessage += String.Format("Failed to load {0} assets", failedAssetRestores); } } @@ -944,88 +1038,31 @@ namespace OpenSim.Region.CoreModules.World.Archiver } } - /// Load an asset - /// - /// - /// - /// true if asset was successfully loaded, false otherwise - private bool LoadAsset(string assetPath, byte[] data) + private bool TryUploadAsset(UUID assetID, sbyte assetType, byte[] data) { - // Right now we're nastily obtaining the UUID from the filename - string filename = assetPath.Remove(0, ArchiveConstants.ASSETS_PATH.Length); - int i = filename.LastIndexOf(ArchiveConstants.ASSET_EXTENSION_SEPARATOR); - - if (i == -1) + if (assetType == (sbyte)AssetType.Unknown) { - m_log.ErrorFormat( - "[ARCHIVER]: Could not find extension information in asset path {0} since it's missing the separator {1}. Skipping", - assetPath, ArchiveConstants.ASSET_EXTENSION_SEPARATOR); + m_log.WarnFormat("[ARCHIVER]: Importing {0} byte asset {1} with unknown type", data.Length, assetID.ToString()); + } + else if (assetType == (sbyte)AssetType.Object) + { + data = SceneObjectSerializer.ModifySerializedObject(assetID, data, + sog => + { + ModifySceneObject(m_rootScene, sog); + return true; + }); - return false; + if (data == null) + return false; } - string extension = filename.Substring(i); - string uuid = filename.Remove(filename.Length - extension.Length); + //m_log.DebugFormat("[ARCHIVER]: Importing asset {0}, type {1}", uuid, assetType); + AssetBase asset = new AssetBase(assetID, string.Empty, assetType, UUID.Zero.ToString()); + asset.Data = data; - if (m_assetService.GetMetadata(uuid) != null) - { - sbyte asype = ArchiveConstants.EXTENSION_TO_ASSET_TYPE[extension]; - - // m_log.DebugFormat("[ARCHIVER]: found existing asset {0}",uuid); - return true; - } - - if (ArchiveConstants.EXTENSION_TO_ASSET_TYPE.ContainsKey(extension)) - { - sbyte assetType = ArchiveConstants.EXTENSION_TO_ASSET_TYPE[extension]; - - if (assetType == (sbyte)AssetType.Unknown) - { - m_log.WarnFormat("[ARCHIVER]: Importing {0} byte asset {1} with unknown type", data.Length, uuid); - } - else if (assetType == (sbyte)AssetType.Object) - { - data = SceneObjectSerializer.ModifySerializedObject(UUID.Parse(uuid), data, - sog => - { - ModifySceneObject(m_rootScene, sog); - return true; - }); - - if (data == null) - return false; - } - - //m_log.DebugFormat("[ARCHIVER]: Importing asset {0}, type {1}", uuid, assetType); - - AssetBase asset = new AssetBase(new UUID(uuid), String.Empty, assetType, UUID.Zero.ToString()); - asset.Data = data; - - // We're relying on the asset service to do the sensible thing and not store the asset if it already - // exists. - m_assetService.Store(asset); - - /** - * Create layers on decode for image assets. This is likely to significantly increase the time to load archives so - * it might be best done when dearchive takes place on a separate thread - if (asset.Type=AssetType.Texture) - { - IJ2KDecoder cacheLayerDecode = scene.RequestModuleInterface(); - if (cacheLayerDecode != null) - cacheLayerDecode.syncdecode(asset.FullID, asset.Data); - } - */ - - return true; - } - else - { - m_log.ErrorFormat( - "[ARCHIVER]: Tried to dearchive data with path {0} with an unknown type extension {1}", - assetPath, extension); - - return false; - } + m_assetService.Store(asset); + return true; // not right } ///