opensim/OpenSim/Framework/Communications/Cache/AssetCache.cs

/*
 * Copyright (c) Contributors, http://opensimulator.org/
 * See CONTRIBUTORS.TXT for a full list of copyright holders.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the OpenSim Project nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

using System;
using System.Collections.Generic;
using System.Reflection;
using System.Threading;
using OpenMetaverse;
using OpenMetaverse.Packets;
using log4net;
using OpenSim.Framework.Statistics;

namespace OpenSim.Framework.Communications.Cache
{
    public delegate void AssetRequestCallback(UUID assetID, AssetBase asset);

    /// <summary>
    /// Manages local cache of assets and their sending to viewers.
    ///
    /// This class actually encapsulates two largely separate mechanisms.  One mechanism fetches assets either
    /// synchronously or async and passes the data back to the requester.  The second mechanism fetches assets and
    /// sends packetised data directly back to the client.  The only point where they meet is AssetReceived() and
    /// AssetNotFound(), which means they do share the same asset and texture caches.
    ///
    /// TODO: Assets in this cache are effectively immortal (they are never disposed of through old age).
    /// This is not a huge problem at the moment since other memory use usually dwarfs that used by assets
    /// but it's something to bear in mind.
    /// </summary>
    public class AssetCache : IAssetReceiver
    {
        private static readonly ILog m_log
            = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);

        /// <summary>
        /// The cache of assets.  This does not include textures.
        /// </summary>
        private Dictionary<UUID, AssetInfo> Assets;

        /// <summary>
        /// The cache of textures.
        /// </summary>
        private Dictionary<UUID, TextureImage> Textures;

        /// <summary>
        /// Assets requests which are waiting for asset server data.  This includes texture requests
        /// </summary>
        private Dictionary<UUID, AssetRequest> RequestedAssets;

        /// <summary>
        /// Asset requests with data which are ready to be sent back to requesters.  This includes textures.
        /// </summary>
        private List<AssetRequest> AssetRequests;

        /// <summary>
        /// Until the asset request is fulfilled, each asset request is associated with a list of requesters
        /// </summary>
        private Dictionary<UUID, AssetRequestsList> RequestLists;

        /// <summary>
        /// The 'server' from which assets can be requested and to which assets are persisted. 
        /// </summary>
        private readonly IAssetServer m_assetServer;

        /// <summary>
        /// Report statistical data.
        /// </summary>
        public void ShowState()
        {
            m_log.InfoFormat("Assets:{0}  Textures:{1}   RequestLists:{2}",
                Assets.Count,
                Textures.Count,
              //   AssetRequests.Count,
              //    RequestedAssets.Count,
                RequestLists.Count);

            int temporaryImages = 0;
            int temporaryAssets = 0;

            long imageBytes = 0;
            long assetBytes = 0;

            foreach (TextureImage texture in Textures.Values)
            {
                if (texture != null)
                {
                    if (texture.Temporary)
                    {
                        temporaryImages++;
                    }

                    imageBytes += texture.Data.GetLongLength(0);
                }
            }

            foreach (AssetInfo asset in Assets.Values)
            {
                if (asset != null)
                {
                    if (asset.Temporary)
                    {
                        temporaryAssets++;
                    }

                    assetBytes += asset.Data.GetLongLength(0);
                }
            }

            m_log.InfoFormat("Temporary Images: {0}  Temporary Assets: {1}",
                temporaryImages,
                temporaryAssets);

            m_log.InfoFormat("Image data: {0}kb  Asset data: {1}kb",
                imageBytes / 1024,
                assetBytes / 1024);
        }

        /// <summary>
        /// Clear the asset cache.
        /// </summary>
        public void Clear()
        {
            m_log.Info("[ASSET CACHE]: Clearing Asset cache");

            if (StatsManager.SimExtraStats != null)
                StatsManager.SimExtraStats.ClearAssetCacheStatistics();

            Initialize();
        }

        /// <summary>
        /// Initialize the cache.
        /// </summary>
        private void Initialize()
        {
            Assets = new Dictionary<UUID, AssetInfo>();
            Textures = new Dictionary<UUID, TextureImage>();
            AssetRequests = new List<AssetRequest>();

            RequestedAssets = new Dictionary<UUID, AssetRequest>();
            RequestLists = new Dictionary<UUID, AssetRequestsList>();
        }

        /// <summary>
        /// Constructor.  Initialize will need to be called separately.
        /// </summary>
        /// <param name="assetServer"></param>
        public AssetCache(IAssetServer assetServer)
        {
            m_log.Info("[ASSET CACHE]: Creating Asset cache");
            Initialize();

            m_assetServer = assetServer;
            m_assetServer.SetReceiver(this);

            Thread assetCacheThread = new Thread(new ThreadStart(RunAssetManager));
            assetCacheThread.Name = "AssetCacheThread";
            assetCacheThread.IsBackground = true;
            assetCacheThread.Start();
            ThreadTracker.Add(assetCacheThread);
        }

        /// <summary>
        /// Process the asset queue which holds data which is packeted up and sent
        /// directly back to the client.
        /// </summary>
        public void RunAssetManager()
        {
            while (true)
            {
                try
                {
                    ProcessAssetQueue();
                    Thread.Sleep(500);
                }
                catch (Exception e)
                {
                    m_log.Error("[ASSET CACHE]: " + e.ToString());
                }
            }
        }

        /// <summary>
        /// Only get an asset if we already have it in the cache.
        /// </summary>
        /// <param name="assetId"></param>
        /// <param name="asset"></param>
        /// <returns>true if the asset was in the cache, false if it was not</returns>
        private bool TryGetCachedAsset(UUID assetId, out AssetBase asset)
        {
            if (Textures.ContainsKey(assetId))
            {
                asset = Textures[assetId];
                return true;
            }
            else if (Assets.ContainsKey(assetId))
            {
                asset = Assets[assetId];
                return true;
            }

            asset = null;
            return false;
        }

        /// <summary>
        /// Asynchronously retrieve an asset.
        /// </summary>
        /// <param name="assetId"></param>
        /// <param name="callback">
        /// A callback invoked when the asset has either been found or not found.
        /// If the asset was found this is called with the asset UUID and the asset data
        /// If the asset was not found this is still called with the asset UUID but with a null asset data reference</param>
        public void GetAsset(UUID assetId, AssetRequestCallback callback, bool isTexture)
        {
            //m_log.DebugFormat("[ASSET CACHE]: Requesting {0} {1}", isTexture ? "texture" : "asset", assetId);

            // Xantor 20080526:
            // if a request is made for an asset which is not in the cache yet, but has already been requested by
            // something else, queue up the callbacks on that requestor instead of swamping the assetserver
            // with multiple requests for the same asset.

            AssetBase asset;

            if (TryGetCachedAsset(assetId, out asset))
            {
                callback(assetId, asset);
            }
            else
            {
                // m_log.DebugFormat("[ASSET CACHE]: Adding request for {0} {1}", isTexture ? "texture" : "asset", assetId);
                
                NewAssetRequest req = new NewAssetRequest(callback);
                AssetRequestsList requestList;

                lock (RequestLists)
                {
                    if (RequestLists.TryGetValue(assetId, out requestList)) // do we already have a request pending?
                    {
                        // m_log.DebugFormat("[ASSET CACHE]: Intercepted Duplicate request for {0} {1}", isTexture ? "texture" : "asset", assetId);
                        // add to callbacks for this assetId
                        RequestLists[assetId].Requests.Add(req);
                    }
                    else
                    {
                        // m_log.DebugFormat("[ASSET CACHE]: Adding request for {0} {1}", isTexture ? "texture" : "asset", assetId);
                        requestList = new AssetRequestsList();
                        requestList.TimeRequested = DateTime.Now;
                        requestList.Requests.Add(req);           
                        
                        RequestLists.Add(assetId, requestList);                        
                        
                        m_assetServer.RequestAsset(assetId, isTexture);
                    }
                }
            }
        }

        /// <summary>
        /// Synchronously retreive an asset.  If the asset isn't in the cache, a request will be made to the persistent store to
        /// load it into the cache.
        ///
        /// XXX We'll keep polling the cache until we get the asset or we exceed
        /// the allowed number of polls.  This isn't a very good way of doing things since a single thread
        /// is processing inbound packets, so if the asset server is slow, we could block this for up to
        /// the timeout period.  What we might want to do is register asynchronous callbacks on asset
        /// receipt in the same manner as the TextureDownloadModule.  Of course,
        /// a timeout before asset receipt usually isn't fatal, the operation will work on the retry when the
        /// asset is much more likely to have made it into the cache.
        /// </summary>
        /// <param name="assetID"></param>
        /// <param name="isTexture"></param>
        /// <returns>null if the asset could not be retrieved</returns>
        public AssetBase GetAsset(UUID assetID, bool isTexture)
        {
            // I'm not going over 3 seconds since this will be blocking processing of all the other inbound
            // packets from the client.
            int pollPeriod = 200;
            int maxPolls = 15;

            AssetBase asset;

            if (TryGetCachedAsset(assetID, out asset))
            {
                return asset;
            }
            else
            {
                m_assetServer.RequestAsset(assetID, isTexture);

                do
                {
                    Thread.Sleep(pollPeriod);

                    if (TryGetCachedAsset(assetID, out asset))
                    {
                        return asset;
                    }
                } while (--maxPolls > 0);

                m_log.WarnFormat("[ASSET CACHE]: {0} {1} was not received before the retrieval timeout was reached",
                                 isTexture ? "texture" : "asset", assetID.ToString());

                return null;
            }
        }

        /// <summary>
        /// Add an asset to both the persistent store and the cache.
        /// </summary>
        /// <param name="asset"></param>
        public void AddAsset(AssetBase asset)
        {
//            m_log.DebugFormat(
//                "[ASSET CACHE]: Uploaded asset {0}, temporary {1}, store local {2}", 
//                asset.ID, asset.Temporary, asset.Local);
            
            if (asset.Type == (int)AssetType.Texture)
            {
                if (!Textures.ContainsKey(asset.FullID))
                {
                    TextureImage textur = new TextureImage(asset);
                    Textures.Add(textur.FullID, textur);

                    if (StatsManager.SimExtraStats != null)
                        StatsManager.SimExtraStats.AddTexture(textur);

                    // According to http://wiki.secondlife.com/wiki/AssetUploadRequest, Local signifies that the 
                    // information is stored locally.  It could disappear, in which case we could send the
                    // ImageNotInDatabase packet to tell the client this.  
                    //
                    // However, this doesn't quite appear to work with local textures that are part of an avatar's 
                    // appearance texture set.  Whilst sending an ImageNotInDatabase does trigger an automatic rebake
                    // and reupload by the client, if those assets aren't pushed to the asset server anyway, then
                    // on crossing onto another region server, other avatars can no longer get the required textures.
                    // There doesn't appear to be any signal from the sim to the newly region border crossed client
                    // asking it to reupload its local texture assets to that region server.
                    //
                    // One can think of other cunning ways around this.  For instance, on a region crossing or teleport,
                    // the original sim could squirt local assets to the new sim.  Or the new sim could have pointers
                    // to the original sim to fetch the 'local' assets (this is getting more complicated).
                    //
                    // But for now, we're going to take the easy way out and store local assets globally.
                    //
                    // TODO: Also, Temporary is now deprecated.  We should start ignoring it and not passing it out from LLClientView.
                    if (!asset.Temporary || asset.Local)
                    {
                        m_assetServer.StoreAsset(asset);
                    }
                }
//                else
//                {
//                    m_log.DebugFormat("[ASSET CACHE]: Textures already contains {0}", asset.ID);                    
//                }                
            }
            else
            {
                if (!Assets.ContainsKey(asset.FullID))
                {
                    AssetInfo assetInf = new AssetInfo(asset);
                    Assets.Add(assetInf.FullID, assetInf);

                    if (StatsManager.SimExtraStats != null)
                        StatsManager.SimExtraStats.AddAsset(assetInf);

                    // See comment above.
                    if (!asset.Temporary || asset.Local)
                    {
                        m_assetServer.StoreAsset(asset);
                    }
                }
//                else
//                {
//                    m_log.DebugFormat("[ASSET CACHE]: Assets already contains {0}", asset.ID);                    
//                }
            }
        }

        /// <summary>
        /// Allows you to clear a specific asset by uuid out
        /// of the asset cache.  This is needed because the osdynamic
        /// texture code grows the asset cache without bounds.  The
        /// real solution here is a much better cache archicture, but
        /// this is a stop gap measure until we have such a thing.
        /// </summary>
        public void ExpireAsset(UUID uuid)
        {
            // uuid is unique, so no need to worry about it showing up
            // in the 2 caches differently.  Also, locks are probably
            // needed in all of this, or move to synchronized non
            // generic forms for Dictionaries.
            if (Textures.ContainsKey(uuid))
            {
                Textures.Remove(uuid);
            }
            else if (Assets.ContainsKey(uuid))
            {
                Assets.Remove(uuid);
            }
        }

        // See IAssetReceiver
        public void AssetReceived(AssetBase asset, bool IsTexture)
        {
//            m_log.DebugFormat("[ASSET CACHE]: Received asset {0}", asset.ID);
            
            //check if it is a texture or not
            //then add to the correct cache list
            //then check for waiting requests for this asset/texture (in the Requested lists)
            //and move those requests into the Requests list.
            if (IsTexture)
            {
                TextureImage image = new TextureImage(asset);
                if (!Textures.ContainsKey(image.FullID))
                {
                    Textures.Add(image.FullID, image);

                    if (StatsManager.SimExtraStats != null)
                    {
                        StatsManager.SimExtraStats.AddTexture(image);
                    }
                }
            }
            else
            {
                AssetInfo assetInf = new AssetInfo(asset);
                if (!Assets.ContainsKey(assetInf.FullID))
                {
                    Assets.Add(assetInf.FullID, assetInf);

                    if (StatsManager.SimExtraStats != null)
                    {
                        StatsManager.SimExtraStats.AddAsset(assetInf);
                    }

                    if (RequestedAssets.ContainsKey(assetInf.FullID))
                    {
                        AssetRequest req = RequestedAssets[assetInf.FullID];
                        req.AssetInf = assetInf;
                        req.NumPackets = CalculateNumPackets(assetInf.Data);

                        RequestedAssets.Remove(assetInf.FullID);
                        // If it's a direct request for a script, drop it
                        // because it's a hacked client
                        if (req.AssetRequestSource != 2 || assetInf.Type != 10)
                            AssetRequests.Add(req);
                    }
                }
            }

            // Notify requesters for this asset
            AssetRequestsList reqList = null;
            
            lock (RequestLists)
            {                                
                if (RequestLists.TryGetValue(asset.FullID, out reqList))                      
                    RequestLists.Remove(asset.FullID);
            }

            if (reqList != null)
            {
                if (StatsManager.SimExtraStats != null)
                    StatsManager.SimExtraStats.AddAssetRequestTimeAfterCacheMiss(DateTime.Now - reqList.TimeRequested);
                
                foreach (NewAssetRequest req in reqList.Requests)
                {
                    // Xantor 20080526 are we really calling all the callbacks if multiple queued for 1 request? -- Yes, checked
                    // m_log.DebugFormat("[ASSET CACHE]: Callback for asset {0}", asset.FullID);
                    req.Callback(asset.FullID, asset);
                }
            }
        }

        // See IAssetReceiver
        public void AssetNotFound(UUID assetID, bool IsTexture)
        {
//            m_log.WarnFormat("[ASSET CACHE]: AssetNotFound for {0}", assetID);

            if (IsTexture)
            {
                Textures[assetID] = null;
            }
            else
            {
                Assets[assetID] = null;
            }

            // Notify requesters for this asset
            AssetRequestsList reqList = null;
            lock (RequestLists)
            {
                if (RequestLists.TryGetValue(assetID, out reqList))                      
                    RequestLists.Remove(assetID);
            }

            if (reqList != null)
            {
                if (StatsManager.SimExtraStats != null)
                    StatsManager.SimExtraStats.AddAssetRequestTimeAfterCacheMiss(DateTime.Now - reqList.TimeRequested);
                
                foreach (NewAssetRequest req in reqList.Requests)
                {
                    req.Callback(assetID, null);
                }
            }
        }

        /// <summary>
        /// Calculate the number of packets required to send the asset to the client.
        /// </summary>
        /// <param name="data"></param>
        /// <returns></returns>
        private static int CalculateNumPackets(byte[] data)
        {
            const uint m_maxPacketSize = 600;
            int numPackets = 1;

            if (data.LongLength > m_maxPacketSize)
            {
                // over max number of bytes so split up file
                long restData = data.LongLength - m_maxPacketSize;
                int restPackets = (int)((restData + m_maxPacketSize - 1) / m_maxPacketSize);
                numPackets += restPackets;
            }

            return numPackets;
        }

        /// <summary>
        /// Handle an asset request from the client.  The result will be sent back asynchronously.
        /// </summary>
        /// <param name="userInfo"></param>
        /// <param name="transferRequest"></param>
        public void AddAssetRequest(IClientAPI userInfo, TransferRequestPacket transferRequest)
        {
            UUID requestID = UUID.Zero;
            byte source = 2;
            if (transferRequest.TransferInfo.SourceType == 2)
            {
                //direct asset request
                requestID = new UUID(transferRequest.TransferInfo.Params, 0);
            }
            else if (transferRequest.TransferInfo.SourceType == 3)
            {
                //inventory asset request
                requestID = new UUID(transferRequest.TransferInfo.Params, 80);
                source = 3;
                //Console.WriteLine("asset request " + requestID);
            }
            
            //check to see if asset is in local cache, if not we need to request it from asset server.
            //Console.WriteLine("asset request " + requestID);
            if (!Assets.ContainsKey(requestID))
            {
                //not found asset
                // so request from asset server
                if (!RequestedAssets.ContainsKey(requestID))
                {
                    AssetRequest request = new AssetRequest();
                    request.RequestUser = userInfo;
                    request.RequestAssetID = requestID;
                    request.TransferRequestID = transferRequest.TransferInfo.TransferID;
                    request.AssetRequestSource = source;
                    request.Params = transferRequest.TransferInfo.Params;
                    RequestedAssets.Add(requestID, request);
                    m_assetServer.RequestAsset(requestID, false);
                }

                return;
            }

            // It has an entry in our cache
            AssetInfo asset = Assets[requestID];

            // FIXME: We never tell the client about assets which do not exist when requested by this transfer mechanism, which can't be right.
            if (null == asset)
            {
                //m_log.DebugFormat("[ASSET CACHE]: Asset transfer request for asset which is {0} already known to be missing.  Dropping", requestID);
                return;
            }

            // Scripts cannot be retrieved by direct request
            if (transferRequest.TransferInfo.SourceType == 2 && asset.Type == 10)
                return;

            // The asset is knosn to exist and is in our cache, so add it to the AssetRequests list
            AssetRequest req = new AssetRequest();
            req.RequestUser = userInfo;
            req.RequestAssetID = requestID;
            req.TransferRequestID = transferRequest.TransferInfo.TransferID;
            req.AssetRequestSource = source;
            req.Params = transferRequest.TransferInfo.Params;
            req.AssetInf = asset;
            req.NumPackets = CalculateNumPackets(asset.Data);
            AssetRequests.Add(req);
        }

        /// <summary>
        /// Process the asset queue which sends packets directly back to the client.
        /// </summary>
        private void ProcessAssetQueue()
        {
            //should move the asset downloading to a module, like has been done with texture downloading
            if (AssetRequests.Count == 0)
            {
                //no requests waiting
                return;
            }
            
            // if less than 5, do all of them
            int num = Math.Min(5, AssetRequests.Count);

            AssetRequest req;
            AssetRequestToClient req2 = null;
            for (int i = 0; i < num; i++)
            {
                req = (AssetRequest)AssetRequests[i];
                if (req2 == null)
                {
                    req2 = new AssetRequestToClient();
                }
                // Trying to limit memory usage by only creating AssetRequestToClient if needed
                //req2 = new AssetRequestToClient();
                req2.AssetInf = (AssetBase)req.AssetInf;
                req2.AssetRequestSource = req.AssetRequestSource;
                req2.DataPointer = req.DataPointer;
                req2.DiscardLevel = req.DiscardLevel;
                req2.ImageInfo = (AssetBase)req.ImageInfo;
                req2.IsTextureRequest = req.IsTextureRequest;
                req2.NumPackets = req.NumPackets;
                req2.PacketCounter = req.PacketCounter;
                req2.Params = req.Params;
                req2.RequestAssetID = req.RequestAssetID;
                req2.TransferRequestID = req.TransferRequestID;
                req.RequestUser.SendAsset(req2);

            }

            //remove requests that have been completed
            for (int i = 0; i < num; i++)
            {
                AssetRequests.RemoveAt(0);
            }
        }

        public class AssetRequest
        {
            public IClientAPI RequestUser;
            public UUID RequestAssetID;
            public AssetInfo AssetInf;
            public TextureImage ImageInfo;
            public UUID TransferRequestID;
            public long DataPointer = 0;
            public int NumPackets = 0;
            public int PacketCounter = 0;
            public bool IsTextureRequest;
            public byte AssetRequestSource = 2;
            public byte[] Params = null;
            //public bool AssetInCache;
            //public int TimeRequested;
            public int DiscardLevel = -1;
        }

        public class AssetInfo : AssetBase
        {
            public AssetInfo(AssetBase aBase)
            {
                Data = aBase.Data;
                FullID = aBase.FullID;
                Type = aBase.Type;
                Name = aBase.Name;
                Description = aBase.Description;
            }
        }

        public class TextureImage : AssetBase
        {
            public TextureImage(AssetBase aBase)
            {
                Data = aBase.Data;
                FullID = aBase.FullID;
                Type = aBase.Type;
                Name = aBase.Name;
                Description = aBase.Description;
            }
        }

        /// <summary>
        /// A list of requests for a particular asset.
        /// </summary>
        public class AssetRequestsList
        {
            /// <summary>
            /// A list of requests for assets
            /// </summary>
            public List<NewAssetRequest> Requests = new List<NewAssetRequest>();
            
            /// <summary>
            /// Record the time that this request was first made.
            /// </summary>
            public DateTime TimeRequested;
        }

        /// <summary>
        /// Represent a request for an asset that has yet to be fulfilled.
        /// </summary>
        public class NewAssetRequest
        {
            public AssetRequestCallback Callback;

            public NewAssetRequest(AssetRequestCallback callback)
            {
                Callback = callback;
            }
        }
    }
}