opensim/OpenSim/Framework/Communications/Cache/AssetCache.cs

/*
 * Copyright (c) Contributors, http://opensimulator.org/
 * See CONTRIBUTORS.TXT for a full list of copyright holders.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the OpenSim Project nor the
 *       names of its contributors may be used to endorse or promote products
 *       derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

using System;
using System.Collections.Generic;
using System.Reflection;
using System.Text.RegularExpressions;
using System.Threading;
using GlynnTucker.Cache;
using log4net;
using OpenMetaverse;
using OpenMetaverse.Packets;
using OpenSim.Framework.Statistics;
using System.Text;

namespace OpenSim.Framework.Communications.Cache
{
    /// <summary>
    /// Manages local cache of assets and their sending to viewers.
    /// </summary>
    ///
    /// This class actually encapsulates two largely separate mechanisms.  One mechanism fetches assets either
    /// synchronously or async and passes the data back to the requester.  The second mechanism fetches assets and
    /// sends packetised data directly back to the client.  The only point where they meet is AssetReceived() and
    /// AssetNotFound(), which means they do share the same asset and texture caches.
    public class AssetCache : IAssetCache
    {
        private static readonly ILog m_log
            = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
        
        protected ICache m_memcache = new SimpleMemoryCache();

        /// <value>
        /// Assets requests which are waiting for asset server data.  This includes texture requests
        /// </value>
        private Dictionary<UUID, AssetRequest> RequestedAssets;

        /// <value>
        /// Asset requests with data which are ready to be sent back to requesters.  This includes textures.
        /// </value>
        private List<AssetRequest> AssetRequests;

        /// <value>
        /// Until the asset request is fulfilled, each asset request is associated with a list of requesters
        /// </value>
        private Dictionary<UUID, AssetRequestsList> RequestLists;   
        
        #region IPlugin

        /// <summary>
        /// The methods and properties in this section are needed to
        /// support the IPlugin interface. They cann all be overridden
        /// as needed by a derived class.
        /// </summary>

        public virtual string Name
        {
            get { return "OpenSim.Framework.Communications.Cache.AssetCache"; }
        }

        public virtual string Version
        {
            get { return "1.0"; }
        }

        public virtual void Initialise()
        {
            m_log.Debug("[ASSET CACHE]: Asset cache null initialisation");
        }

        public virtual void Initialise(IAssetServer assetServer)
        {
            m_log.Debug("[ASSET CACHE]: Asset cache server-specified initialisation");
            m_log.InfoFormat("[ASSET CACHE]: Asset cache initialisation [{0}/{1}]", Name, Version);

            Reset();

            m_assetServer = assetServer;
            m_assetServer.SetReceiver(this);

            Thread assetCacheThread = new Thread(RunAssetManager);
            assetCacheThread.Name = "AssetCacheThread";
            assetCacheThread.IsBackground = true;
            assetCacheThread.Start();
            ThreadTracker.Add(assetCacheThread);
        }

        public virtual void Initialise(ConfigSettings settings, IAssetServer assetServer)
        {
            m_log.Debug("[ASSET CACHE]: Asset cache configured initialisation");
            Initialise(assetServer);
        }

        public void Dispose()
        {
        }

        #endregion     

        public IAssetServer AssetServer
        {
            get { return m_assetServer; }
        }
        private IAssetServer m_assetServer;
        
        public AssetCache()
        {
            m_log.Debug("[ASSET CACHE]: Asset cache (plugin constructor)");
        }        
        
        /// <summary>
        /// Constructor.
        /// </summary>
        /// <param name="assetServer"></param>
        public AssetCache(IAssetServer assetServer)
        {
            Initialise(assetServer);
        }        

        public void ShowState()
        {
            m_log.InfoFormat("Memcache:{0}   RequestLists:{1}",
                m_memcache.Count,
              //   AssetRequests.Count,
              //    RequestedAssets.Count,
                RequestLists.Count);
        }

        public void Clear()
        {
            m_log.Info("[ASSET CACHE]: Clearing Asset cache");

            if (StatsManager.SimExtraStats != null)
                StatsManager.SimExtraStats.ClearAssetCacheStatistics();

            Reset();
        }

        /// <summary>
        /// Reset the cache.
        /// </summary>
        private void Reset()
        {
            AssetRequests = new List<AssetRequest>();
            RequestedAssets = new Dictionary<UUID, AssetRequest>();
            RequestLists = new Dictionary<UUID, AssetRequestsList>();
        }

        /// <summary>
        /// Process the asset queue which holds data which is packeted up and sent
        /// directly back to the client.
        /// </summary>
        private void RunAssetManager()
        {
            while (true)
            {
                try
                {
                    ProcessAssetQueue();
                    Thread.Sleep(500);
                }
                catch (Exception e)
                {
                    if (e != null) 
                    {
                        m_log.ErrorFormat("[ASSET CACHE]: {0}", e);
                    } 
                    else 
                    {
                        // this looks weird, but we've seen this show up as an issue in unit tests, so leave it here until we know why
                        m_log.Error("[ASSET CACHE]: an exception was thrown in RunAssetManager, but is now null.  Something is very wrong.");
                    }
                }
            }
        }

        public bool TryGetCachedAsset(UUID assetId, out AssetBase asset)
        {
            Object tmp;
            if (m_memcache.TryGet(assetId, out tmp))
            {
                asset = (AssetBase)tmp;
                //m_log.Info("Retrieved from cache " + assetId);
                return true;
            }

            asset = null;
            return false;
        }

        public void GetAsset(UUID assetId, AssetRequestCallback callback, bool isTexture)
        {
            //m_log.DebugFormat("[ASSET CACHE]: Requesting {0} {1}", isTexture ? "texture" : "asset", assetId);

            // Xantor 20080526:
            // if a request is made for an asset which is not in the cache yet, but has already been requested by
            // something else, queue up the callbacks on that requestor instead of swamping the assetserver
            // with multiple requests for the same asset.

            AssetBase asset;

            if (TryGetCachedAsset(assetId, out asset))
            {
                callback(assetId, asset);
            }
            else
            {
                // m_log.DebugFormat("[ASSET CACHE]: Adding request for {0} {1}", isTexture ? "texture" : "asset", assetId);

                NewAssetRequest req = new NewAssetRequest(callback);
                AssetRequestsList requestList;

                lock (RequestLists)
                {
                    if (RequestLists.TryGetValue(assetId, out requestList)) // do we already have a request pending?
                    {
                        // m_log.DebugFormat("[ASSET CACHE]: Intercepted Duplicate request for {0} {1}", isTexture ? "texture" : "asset", assetId);
                        // add to callbacks for this assetId
                        RequestLists[assetId].Requests.Add(req);
                    }
                    else
                    {
                        // m_log.DebugFormat("[ASSET CACHE]: Adding request for {0} {1}", isTexture ? "texture" : "asset", assetId);
                        requestList = new AssetRequestsList();
                        requestList.TimeRequested = DateTime.Now;
                        requestList.Requests.Add(req);

                        RequestLists.Add(assetId, requestList);

                        m_assetServer.RequestAsset(assetId, isTexture);
                    }
                }
            }
        }

        public AssetBase GetAsset(UUID assetID, bool isTexture)
        {
            // I'm not going over 3 seconds since this will be blocking processing of all the other inbound
            // packets from the client.
            const int pollPeriod = 200;
            int maxPolls = 15;

            AssetBase asset;

            if (TryGetCachedAsset(assetID, out asset))
            {
                return asset;
            }

            m_assetServer.RequestAsset(assetID, isTexture);

            do
            {
                Thread.Sleep(pollPeriod);

                if (TryGetCachedAsset(assetID, out asset))
                {
                    return asset;
                }
            }
            while (--maxPolls > 0);

            m_log.WarnFormat("[ASSET CACHE]: {0} {1} was not received before the retrieval timeout was reached",
                             isTexture ? "texture" : "asset", assetID.ToString());

            return null;
        }

        public void AddAsset(AssetBase asset)
        {
            if (!m_memcache.Contains(asset.FullID))
            {
                m_log.Info("[CACHE] Caching " + asset.FullID + " for 24 hours from last access");
                // Use 24 hour rolling asset cache.
                m_memcache.AddOrUpdate(asset.FullID, asset, TimeSpan.FromHours(24));

                // According to http://wiki.secondlife.com/wiki/AssetUploadRequest, Local signifies that the
                // information is stored locally.  It could disappear, in which case we could send the
                // ImageNotInDatabase packet to tell the client this.
                //
                // However, this doesn't quite appear to work with local textures that are part of an avatar's
                // appearance texture set.  Whilst sending an ImageNotInDatabase does trigger an automatic rebake
                // and reupload by the client, if those assets aren't pushed to the asset server anyway, then
                // on crossing onto another region server, other avatars can no longer get the required textures.
                // There doesn't appear to be any signal from the sim to the newly region border crossed client
                // asking it to reupload its local texture assets to that region server.
                //
                // One can think of other cunning ways around this.  For instance, on a region crossing or teleport,
                // the original sim could squirt local assets to the new sim.  Or the new sim could have pointers
                // to the original sim to fetch the 'local' assets (this is getting more complicated).
                //
                // But for now, we're going to take the easy way out and store local assets globally.
                //
                // TODO: Also, Temporary is now deprecated.  We should start ignoring it and not passing it out from LLClientView.
                if (!asset.Temporary || asset.Local)
                {
                    m_assetServer.StoreAsset(asset);
                }
                
                if (StatsManager.SimExtraStats != null)
                    StatsManager.SimExtraStats.AddAsset(asset);                
            }
        }

        public void ExpireAsset(UUID uuid)
        {
            if (m_memcache.Contains(uuid))
            {
                m_memcache.Remove(uuid);

                if (StatsManager.SimExtraStats != null)
                    StatsManager.SimExtraStats.RemoveAsset(uuid);                
            }
        }

        // See IAssetReceiver
        public virtual void AssetReceived(AssetBase asset, bool IsTexture)
        {
            AssetInfo assetInf = new AssetInfo(asset);

            ProcessReceivedAsset(IsTexture, assetInf, null);

            if (!m_memcache.Contains(assetInf.FullID))
            {
                m_memcache.AddOrUpdate(assetInf.FullID, assetInf, TimeSpan.FromHours(24));

                if (StatsManager.SimExtraStats != null)
                    StatsManager.SimExtraStats.AddAsset(assetInf);

                if (RequestedAssets.ContainsKey(assetInf.FullID))
                {
                    AssetRequest req = RequestedAssets[assetInf.FullID];
                    req.AssetInf = assetInf;
                    req.NumPackets = CalculateNumPackets(assetInf.Data);

                    RequestedAssets.Remove(assetInf.FullID);
            
                    if (req.AssetRequestSource == 2 && assetInf.Type == 10)
                    {
                        // If it's a direct request for a script, drop it
                        // because it's a hacked client
                    }
                    else
                    {
                        lock (AssetRequests)
                        {
                            AssetRequests.Add(req);
                        }
                    }
                }
            }

            // Notify requesters for this asset
            AssetRequestsList reqList;

            lock (RequestLists)
            {
                if (RequestLists.TryGetValue(asset.FullID, out reqList))
                    RequestLists.Remove(asset.FullID);
            }

            if (reqList != null)
            {
                if (StatsManager.SimExtraStats != null)
                    StatsManager.SimExtraStats.AddAssetRequestTimeAfterCacheMiss(DateTime.Now - reqList.TimeRequested);

                foreach (NewAssetRequest req in reqList.Requests)
                {
                    // Xantor 20080526 are we really calling all the callbacks if multiple queued for 1 request? -- Yes, checked
                    // m_log.DebugFormat("[ASSET CACHE]: Callback for asset {0}", asset.FullID);
                    req.Callback(asset.FullID, asset);
                }
            }
        }

        protected void ProcessReceivedAsset(bool IsTexture, AssetInfo assetInf, IUserService userService)
        {            
            // if (!IsTexture && assetInf.ContainsReferences && false)
            // {
            //     assetInf.Data = ProcessAssetData(assetInf.Data, userService);
            // }
        }

        // See IAssetReceiver
        public virtual void AssetNotFound(UUID assetId, bool isTexture)
        {
//            m_log.WarnFormat("[ASSET CACHE]: AssetNotFound for {0}", assetId);

            // Remember the fact that this asset could not be found to prevent delays from repeated requests
            m_memcache.Add(assetId, null, TimeSpan.FromHours(24));

            // Notify requesters for this asset
            AssetRequestsList reqList;
            lock (RequestLists)
            {
                if (RequestLists.TryGetValue(assetId, out reqList))
                    RequestLists.Remove(assetId);
            }

            if (reqList != null)
            {
                if (StatsManager.SimExtraStats != null)
                    StatsManager.SimExtraStats.AddAssetRequestTimeAfterCacheMiss(DateTime.Now - reqList.TimeRequested);

                foreach (NewAssetRequest req in reqList.Requests)
                {
                    req.Callback(assetId, null);
                }
            }
        }

        /// <summary>
        /// Calculate the number of packets required to send the asset to the client.
        /// </summary>
        /// <param name="data"></param>
        /// <returns></returns>
        private static int CalculateNumPackets(byte[] data)
        {
            const uint m_maxPacketSize = 600;
            int numPackets = 1;

            if (data.LongLength > m_maxPacketSize)
            {
                // over max number of bytes so split up file
                long restData = data.LongLength - m_maxPacketSize;
                int restPackets = (int)((restData + m_maxPacketSize - 1) / m_maxPacketSize);
                numPackets += restPackets;
            }

            return numPackets;
        }

        public void AddAssetRequest(IClientAPI userInfo, TransferRequestPacket transferRequest)
        {
            UUID requestID = UUID.Zero;
            byte source = 2;
            if (transferRequest.TransferInfo.SourceType == 2)
            {
                //direct asset request
                requestID = new UUID(transferRequest.TransferInfo.Params, 0);
            }
            else if (transferRequest.TransferInfo.SourceType == 3)
            {
                //inventory asset request
                requestID = new UUID(transferRequest.TransferInfo.Params, 80);
                source = 3;
                //m_log.Debug("asset request " + requestID);
            }

            //check to see if asset is in local cache, if not we need to request it from asset server.
            //m_log.Debug("asset request " + requestID);
            if (!m_memcache.Contains(requestID))
            {
                //not found asset
                // so request from asset server
                if (!RequestedAssets.ContainsKey(requestID))
                {
                    AssetRequest request = new AssetRequest();
                    request.RequestUser = userInfo;
                    request.RequestAssetID = requestID;
                    request.TransferRequestID = transferRequest.TransferInfo.TransferID;
                    request.AssetRequestSource = source;
                    request.Params = transferRequest.TransferInfo.Params;
                    RequestedAssets.Add(requestID, request);
                    m_assetServer.RequestAsset(requestID, false);
                }

                return;
            }

            // It has an entry in our cache
            AssetBase asset = (AssetBase)m_memcache[requestID];

            // FIXME: We never tell the client about assets which do not exist when requested by this transfer mechanism, which can't be right.
            if (null == asset)
            {
                //m_log.DebugFormat("[ASSET CACHE]: Asset transfer request for asset which is {0} already known to be missing.  Dropping", requestID);
                return;
            }

            // Scripts cannot be retrieved by direct request
            if (transferRequest.TransferInfo.SourceType == 2 && asset.Type == 10)
                return;

            // The asset is knosn to exist and is in our cache, so add it to the AssetRequests list
            AssetRequest req = new AssetRequest();
            req.RequestUser = userInfo;
            req.RequestAssetID = requestID;
            req.TransferRequestID = transferRequest.TransferInfo.TransferID;
            req.AssetRequestSource = source;
            req.Params = transferRequest.TransferInfo.Params;
            req.AssetInf = new AssetInfo(asset);
            req.NumPackets = CalculateNumPackets(asset.Data);
            lock (AssetRequests) 
            {
                AssetRequests.Add(req);
            }
        }

        /// <summary>
        /// Process the asset queue which sends packets directly back to the client.
        /// </summary>
        private void ProcessAssetQueue()
        {
            //should move the asset downloading to a module, like has been done with texture downloading
            if (AssetRequests.Count == 0)
            {
                //no requests waiting
                return;
            }

            // if less than 5, do all of them
            int num = Math.Min(5, AssetRequests.Count);

            AssetRequest req;
            AssetRequestToClient req2 = new AssetRequestToClient();

            lock (AssetRequests)
            {
                for (int i = 0; i < num; i++)
                {
                   req = AssetRequests[0];
                   AssetRequests.RemoveAt(0);
                   req2.AssetInf = req.AssetInf;
                   req2.AssetRequestSource = req.AssetRequestSource;
                   req2.DataPointer = req.DataPointer;
                   req2.DiscardLevel = req.DiscardLevel;
                   req2.ImageInfo = req.ImageInfo;
                   req2.IsTextureRequest = req.IsTextureRequest;
                   req2.NumPackets = req.NumPackets;
                   req2.PacketCounter = req.PacketCounter;
                   req2.Params = req.Params;
                   req2.RequestAssetID = req.RequestAssetID;
                   req2.TransferRequestID = req.TransferRequestID;
                   req.RequestUser.SendAsset(req2);
                }
            }
        }

        public byte[] ProcessAssetData(byte[] assetData, IUserService userService)
        {
            string data = Encoding.ASCII.GetString(assetData);

            data = ProcessAssetDataString(data, userService);

            return Encoding.ASCII.GetBytes(data);
        }

        public string ProcessAssetDataString(string data, IUserService userService)
        {
            Regex regex = new Regex("(creator_url|owner_url)\\s+(\\S+)");

            data = regex.Replace(data, delegate(Match m)
            {
                string result = String.Empty;

                string key = m.Groups[1].Captures[0].Value;

                string value = m.Groups[2].Captures[0].Value;

                Uri userUri;

                switch (key)
                {
                    case "creator_url":
                        userUri = new Uri(value);
                        result = "creator_id " + ResolveUserUri(userService, userUri);
                        break;

                    case "owner_url":
                        userUri = new Uri(value);
                        result = "owner_id " + ResolveUserUri(userService, userUri);
                        break;
                }

                return result;
            });

            return data;
        }

        private Guid ResolveUserUri(IUserService userService, Uri userUri)
        {
            Guid id;
            UserProfileData userProfile = userService.GetUserProfile(userUri);
            if (userProfile == null)
            {
                id = Guid.Empty;
            }
            else
            {
                id = userProfile.ID.Guid;
            }
            return id;
        }


        public class AssetRequest
        {
            public IClientAPI RequestUser;
            public UUID RequestAssetID;
            public AssetInfo AssetInf;
            public TextureImage ImageInfo;
            public UUID TransferRequestID;
            public long DataPointer = 0;
            public int NumPackets = 0;
            public int PacketCounter = 0;
            public bool IsTextureRequest;
            public byte AssetRequestSource = 2;
            public byte[] Params = null;
            //public bool AssetInCache;
            //public int TimeRequested;
            public int DiscardLevel = -1;
        }

        public class AssetInfo : AssetBase
        {
            public AssetInfo(AssetBase aBase)
            {
                Data = aBase.Data;
                FullID = aBase.FullID;
                Type = aBase.Type;
                Name = aBase.Name;
                Description = aBase.Description;
            }

            public const string Secret = "secret";
        }

        public class TextureImage : AssetBase
        {
            public TextureImage(AssetBase aBase)
            {
                Data = aBase.Data;
                FullID = aBase.FullID;
                Type = aBase.Type;
                Name = aBase.Name;
                Description = aBase.Description;
            }
        }

        /// <summary>
        /// A list of requests for a particular asset.
        /// </summary>
        public class AssetRequestsList
        {
            /// <summary>
            /// A list of requests for assets
            /// </summary>
            public List<NewAssetRequest> Requests = new List<NewAssetRequest>();

            /// <summary>
            /// Record the time that this request was first made.
            /// </summary>
            public DateTime TimeRequested;
        }

        /// <summary>
        /// Represent a request for an asset that has yet to be fulfilled.
        /// </summary>
        public class NewAssetRequest
        {
            public AssetRequestCallback Callback;

            public NewAssetRequest(AssetRequestCallback callback)
            {
                Callback = callback;
            }
        }
    }
}