/*
* Copyright (c) Contributors, http://opensimulator.org/
* See CONTRIBUTORS.TXT for a full list of copyright holders.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of the OpenSimulator Project nor the
* names of its contributors may be used to endorse or promote products
* derived from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY
* EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
* WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
* DISCLAIMED. IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY
* DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
* (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
* LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
* ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Reflection;
using System.Runtime;
using System.Text.RegularExpressions;
using System.Threading;
using System.Xml;
using log4net;
using OpenMetaverse;
using OpenSim.Framework;
using OpenSim.Framework.Monitoring;
using OpenSim.Framework.Serialization;
using OpenSim.Region.CoreModules.World.Terrain;
using OpenSim.Region.Framework.Interfaces;
using OpenSim.Region.Framework.Scenes;
using Ionic.Zlib;
using GZipStream = Ionic.Zlib.GZipStream;
using CompressionMode = Ionic.Zlib.CompressionMode;
using CompressionLevel = Ionic.Zlib.CompressionLevel;
using OpenSim.Framework.Serialization.External;
using PermissionMask = OpenSim.Framework.PermissionMask;
namespace OpenSim.Region.CoreModules.World.Archiver
{
///
/// Prepare to write out an archive.
///
public class ArchiveWriteRequest
{
private static readonly ILog m_log = LogManager.GetLogger(MethodBase.GetCurrentMethod().DeclaringType);
///
/// The minimum major version of OAR that we can write.
///
public static int MIN_MAJOR_VERSION = 0;
///
/// The maximum major version of OAR that we can write.
///
public static int MAX_MAJOR_VERSION = 1;
///
/// Whether we're saving a multi-region archive.
///
public bool MultiRegionFormat { get; set; }
///
/// Determine whether this archive will save assets. Default is true.
///
public bool SaveAssets { get; set; }
///
/// Determines which objects will be included in the archive, according to their permissions.
/// Default is null, meaning no permission checks.
///
public string FilterContent { get; set; }
protected Scene m_rootScene;
protected Stream m_saveStream;
protected TarArchiveWriter m_archiveWriter;
protected Guid m_requestId;
protected Dictionary m_options;
///
/// Constructor
///
/// Calling module
/// The path to which to save data.
/// The id associated with this request
///
/// If there was a problem opening a stream for the file specified by the savePath
///
public ArchiveWriteRequest(Scene scene, string savePath, Guid requestId) : this(scene, requestId)
{
try
{
m_saveStream = new GZipStream(new FileStream(savePath, FileMode.Create), CompressionMode.Compress, CompressionLevel.BestCompression);
}
catch (EntryPointNotFoundException e)
{
m_log.ErrorFormat(
"[ARCHIVER]: Mismatch between Mono and zlib1g library version when trying to create compression stream."
+ "If you've manually installed Mono, have you appropriately updated zlib1g as well?");
m_log.ErrorFormat("{0} {1}", e.Message, e.StackTrace);
}
}
///
/// Constructor.
///
/// The root scene to archive
/// The stream to which to save data.
/// The id associated with this request
public ArchiveWriteRequest(Scene scene, Stream saveStream, Guid requestId) : this(scene, requestId)
{
m_saveStream = saveStream;
}
protected ArchiveWriteRequest(Scene scene, Guid requestId)
{
m_rootScene = scene;
m_requestId = requestId;
m_archiveWriter = null;
MultiRegionFormat = false;
SaveAssets = true;
FilterContent = null;
}
///
/// Archive the region requested.
///
/// if there was an io problem with creating the file
public void ArchiveRegion(Dictionary options)
{
m_options = options;
if (options.ContainsKey("all") && (bool)options["all"])
MultiRegionFormat = true;
if (options.ContainsKey("noassets") && (bool)options["noassets"])
SaveAssets = false;
if (options.TryGetValue("checkPermissions", out Object temp))
FilterContent = (string)temp;
// Find the regions to archive
ArchiveScenesGroup scenesGroup = new ArchiveScenesGroup();
if (MultiRegionFormat)
{
m_log.InfoFormat("[ARCHIVER]: Saving {0} regions", SceneManager.Instance.Scenes.Count);
SceneManager.Instance.ForEachScene(delegate(Scene scene)
{
scenesGroup.AddScene(scene);
});
}
else
{
scenesGroup.AddScene(m_rootScene);
}
scenesGroup.CalcSceneLocations();
m_archiveWriter = new TarArchiveWriter(m_saveStream);
try
{
// Write out control file. It should be first so that it will be found ASAP when loading the file.
m_archiveWriter.WriteFile(ArchiveConstants.CONTROL_FILE_PATH, CreateControlFile(scenesGroup));
m_log.InfoFormat("[ARCHIVER]: Added control file to archive.");
// Archive the regions
Dictionary assetUuids = new Dictionary();
HashSet failedIDs = new HashSet();
HashSet uncertainAssetsUUIDs = new HashSet();
scenesGroup.ForEachScene(delegate(Scene scene)
{
string regionDir = MultiRegionFormat ? scenesGroup.GetRegionDir(scene.RegionInfo.RegionID) : "";
ArchiveOneRegion(scene, regionDir, assetUuids, failedIDs, uncertainAssetsUUIDs);
});
// Archive the assets
if (SaveAssets)
{
m_log.DebugFormat("[ARCHIVER]: Saving {0} assets", assetUuids.Count);
AssetsRequest ar = new AssetsRequest(
new AssetsArchiver(m_archiveWriter), assetUuids,
failedIDs.Count,
m_rootScene.AssetService, m_rootScene.UserAccountService,
m_rootScene.RegionInfo.ScopeID, options, null);
ar.Execute();
assetUuids = null;
}
else
{
m_log.DebugFormat("[ARCHIVER]: Not saving assets since --noassets was specified");
// CloseArchive(string.Empty);
}
CloseArchive(string.Empty);
}
catch (Exception e)
{
CloseArchive(e.Message);
throw;
}
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.Default;
}
private void ArchiveOneRegion(Scene scene, string regionDir, Dictionary assetUuids,
HashSet failedIDs, HashSet uncertainAssetsUUIDs)
{
m_log.InfoFormat("[ARCHIVER]: Writing region {0}", scene.Name);
EntityBase[] entities = scene.GetEntities();
List sceneObjects = new List();
int numObjectsSkippedPermissions = 0;
// Filter entities so that we only have scene objects.
// FIXME: Would be nicer to have this as a proper list in SceneGraph, since lots of methods
// end up having to do this
IPermissionsModule permissionsModule = scene.RequestModuleInterface();
foreach (EntityBase entity in entities)
{
if (entity is SceneObjectGroup)
{
SceneObjectGroup sceneObject = entity as SceneObjectGroup;
if (!sceneObject.IsDeleted && !sceneObject.IsAttachment && !sceneObject.IsTemporary && !sceneObject.inTransit)
{
if (!CanUserArchiveObject(scene.RegionInfo.EstateSettings.EstateOwner, sceneObject, FilterContent, permissionsModule))
{
// The user isn't allowed to copy/transfer this object, so it will not be included in the OAR.
++numObjectsSkippedPermissions;
}
else
{
sceneObjects.Add(sceneObject);
}
}
}
}
if (SaveAssets)
{
UuidGatherer assetGatherer = new UuidGatherer(scene.AssetService, assetUuids, failedIDs, uncertainAssetsUUIDs);
int prevAssets = assetUuids.Count;
foreach (SceneObjectGroup sceneObject in sceneObjects)
{
int curErrorCntr = assetGatherer.ErrorCount;
int possible = assetGatherer.possibleNotAssetCount;
assetGatherer.AddForInspection(sceneObject);
assetGatherer.GatherAll();
curErrorCntr = assetGatherer.ErrorCount - curErrorCntr;
possible = assetGatherer.possibleNotAssetCount - possible;
if(curErrorCntr > 0)
{
m_log.ErrorFormat("[ARCHIVER]: object {0} '{1}', at {2}, contains {3} references to missing or damaged assets",
sceneObject.UUID, sceneObject.Name ,sceneObject.AbsolutePosition.ToString(), curErrorCntr);
if(possible > 0)
m_log.WarnFormat("[ARCHIVER Warning]: object also contains {0} references that may not be assets or are missing", possible);
}
else if(possible > 0)
{
m_log.WarnFormat("[ARCHIVER Warning]: object {0} '{1}', at {2}, contains {3} references that may not be assets or are missing",
sceneObject.UUID, sceneObject.Name ,sceneObject.AbsolutePosition.ToString(), possible);
}
}
assetGatherer.GatherAll();
GC.Collect();
int errors = assetGatherer.FailedUUIDs.Count;
m_log.DebugFormat(
"[ARCHIVER]: {0} region scene objects to save reference {1} possible assets",
sceneObjects.Count, assetUuids.Count - prevAssets + errors);
if(errors > 0)
m_log.DebugFormat("[ARCHIVER]: {0} of these have problems or are not assets and will be ignored", errors);
}
if (numObjectsSkippedPermissions > 0)
{
m_log.DebugFormat(
"[ARCHIVER]: {0} scene objects skipped due to lack of permissions",
numObjectsSkippedPermissions);
}
// Make sure that we also request terrain texture assets
RegionSettings regionSettings = scene.RegionInfo.RegionSettings;
if (regionSettings.TerrainTexture1 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_1)
assetUuids[regionSettings.TerrainTexture1] = (sbyte)AssetType.Texture;
if (regionSettings.TerrainTexture2 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_2)
assetUuids[regionSettings.TerrainTexture2] = (sbyte)AssetType.Texture;
if (regionSettings.TerrainTexture3 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_3)
assetUuids[regionSettings.TerrainTexture3] = (sbyte)AssetType.Texture;
if (regionSettings.TerrainTexture4 != RegionSettings.DEFAULT_TERRAIN_TEXTURE_4)
assetUuids[regionSettings.TerrainTexture4] = (sbyte)AssetType.Texture;
if (regionSettings.TerrainPBR1 != RegionSettings.DEFAULT_TERRAIN_PBR_1)
assetUuids[regionSettings.TerrainPBR1] = (sbyte)AssetType.Texture; // it can be both tex or material
if (regionSettings.TerrainPBR2 != RegionSettings.DEFAULT_TERRAIN_PBR_2)
assetUuids[regionSettings.TerrainPBR2] = (sbyte)AssetType.Texture;
if (regionSettings.TerrainPBR3 != RegionSettings.DEFAULT_TERRAIN_PBR_3)
assetUuids[regionSettings.TerrainPBR3] = (sbyte)AssetType.Texture;
if (regionSettings.TerrainPBR4 != RegionSettings.DEFAULT_TERRAIN_PBR_4)
assetUuids[regionSettings.TerrainPBR4] = (sbyte)AssetType.Texture;
if (scene.RegionEnvironment != null)
scene.RegionEnvironment.GatherAssets(assetUuids);
List landObjects = scene.LandChannel.AllParcels();
foreach (ILandObject lo in landObjects)
{
if(lo.LandData != null && lo.LandData.Environment != null)
lo.LandData.Environment.GatherAssets(assetUuids);
}
Save(scene, sceneObjects, regionDir);
GC.Collect();
}
///
/// Checks whether the user has permission to export an object group to an OAR.
///
/// The user
/// The object group
/// Which permissions to check: "C" = Copy, "T" = Transfer
/// The scene's permissions module
/// Whether the user is allowed to export the object to an OAR
private bool CanUserArchiveObject(UUID user, SceneObjectGroup objGroup, string filterContent, IPermissionsModule permissionsModule)
{
if (filterContent == null)
return true;
if (permissionsModule == null)
return true; // this shouldn't happen
// Check whether the user is permitted to export all of the parts in the SOG. If any
// part can't be exported then the entire SOG can't be exported.
bool permitted = true;
//int primNumber = 1;
foreach (SceneObjectPart obj in objGroup.Parts)
{
uint perm;
PermissionClass permissionClass = permissionsModule.GetPermissionClass(user, obj);
switch (permissionClass)
{
case PermissionClass.Owner:
perm = obj.BaseMask;
break;
case PermissionClass.Group:
perm = obj.GroupMask | obj.EveryoneMask;
break;
case PermissionClass.Everyone:
default:
perm = obj.EveryoneMask;
break;
}
bool canCopy = (perm & (uint)PermissionMask.Copy) != 0;
bool canTransfer = (perm & (uint)PermissionMask.Transfer) != 0;
// Special case: if Everyone can copy the object then this implies it can also be
// Transferred.
// However, if the user is the Owner then we don't check EveryoneMask, because it seems that the mask
// always (incorrectly) includes the Copy bit set in this case. But that's a mistake: the viewer
// does NOT show that the object has Everyone-Copy permissions, and doesn't allow it to be copied.
if (permissionClass != PermissionClass.Owner)
canTransfer |= (obj.EveryoneMask & (uint)PermissionMask.Copy) != 0;
bool partPermitted = true;
if (filterContent.Contains("C") && !canCopy)
partPermitted = false;
if (filterContent.Contains("T") && !canTransfer)
partPermitted = false;
// If the user is the Creator of the object then it can always be included in the OAR
bool creator = (obj.CreatorID.Guid == user.Guid);
if (creator)
partPermitted = true;
//string name = (objGroup.PrimCount == 1) ? objGroup.Name : string.Format("{0} ({1}/{2})", obj.Name, primNumber, objGroup.PrimCount);
//m_log.DebugFormat("[ARCHIVER]: Object permissions: {0}: Base={1:X4}, Owner={2:X4}, Everyone={3:X4}, permissionClass={4}, checkPermissions={5}, canCopy={6}, canTransfer={7}, creator={8}, permitted={9}",
// name, obj.BaseMask, obj.OwnerMask, obj.EveryoneMask,
// permissionClass, checkPermissions, canCopy, canTransfer, creator, partPermitted);
if (!partPermitted)
{
permitted = false;
break;
}
//++primNumber;
}
return permitted;
}
///
/// Create the control file.
///
///
public string CreateControlFile(ArchiveScenesGroup scenesGroup)
{
int majorVersion;
int minorVersion;
if (MultiRegionFormat)
{
majorVersion = MAX_MAJOR_VERSION;
minorVersion = 0;
}
else
{
// To support older versions of OpenSim, we continue to create single-region OARs
// using the old file format. In the future this format will be discontinued.
majorVersion = 0;
minorVersion = 8;
}
//
// if (m_options.ContainsKey("version"))
// {
// string[] parts = m_options["version"].ToString().Split('.');
// if (parts.Length >= 1)
// {
// majorVersion = Int32.Parse(parts[0]);
//
// if (parts.Length >= 2)
// minorVersion = Int32.Parse(parts[1]);
// }
// }
//
// if (majorVersion < MIN_MAJOR_VERSION || majorVersion > MAX_MAJOR_VERSION)
// {
// throw new Exception(
// string.Format(
// "OAR version number for save must be between {0} and {1}",
// MIN_MAJOR_VERSION, MAX_MAJOR_VERSION));
// }
// else if (majorVersion == MAX_MAJOR_VERSION)
// {
// // Force 1.0
// minorVersion = 0;
// }
// else if (majorVersion == MIN_MAJOR_VERSION)
// {
// // Force 0.4
// minorVersion = 4;
// }
m_log.InfoFormat("[ARCHIVER]: Creating version {0}.{1} OAR", majorVersion, minorVersion);
if (majorVersion == 1)
{
m_log.WarnFormat("[ARCHIVER]: Please be aware that version 1.0 OARs are not compatible with OpenSim versions prior to 0.7.4. Do not use the --all option if you want to produce a compatible OAR");
}
String s;
using (StringWriter sw = new StringWriter())
{
using (XmlTextWriter xtw = new XmlTextWriter(sw))
{
xtw.Formatting = Formatting.Indented;
xtw.WriteStartDocument();
xtw.WriteStartElement("archive");
xtw.WriteAttributeString("major_version", majorVersion.ToString());
xtw.WriteAttributeString("minor_version", minorVersion.ToString());
xtw.WriteStartElement("creation_info");
DateTime now = DateTime.UtcNow;
TimeSpan t = now - new DateTime(1970, 1, 1);
xtw.WriteElementString("datetime", ((int)t.TotalSeconds).ToString());
if (!MultiRegionFormat)
xtw.WriteElementString("id", m_rootScene.RegionInfo.RegionID.ToString());
xtw.WriteEndElement();
xtw.WriteElementString("assets_included", SaveAssets.ToString());
if (MultiRegionFormat)
{
WriteRegionsManifest(scenesGroup, xtw);
}
else
{
xtw.WriteStartElement("region_info");
WriteRegionInfo(m_rootScene, xtw);
xtw.WriteEndElement();
}
xtw.WriteEndElement();
xtw.Flush();
}
s = sw.ToString();
}
return s;
}
///
/// Writes the list of regions included in a multi-region OAR.
///
private static void WriteRegionsManifest(ArchiveScenesGroup scenesGroup, XmlTextWriter xtw)
{
xtw.WriteStartElement("regions");
// Write the regions in order: rows from South to North, then regions from West to East.
// The list of regions can have "holes"; we write empty elements in their position.
for (uint y = (uint)scenesGroup.Rect.Top; y < scenesGroup.Rect.Bottom; ++y)
{
if (scenesGroup.Regions.TryGetValue(y, out SortedDictionary row))
{
xtw.WriteStartElement("row");
for (uint x = (uint)scenesGroup.Rect.Left; x < scenesGroup.Rect.Right; ++x)
{
if (row.TryGetValue(x, out Scene scene))
{
xtw.WriteStartElement("region");
xtw.WriteElementString("id", scene.RegionInfo.RegionID.ToString());
xtw.WriteElementString("dir", scenesGroup.GetRegionDir(scene.RegionInfo.RegionID));
WriteRegionInfo(scene, xtw);
xtw.WriteEndElement();
}
else
{
// Write a placeholder for a missing region
xtw.WriteElementString("region", "");
}
}
xtw.WriteEndElement();
}
else
{
// Write a placeholder for a missing row
xtw.WriteElementString("row", "");
}
}
xtw.WriteEndElement(); // "regions"
}
protected static void WriteRegionInfo(Scene scene, XmlTextWriter xtw)
{
Vector2 size;
size = new Vector2((float)scene.RegionInfo.RegionSizeX, (float)scene.RegionInfo.RegionSizeY);
xtw.WriteElementString("size_in_meters", string.Format("{0},{1}", size.X, size.Y));
}
protected void Save(Scene scene, List sceneObjects, string regionDir)
{
if (regionDir != string.Empty)
regionDir = ArchiveConstants.REGIONS_PATH + regionDir + "/";
m_log.InfoFormat("[ARCHIVER]: Adding region settings to archive.");
// Write out region settings
string settingsPath = String.Format("{0}{1}{2}.xml",
regionDir, ArchiveConstants.SETTINGS_PATH, scene.RegionInfo.RegionName);
m_archiveWriter.WriteFile(settingsPath, RegionSettingsSerializer.Serialize(scene.RegionInfo.RegionSettings, scene.RegionEnvironment, scene.RegionInfo.EstateSettings));
m_log.InfoFormat("[ARCHIVER]: Adding parcel settings to archive.");
// Write out land data (aka parcel) settings
List landObjects = scene.LandChannel.AllParcels();
foreach (ILandObject lo in landObjects)
{
LandData landData = lo.LandData;
string landDataPath
= String.Format("{0}{1}", regionDir, ArchiveConstants.CreateOarLandDataPath(landData));
m_archiveWriter.WriteFile(landDataPath, LandDataSerializer.Serialize(landData, m_options));
}
m_log.InfoFormat("[ARCHIVER]: Adding terrain information to archive.");
// Write out terrain
string terrainPath = String.Format("{0}{1}{2}.r32",
regionDir, ArchiveConstants.TERRAINS_PATH, scene.RegionInfo.RegionName);
using (MemoryStream ms = new MemoryStream())
{
scene.RequestModuleInterface().SaveToStream(terrainPath, ms);
m_archiveWriter.WriteFile(terrainPath, ms.ToArray());
}
m_log.InfoFormat("[ARCHIVER]: Adding scene objects to archive.");
// Write out scene object metadata
IRegionSerialiserModule serializer = scene.RequestModuleInterface();
foreach (SceneObjectGroup sceneObject in sceneObjects)
{
//m_log.DebugFormat("[ARCHIVER]: Saving {0} {1}, {2}", entity.Name, entity.UUID, entity.GetType());
if(sceneObject.IsDeleted || sceneObject.inTransit)
continue;
string serializedObject = serializer.SerializeGroupToXml2(sceneObject, m_options);
string objectPath = string.Format("{0}{1}", regionDir, ArchiveHelpers.CreateObjectPath(sceneObject));
m_archiveWriter.WriteFile(objectPath, serializedObject);
}
}
protected void ReceivedAllAssets(ICollection assetsFoundUuids, ICollection assetsNotFoundUuids, bool timedOut)
{
string errorMessage;
if (timedOut)
{
errorMessage = "Loading assets timed out";
}
else
{
foreach (UUID uuid in assetsNotFoundUuids)
{
m_log.DebugFormat("[ARCHIVER]: Could not find asset {0}", uuid);
}
// m_log.InfoFormat(
// "[ARCHIVER]: Received {0} of {1} assets requested",
// assetsFoundUuids.Count, assetsFoundUuids.Count + assetsNotFoundUuids.Count);
errorMessage = String.Empty;
}
CloseArchive(errorMessage);
}
///
/// Closes the archive and notifies that we're done.
///
/// The error that occurred, or empty for success
protected void CloseArchive(string errorMessage)
{
try
{
if (m_archiveWriter != null)
m_archiveWriter.Close();
m_saveStream.Close();
}
catch (Exception e)
{
m_log.Error(string.Format("[ARCHIVER]: Error closing archive: {0} ", e.Message), e);
if (errorMessage.Length == 0)
errorMessage = e.Message;
}
m_log.InfoFormat("[ARCHIVER]: Finished writing out OAR for {0}", m_rootScene.RegionInfo.RegionName);
m_rootScene.EventManager.TriggerOarFileSaved(m_requestId, errorMessage);
}
}
}