using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; using JD.Collections.Extensions; using JD.Core.ProjectManaging; using JD.Serializers.Stream; using NUnit.Framework; using Packager.Parameters; using PluginCommon.Bundle; using PluginCommon.Bundle.Core; using TeaBox; using DictionaryExtensions = Ubisoft.Collections.DictionaryExtensions; using File = System.IO.File; using LogCategory = JD.LogService.LogCategory; namespace Packager.Tasks { internal class BundleHeaders { public BundleBootHeader BootHeader { get; set; } public List FileHeaders { get; set; } public BundleHeaders() { BootHeader = new BundleBootHeader(); FileHeaders = new List(); } } internal class BundleFileMap : Dictionary> { public void Add(string bundlePath, string filePath) { if (ContainsKey(bundlePath)) { this[bundlePath].Add(filePath); } else { Add(bundlePath, new List { filePath }); } } } /// /// In the bundle binary stream we just have to : /// 1. find the file headers of the files that has to be replaced. /// 2. construct the their file headers again. /// 3. append their modified content stream at the end of the binary stream. /// /// Information needed: /// 1. composition of the binary .ipk bundle file? /// The bundle is divided into 3 parts: /// 1. bootHeader /// 2. List of file headers /// 3. data/content of each file /// /// 2. how is the bundle binary file content read? /// The content is read using file headers. The file header has the starting location of their respective content. /// There is a header list in which would be traversed one by one, and thus all the binary content is read. /// /// 3. what is the role of the "fileStart" location, in bundle's boot header? /// This information is needed while writing the files' content in the bundle to mark the starting location where the content writing starts. /// /// The file header's the field: 'Position' denotes the location of the content respective to that file header. /// eg. If 6024, is the Position in the file header. Then this is the location of this file's content relative to the starting location of data section of the bundle, /// the real absolute location for this content in the entire binary stream is : /// "fileStart Location" + 6024. /// | /// |------> = sizeof(bootHeader + all fileHeaders) /// /// Since, the size of the file header is fixed, and we already know how many file headers are actually there, /// and the size of the boot header is also fixed, this way we know the exact location from where start reading the file header to put them in a list. /// public class ReplaceFilesInBundleTask : AssetCookerTask { [Input("Bundles to Inject In", IOCardinality.Multi, IOType.File)] public ReferenceList Bundles { get; set; } [Input("Files to inject in the bundles", IOCardinality.Multi, IOType.String)] public ReferenceList FilestoInject { get; set; } private readonly Dictionary _bundlePointerLocations = new Dictionary(); private readonly BundleFileMap _bundleFileMap = new BundleFileMap(); public ReplaceFilesInBundleTask() { Bundles = new ReferenceList(); FilestoInject = new ReferenceList(); CookList = true; } protected override void StartInternal(BuildConfiguration profile, cookerWrapper cooker) { foreach (var bundlePath in Bundles) { ModifyBundle(bundlePath); } LogTaskInformation(); } internal void LogTaskInformation() { var taskInfo = new StringBuilder(); taskInfo.Append("Files replaced : \n"); FilestoInject.ForEach(x => taskInfo.Append(x + "\n")); taskInfo.Append("Bundles modified: \n"); Bundles.ForEach(bundle => taskInfo.Append(bundle + "\n")); LogMessage(taskInfo.ToString(), LogCategory.Message); } public void ModifyBundle(string bundlePath) { var bundleHeader = GetBundleHeaderInfo(bundlePath); if (ModifyBundleFileHeaders(bundlePath, bundleHeader)) { using (var bundleStream = File.Open(bundlePath, FileMode.Open, FileAccess.ReadWrite)) { UpdateOriginalBundleHeaders(bundleStream, bundleHeader); UpdateOriginalBundleContent(bundlePath, bundleStream); } } } internal BundleHeaders GetBundleHeaderInfo(string bundlePath) { var bundleHeaderInfo = new BundleHeaders(); using (var bundle = new Bundle(bundlePath)) { bundleHeaderInfo.BootHeader = bundle.BootHeader; bundleHeaderInfo.FileHeaders.AddRange(bundle.FileHeaders); // maintaining a dictionary to keep the track of, end of stream for the bundle's binary stream. DictionaryExtensions.AddOrUpdate(_bundlePointerLocations, bundlePath, (ulong) bundle.GetBundleBinaryStreamLength(), UpdateValueFactory); } return bundleHeaderInfo; } private ulong UpdateValueFactory(string key, ulong value) { _bundlePointerLocations[key] = value; return value; } private bool ModifyBundleFileHeaders(string bundlePath, BundleHeaders bundleHeader) { var isHeaderModified = false; FilestoInject.ForEach(file => { if (ModifyFileHeader(file, bundleHeader, bundlePath)) { UpdateBundlePointer(file, bundlePath); _bundleFileMap.Add(bundlePath, file); isHeaderModified = true; } }); return isHeaderModified; } private static void UpdateOriginalBundleHeaders(FileStream bundleStream, BundleHeaders bundleHeader) { var bundleWriter = new BundleWriter(bundleStream, bundleHeader.BootHeader, bundleHeader.FileHeaders); bundleWriter.WriteHeader(); } private void UpdateOriginalBundleContent(string bundlePath, FileStream bundleStream) { _bundleFileMap[bundlePath].ForEach(file => { bundleStream.Seek(0, SeekOrigin.End); var fileStream = File.Open(DataPath.GetAbsolutePath(file), FileMode.Open, FileAccess.Read); fileStream.AppendTo(bundleStream); fileStream.Close(); }); } internal bool ModifyFileHeader(string filePath, BundleHeaders bundleHeader, string bundlePath) { var fileHeader = bundleHeader.FileHeaders.FirstOrDefault(file => file.FilePath.Contains(filePath)); if (fileHeader == null) return false; fileHeader.RuntimeHeader.Positions[0] = _bundlePointerLocations[bundlePath] - bundleHeader.BootHeader.FilesStart; UpdateSizeInformationInHeader(filePath, fileHeader); return true; } private void UpdateSizeInformationInHeader(string filePath, FileHeaderBuild fileHeader) { var fileDataStream = File.Open(DataPath.GetAbsolutePath(filePath), FileMode.Open, FileAccess.Read); fileHeader.RuntimeHeader.OriginalSize = (uint)fileDataStream.Length; fileHeader.RuntimeHeader.CompressedSize = GetStreamCompressedSize(fileDataStream); fileDataStream.Close(); } private void UpdateBundlePointer(string filePath, string bundlePath) { using (var fileContentStream = File.Open(DataPath.GetAbsolutePath(filePath), FileMode.Open, FileAccess.Read)) { _bundlePointerLocations[bundlePath] += (ulong)fileContentStream.Length; } } /// /// this is the function responsible for calculating the compresses length of a stream, /// the functionality of this function has to be exactly same the compression function in the engine /// in file X:\JD_CODE\main\src\engine\file\BundleBuilder.cpp, in function : compressFile() /// /// /// private uint GetStreamCompressedSize(FileStream newContent) { uint compressedSize = 0; return compressedSize; } } [TestFixture] internal class TestInjectFilesTask { private bool _filesLogged; private bool _bundlesLogged; private readonly string _bundlePath =Path.Combine(ProjectManager.Instance.CurrentProject.DataPath, @"output\durango_loose\bundle_durango.ipk"); private readonly string _fileToReplace = @"cache/itf_cooked/durango/enginedata/gameconfig/padrumbleconfig.isg.ckd"; private void Initialization(ReplaceFilesInBundleTask injector) { injector.Bundles.Add(_bundlePath); injector.FilestoInject.Add(_fileToReplace); injector.InstanceName = "Testing Replacement"; } private void Logger(object sender, string message) { Console.WriteLine(message); if (message.Contains("Files")) { _filesLogged = true; } if (message.Contains("Bundles")) { _bundlesLogged = true; } } [Test] public void TestLogFilesToReplace() { var injector = new ReplaceFilesInBundleTask(); Initialization(injector); injector.Logger.OnInfoLogged += Logger; injector.Logger.OnWarningLogged += Logger; injector.Logger.OnErrorLogged += Logger; injector.LogTaskInformation(); Assert.IsTrue(_filesLogged); Assert.IsTrue(_bundlesLogged); injector.Logger.OnErrorLogged -= Logger; injector.Logger.OnWarningLogged -= Logger; injector.Logger.OnInfoLogged -= Logger; } [Test] public void TestExtractHeaderInfoForBundle() { var injector = new ReplaceFilesInBundleTask(); Initialization(injector); var bundleHeader = injector.GetBundleHeaderInfo(_bundlePath); Assert.NotNull(bundleHeader); } [Test] public void TestModifyFileHeaderForFileToReplace() { var injector = new ReplaceFilesInBundleTask(); Initialization(injector); var bundleHeader = injector.GetBundleHeaderInfo(_bundlePath); var runtimeHeaderForFile = bundleHeader.FileHeaders.FirstOrDefault(file => file.FilePath.Contains(_fileToReplace)); if (runtimeHeaderForFile != null) { var filePosition = runtimeHeaderForFile.RuntimeHeader.Positions[0]; injector.ModifyFileHeader(_fileToReplace, bundleHeader, _bundlePath); var postModification = runtimeHeaderForFile.RuntimeHeader.Positions[0]; Assert.AreNotEqual(postModification, filePosition); return; } Assert.IsTrue(true, "the file is not present in the bundle, therefore the test case is invalid"); } [Test] public void TestModifiedBundleForReplacedContent() { var injector = new ReplaceFilesInBundleTask(); Initialization(injector); var oldBundleSize = GetBundleStreamSize(); injector.ModifyBundle(_bundlePath); var newBundleSize = GetBundleStreamSize(); long fileSize; using (var file = File.Open(DataPath.GetAbsolutePath(_fileToReplace), FileMode.Open, FileAccess.Read)) { fileSize = file.Length; } Assert.AreEqual(oldBundleSize + fileSize, newBundleSize); } private long GetBundleStreamSize() { var bundle = new Bundle(_bundlePath); var size = bundle.GetBundleBinaryStreamLength(); bundle.Dispose(); return size; } } }