401 lines
15 KiB
C#
401 lines
15 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.ComponentModel;
|
|
using System.IO;
|
|
using System.Linq;
|
|
using System.Net;
|
|
using JD.Core.ProjectManaging;
|
|
using Newtonsoft.Json.Linq;
|
|
using TeaBox.Macros;
|
|
using TeaBox.ObjectModel;
|
|
using TeaBox.Online;
|
|
using TeaBox.UAF;
|
|
using TeaBox.Utilities.Secrets;
|
|
using Ubisoft;
|
|
|
|
namespace TeaBox.Scripts.Data
|
|
{
|
|
[Module(Icon = "Publish")]
|
|
[System.ComponentModel.Category("Data")]
|
|
public class PathIdCollisionCheckMacro : IDisposable
|
|
{
|
|
internal Dictionary<string, List<OnlinePackageMapDetails>> PlatformDetails = new Dictionary<string, List<OnlinePackageMapDetails>>();
|
|
internal ServerName Server { get; private set; }
|
|
internal class OnlinePackageMapDetails
|
|
{
|
|
public string Md5 { get; set; }
|
|
|
|
public int StorageType { get; set; }
|
|
|
|
public string Url { get; set; }
|
|
|
|
public int Version { get; set; }
|
|
|
|
public string MapZip()
|
|
{
|
|
return Path.GetFileName(Url);
|
|
}
|
|
}
|
|
|
|
[Macro(ThreadAffinity.ExecuteAsync, Icon = "run.png")]
|
|
[System.ComponentModel.Description("Download package from the server")]
|
|
[DisplayName(@"Download package from Server")]
|
|
[System.ComponentModel.Category(@"Data\Advanced\Path ID Collision")]
|
|
public static void DownloadOnlinePackageFromServer(IExecutionContext context,
|
|
ServerName serverName,
|
|
RelativePath outputPath,
|
|
bool extractBundlesPostDownload = false)
|
|
{
|
|
|
|
using (var crcCollisionCheck = new PathIdCollisionCheckMacro() { Server = serverName })
|
|
{
|
|
context.AddStep(
|
|
$"Getting online package related Information from {serverName} Server",
|
|
step =>
|
|
{
|
|
crcCollisionCheck.GetPackagesInformation();
|
|
});
|
|
|
|
context.AddStep($"Downloading the packages and dumping them in {outputPath.FullPath}",
|
|
step =>
|
|
{
|
|
crcCollisionCheck.DownloadPackages(outputPath);
|
|
});
|
|
}
|
|
|
|
if (extractBundlesPostDownload)
|
|
{
|
|
context.AddStep(
|
|
"Extracting bundles from downloaded packages",
|
|
step =>
|
|
{
|
|
ExtractBundlesFromZipFiles(outputPath);
|
|
});
|
|
}
|
|
}
|
|
|
|
private void DownloadPackages(RelativePath outputPath)
|
|
{
|
|
var webClient = new WebClient();
|
|
|
|
foreach (var key in PlatformDetails.Keys)
|
|
{
|
|
var packageDetail = PlatformDetails[key];
|
|
|
|
packageDetail.ForEach(packageMapInfo =>
|
|
{
|
|
DownloadZipFile(outputPath, packageMapInfo, key, webClient);
|
|
});
|
|
}
|
|
}
|
|
|
|
private void DownloadZipFile(RelativePath outputPath, OnlinePackageMapDetails packageMapInfo, string key, WebClient webClient)
|
|
{
|
|
var mapZipName = packageMapInfo.MapZip();
|
|
var outputDirectory = Path.Combine(outputPath.FullPath, Server.ToString(), key);
|
|
try
|
|
{
|
|
if (!Directory.Exists(outputDirectory)) Directory.CreateDirectory(outputDirectory);
|
|
|
|
var outputFilePath = Path.Combine(outputDirectory, mapZipName);
|
|
webClient.DownloadFile(new Uri(packageMapInfo.Url), outputFilePath);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
using (var testing = System.IO.File.AppendText(Path.Combine(outputPath.FullPath + "Errorlog.txt")))
|
|
{
|
|
testing.WriteLine($"Issue occured for >> key {key} and for Map >> {mapZipName}, with exception {ex.Message}");
|
|
}
|
|
}
|
|
}
|
|
|
|
private void GetPackagesInformation()
|
|
{
|
|
var secretsManager = SecretsManagement.GetSecretsManager();
|
|
var onlineConfiguration = secretsManager.GetOnlineConfiguration();
|
|
|
|
var restClient = new OnlineRequests(Server, onlineConfiguration);
|
|
var packagesJson = restClient.GetOnlinePackages();
|
|
|
|
if (packagesJson.IsEmpty()) return;
|
|
|
|
var packageInfoJson = JObject.Parse(packagesJson);
|
|
|
|
foreach (var platformInfoObject in packageInfoJson.Children())
|
|
{
|
|
var platformInfo = GetPlatformInfo(platformInfoObject);
|
|
var onlinePackageMapInfo = FetchMapDetailsFromPlatformInfo(platformInfoObject);
|
|
|
|
if (onlinePackageMapInfo.Any())
|
|
{
|
|
PlatformDetails.Add(platformInfo, onlinePackageMapInfo);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static List<OnlinePackageMapDetails> FetchMapDetailsFromPlatformInfo(JToken childObject)
|
|
{
|
|
var onlinePackageMapInfo = new List<OnlinePackageMapDetails>();
|
|
|
|
foreach (var entireObject in childObject.Children())
|
|
{
|
|
ProcessAllMapObjects(entireObject, onlinePackageMapInfo);
|
|
}
|
|
return onlinePackageMapInfo;
|
|
}
|
|
|
|
private static void ProcessAllMapObjects(JToken maps, List<OnlinePackageMapDetails> onlinePackageMapInfo)
|
|
{
|
|
foreach (var map in maps.Children())
|
|
{
|
|
ProcessMapForDetails(onlinePackageMapInfo, map);
|
|
}
|
|
}
|
|
|
|
private static void ProcessMapForDetails(List<OnlinePackageMapDetails> onlinePackageMapInfo, JToken map)
|
|
{
|
|
foreach (var map_property in map.Children())
|
|
{
|
|
onlinePackageMapInfo.Add(map_property.ToObject<OnlinePackageMapDetails>());
|
|
}
|
|
}
|
|
|
|
private static string GetPlatformInfo(JToken childObject)
|
|
{
|
|
var platformName = childObject.ToString().Split(':')[0].Trim();
|
|
platformName = platformName.Substring(1, platformName.Length - 2);
|
|
return platformName;
|
|
}
|
|
|
|
[Macro(ThreadAffinity.ExecuteAsync, Icon = "run.png")]
|
|
[System.ComponentModel.Description("Download all the packages from all the servers one by one")]
|
|
[DisplayName(@"Download packages from all the servers one by one")]
|
|
[System.ComponentModel.Category(@"Data\Advanced\Path ID Collision")]
|
|
public static void DownloadPackagesFromAllServers(
|
|
IExecutionContext context,
|
|
RelativePath outputPath,
|
|
bool extractBundlesPostDownload = false)
|
|
{
|
|
var servers = Enum.GetValues(typeof(ServerName));
|
|
foreach (var server in servers)
|
|
{
|
|
try
|
|
{
|
|
DownloadOnlinePackageFromServer(context, (ServerName)server, new RelativePath(Path.Combine(outputPath.Value, server.ToString())), false);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Application.Main.CreateLogger().LogMessage($"Excetion occured while reaching working with server {(ServerName)server}, Exception : {ex.Message}");
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
[Macro(ThreadAffinity.ExecuteAsync, Icon = "run.png")]
|
|
[System.ComponentModel.Description("Extract package from Zip files")]
|
|
[DisplayName(@"Extracting bundles from Zip Files")]
|
|
[System.ComponentModel.Category(@"Data\Advanced\Path ID Collision")]
|
|
public static void ExtractBundlesFromZipFiles(RelativePath inputPath, RelativePath outputPath = null)
|
|
{
|
|
if (outputPath == null) outputPath = inputPath;
|
|
|
|
var listOfZipFiles = System.IO.Directory.GetFiles(inputPath.FullPath, "*.zip", SearchOption.AllDirectories);
|
|
listOfZipFiles.ForEach(zipFile =>
|
|
{
|
|
var rootFolderForExtraction = System.IO.Directory.GetParent(zipFile).Name;
|
|
var pathForExtractingFile = Path.Combine(outputPath.FullPath, rootFolderForExtraction, Path.GetFileNameWithoutExtension(zipFile));
|
|
try
|
|
{
|
|
ZipUtils.FindAndUnzipEntries(zipFile, ".ipk", pathForExtractingFile);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
using (StreamWriter testing = System.IO.File.AppendText(Path.Combine(outputPath.FullPath, rootFolderForExtraction, "Errorlog.txt")))
|
|
{
|
|
testing.WriteLine(
|
|
$"Issue occured inside >> {zipFile}, while unzipping for Map >> {Path.GetFileNameWithoutExtension(zipFile)}, "
|
|
+ $"with exception {ex.Message}");
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
|
|
[Macro(ThreadAffinity.ExecuteAsync, Icon = "run.png")]
|
|
[Description("Checks for Path ID collision in the bundles")]
|
|
[DisplayName(@"Path ID Collision Check")]
|
|
[Category(@"Data\Advanced\Path ID Collision")]
|
|
public static void PathIDCollisionCheck(IExecutionContext context,
|
|
[Description(@"Location of Bundles to be considered for Path ID collision check, each path seperated by new line")]
|
|
string dataBundlePaths,
|
|
[Description("Output folder for dumping the result of Path ID collision check")]
|
|
RelativePath outputPath)
|
|
{
|
|
var folderName = "Path_ID_Collision_output_" + DateTime.Now.ToString("yyyy-MM-dd-HH-mm-ss-fff");
|
|
|
|
var outputDirectory = !outputPath.Value.IsEmpty() ? outputPath.Value : DataPath.GetAbsolutePath("output");
|
|
outputDirectory = Path.Combine(outputDirectory, folderName);
|
|
|
|
var packagerCommand = new PackagerCommand
|
|
{
|
|
BundleDirectory = dataBundlePaths.Replace('\n', ','),
|
|
OutputPath = outputDirectory,
|
|
TaskListType = PackagerTaskList.BundleFilesPathStringId,
|
|
};
|
|
|
|
packagerCommand.Execute(context);
|
|
|
|
using (var steps = context.ExecuteSteps())
|
|
{
|
|
steps.AddStep("Checking for Collisions", step =>
|
|
{
|
|
var result = ComputePathIdCollisions(outputDirectory);
|
|
DeleteAllIntermediateFiles(outputDirectory);
|
|
var finalCollisionFile = Path.Combine(outputDirectory, "FinalCollision.txt");
|
|
if (File.Exists(finalCollisionFile))
|
|
{
|
|
System.IO.File.Delete(finalCollisionFile);
|
|
}
|
|
WriteCollisionInformationToFile(result, finalCollisionFile, dataBundlePaths);
|
|
});
|
|
}
|
|
}
|
|
|
|
public static Dictionary<string, StringIDInfo> ComputePathIdCollisions(string outputPath)
|
|
{
|
|
var bundleDirectory = new System.IO.DirectoryInfo(outputPath);
|
|
|
|
var stringIdAssetAndIpk = new Dictionary<string, StringIDInfo>();
|
|
|
|
foreach (var intermediateFile in bundleDirectory.GetFiles().Where(x => x.Name.EndsWith(".extra")))
|
|
{
|
|
var fileContentDictionary = System.IO.File
|
|
.ReadLines(outputPath + "\\" + intermediateFile)
|
|
.Select(line => line.Split('\t'))
|
|
.ToDictionary(item => item[0], item => item[1]);
|
|
|
|
// this is for showing the error logs in more precise manner
|
|
var ipkFileName = intermediateFile.Name.Split('(', ')')[1].Replace("_i_", "\\") + ".ipk";
|
|
|
|
foreach (var fileContent in fileContentDictionary)
|
|
{
|
|
var stringIDInfo = new StringIDInfo();
|
|
|
|
if (stringIdAssetAndIpk.ContainsKey(fileContent.Key))
|
|
{
|
|
UpdateStringIdInfo(stringIdAssetAndIpk[fileContent.Key], fileContent.Value, ipkFileName);
|
|
}
|
|
else
|
|
{
|
|
AddNewStringIdInfo(stringIdAssetAndIpk, stringIDInfo, fileContent, ipkFileName);
|
|
}
|
|
}
|
|
}
|
|
|
|
return stringIdAssetAndIpk;
|
|
|
|
}
|
|
|
|
private static void AddNewStringIdInfo(Dictionary<string, StringIDInfo> stringIdAssetAndIPK,
|
|
StringIDInfo stringIDInfo, KeyValuePair<string, string> fileContent, string textFile)
|
|
{
|
|
stringIDInfo.AddAsset(fileContent.Value);
|
|
stringIDInfo.AddIpkFile(textFile);
|
|
stringIdAssetAndIPK.Add(fileContent.Key, stringIDInfo);
|
|
}
|
|
|
|
private static void UpdateStringIdInfo(StringIDInfo stringIDInfo, string fileContent, string textFile)
|
|
{
|
|
stringIDInfo.AddAsset(fileContent);
|
|
stringIDInfo.AddIpkFile(textFile);
|
|
}
|
|
|
|
|
|
private static void WriteCollisionInformationToFile(Dictionary<string, StringIDInfo> collision, string outputPath, string dataBundlePaths)
|
|
{
|
|
var bundlePaths = new List<string>();
|
|
var errors = new List<string>();
|
|
var warnings = new List<string>();
|
|
|
|
bundlePaths.Add("Paths Considered For Path ID Collision Check : \n");
|
|
bundlePaths.AddRange(dataBundlePaths.Split('\n'));
|
|
|
|
errors.Add("\n\nCollisions : \n");
|
|
warnings.Add("\n\nWarnings : ");
|
|
|
|
foreach (var collisionData in collision)
|
|
{
|
|
var stringInfo = collisionData.Value;
|
|
|
|
if (stringInfo.HasCollision)
|
|
errors.Add($"{collisionData.Key}\t{collisionData.Value}\n");
|
|
else if (stringInfo.AssetList.Count > 1)
|
|
warnings.Add($"{collisionData.Key}\t{collisionData.Value}\n");
|
|
}
|
|
|
|
System.IO.File.WriteAllLines(outputPath, bundlePaths);
|
|
System.IO.File.AppendAllLines(outputPath, errors);
|
|
System.IO.File.AppendAllLines(outputPath, warnings);
|
|
|
|
WindowsExplorer.OpenFile(outputPath);
|
|
}
|
|
|
|
private static void DeleteAllIntermediateFiles(string outputPath)
|
|
{
|
|
new System.IO.DirectoryInfo(outputPath).GetFiles("*.extra").ForEach(file => { File.Delete(file.FullName); });
|
|
}
|
|
|
|
public void Dispose()
|
|
{
|
|
|
|
}
|
|
}
|
|
|
|
public class StringIDInfo
|
|
{
|
|
private readonly List<string> _assetList;
|
|
private readonly List<string> _ipkFileName;
|
|
public bool HasCollision { get; private set; }
|
|
|
|
public List<string> AssetList
|
|
{
|
|
get { return _assetList; }
|
|
}
|
|
|
|
public List<string> IpkFileNameList
|
|
{
|
|
get { return _ipkFileName; }
|
|
}
|
|
|
|
public StringIDInfo()
|
|
{
|
|
_assetList = new List<string>();
|
|
_ipkFileName = new List<string>();
|
|
HasCollision = false;
|
|
}
|
|
|
|
public void AddAsset(string assetName)
|
|
{
|
|
if (!_assetList.IsEmpty() && !_assetList.Contains(assetName))
|
|
HasCollision = true;
|
|
_assetList.Add(assetName);
|
|
}
|
|
public void AddIpkFile(string ipkFileName)
|
|
{
|
|
_ipkFileName.Add(ipkFileName);
|
|
}
|
|
|
|
public override string ToString()
|
|
{
|
|
string resultString = null;
|
|
|
|
for (int index = 0; index < _assetList.Count(); index++)
|
|
{
|
|
resultString += string.Format("\n{0} present in {1}", _assetList[index].ToString(), _ipkFileName[index].ToString());
|
|
}
|
|
|
|
return resultString;
|
|
}
|
|
}
|
|
}
|