using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Windows; using System.Xml.Linq; using UAF; using UAF.Interfaces.Logging; using System.Windows.Threading; using System.Diagnostics; using System.Threading; using UAF.Serialization; using Path = System.IO.Path; namespace DataFileAnalyzer { /// /// All the data files of a project /// public class FileLibrary : IDisposable { // The file (in the data folder) where the file library data is saved private static readonly string libraryFilePath = "cache\\FileLibrary.xml"; private string m_dataFolder; public string DataFolder { get { return m_dataFolder; } } private List m_excludeFolders; public List ExcludeFolders { get { return m_excludeFolders; } } private ILog m_logger = null; private FileKeyManager m_keyManager = new FileKeyManager(); public FileKeyManager KeyManager { get { return m_keyManager; } } private bool m_running = false; private bool m_paused = false; private DispatcherTimer m_updateTimer; private Dictionary m_files; public Dictionary Files { get { return m_files; } private set { m_files = value; } } private FileUpdater m_fileUpdater; private bool m_updatingFiles = false; private bool m_updateModifiedFiles = false; private bool m_updateAddedFiles = false; private Stopwatch m_resolveRefStopwatch = new Stopwatch(); // The name of the library thread private string m_threadName = Thread.CurrentThread.Name; // Mutex to lock the members of the class related to the status (running, paused, exiting, etc...) private Mutex m_statusMutex = new Mutex(false, "DataFileAnalyzer.FileLibrary.m_statusMutex"); // Pause as soon as possible private bool m_pauseAsked = false; // Event to set when we exit the pause private readonly AutoResetEvent m_unpauseEvent = new AutoResetEvent(false); // Event to set when the update thread is paused private readonly AutoResetEvent m_pauseSyncEvent = new AutoResetEvent(false); /// /// DEPRECATED - Event invoked when the list of files is modified /// public event EventHandler Changed; /// /// Constructor /// public FileLibrary() { Logger = new ConsoleLog(); m_excludeFolders = new List(); m_excludeFolders.Add("cache"); m_excludeFolders.Add("datatoolbox"); m_excludeFolders.Add("DataToolbox"); m_excludeFolders.Add("Nintendo DLC sheets"); Reset(); } public void Dispose() { if(m_fileUpdater != null) m_fileUpdater.Dispose(); } /// /// Resets the library before changing the data folder. /// private void Reset() { Clear(); m_dataFolder = ""; if(m_fileUpdater != null) m_fileUpdater.Dispose(); m_fileUpdater = new FileUpdater(this); } /// /// DEPRECATED - Invoke the Changed event /// private void OnChanged() { if (Changed != null) { if (m_updatingFiles) m_updateModifiedFiles = true; else Changed(this, EventArgs.Empty); } } /// /// Developper mode /// public bool DevMode { get; set; } /// /// Time spent in ResolveDependencies /// public TimeSpan ResolveDependenciesDuration { get { return m_resolveRefStopwatch.Elapsed; } } /// /// /// Set the data folder. /// true if successful. /// public bool SetDataFolder(string _folder) { Stop(); Reset(); m_dataFolder = _folder; m_fileUpdater.OnDataFolderChanged(_folder, m_excludeFolders); if (!Launch()) { Reset(); return false; } return true; } /// /// The logger. Cannot be set to a null value. /// public ILog Logger { get { return m_logger; } set { if (value != null) { m_logger = value; m_keyManager.Logger = value; if (m_fileUpdater != null) m_fileUpdater.Logger = value; } } } #region Load/Save /// /// Save the current state of the library to a file in the "cache" folder of the data folder. /// true if successful. /// public bool Save() { bool result = true; Pause(); try { // Build the XML tree XDocument document = new XDocument(); document.Add(new XElement("DataFileLibrary")); XElement libraryElement = document.Root; libraryElement.SetAttributeValue("Version", FileLibraryVersion.Latest); foreach (KeyValuePair kvp in m_files) { XElement fileElement = new XElement("DataFile"); result &= kvp.Value.Save(fileElement); libraryElement.Add(fileElement); } if (result) result = m_fileUpdater.Save(libraryElement); // Save the XML tree if (result) { string filePath = Path.Combine(m_dataFolder, libraryFilePath); string outputDir = Path.GetDirectoryName(filePath); if (!Directory.Exists(outputDir)) Directory.CreateDirectory(outputDir); document.Save(filePath); } } catch (System.Exception ex) { m_logger.Error("Exception while saving:\n" + ex.ToString()); return false; } Resume(); if (result) m_logger.Message("Database saved."); else m_logger.Error("Could not save the database correctly."); return result; } /// /// Load the current state of the library from a file in the "cache" folder of the data folder. /// true if successful. /// public bool Load() { // We cannot load while running if (m_running) return false; Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); string filePath = Path.Combine(m_dataFolder, libraryFilePath); if (!File.Exists(filePath)) return false; bool result = true; try { XDocument document = XDocument.Load(filePath); XElement libraryElement = document.Root; uint libraryVersion; XAttribute attr = libraryElement.Attribute("Version"); if (attr == null || !uint.TryParse(attr.Value, out libraryVersion)) libraryVersion = FileLibraryVersion.Oldest; if (libraryVersion > FileLibraryVersion.Latest) { m_logger.Error("The database is too recent for this version of the Data File Analyzer. Please, update your program."); return false; } if (libraryVersion != FileLibraryVersion.Latest) { if (CanUpgradeLibrary(libraryVersion, FileLibraryVersion.Latest)) { m_logger.Message("Library is not at the latest version. It will be upgraded after loading. "); } else { m_logger.Warning("Library is not at the latest version, and can not be upgraded. Will have to rebuild the database."); return false; } } foreach (XElement fileElement in libraryElement.Elements("DataFile")) { DataFile dataFile = new DataFile(); if (dataFile.Load(fileElement)) { m_files.Add(dataFile.Key, dataFile); result &= m_keyManager.RegisterFile(dataFile.Key, dataFile.NormalizedPath); } else result = false; } if (result) result = m_fileUpdater.Load(libraryElement); if (libraryVersion != FileLibraryVersion.Latest) { if (!UpgradeLibrary(libraryVersion, FileLibraryVersion.Latest)) { m_logger.Error("Database upgrade failed. Will have to rebuild the database."); result = false; } } } catch (Exception ex) { m_logger.Error("Exception while loading:\n" + ex.ToString()); result = false; } if (result) { // Create reverse dependencies foreach (KeyValuePair kvp in m_files) { DataFile dataFile = kvp.Value; foreach (DataFileDependency dependency in dataFile.Children) { DataFile childFile = GetFile(dependency.Path); if (childFile != null) childFile.AddParent(dataFile.NormalizedPath, dataFile.Key, dependency.Platforms); } } stopwatch.Stop(); m_logger.Message("Database loaded in {0}.", stopwatch.Elapsed.ToReadableString()); } else { Clear(); m_logger.Error("Could not load the database correctly. Will have to rebuild the database."); } return result; } public bool CanUpgradeLibrary(uint oldVersion, uint requiredVersion) { if (oldVersion < requiredVersion) return false; return true; } public bool UpgradeLibrary(uint oldVersion, uint requiredVersion) { return false; } #endregion #region Launch/Stop/Pause/Resume management /// /// Launch the analysis of the current data folder. /// Try to load the informations saved in the cache folder. /// Launch the FileUpdater that will update the library. /// true if successful. /// public bool Launch() { if (!Running) { // Load the XML file string filePath = Path.Combine(m_dataFolder, libraryFilePath); Load(); m_fileUpdater.Start(); m_updateTimer = new DispatcherTimer(); m_updateTimer.Interval = TimeSpan.FromMilliseconds(100); m_updateTimer.Tick += new EventHandler(Update); m_updateTimer.Start(); m_running = true; m_paused = false; DependenciesCheck(); } return true; } /// /// Stop all the operations /// true if successful. /// public bool Stop() { if (Running) { Resume(); m_updateTimer.Stop(); m_updateTimer = null; m_fileUpdater.Stop(); m_running = false; m_resolveRefStopwatch.Reset(); Save(); } return true; } /// /// Ask for a pause and wait for the library thread to pause. /// private bool PausedAsked { get { m_statusMutex.WaitOne(); bool result = m_pauseAsked; m_statusMutex.ReleaseMutex(); return result; } set { m_statusMutex.WaitOne(); m_pauseAsked = value; m_statusMutex.ReleaseMutex(); } } /// /// Running? /// public bool Running { get { m_statusMutex.WaitOne(); bool result = m_running; m_statusMutex.ReleaseMutex(); return result; } private set { m_statusMutex.WaitOne(); m_running = value; m_statusMutex.ReleaseMutex(); } } /// /// Paused? /// public bool Paused { get { m_statusMutex.WaitOne(); bool result = m_paused; m_statusMutex.ReleaseMutex(); return result; } private set { m_statusMutex.WaitOne(); m_paused = value; m_statusMutex.ReleaseMutex(); } } /// /// Pause all the operation /// public void Pause() { if (Thread.CurrentThread.Name == m_threadName) { // Pause from library thread if (m_running && !m_paused) { m_paused = true; m_fileUpdater.Pause(); } } else { // Pause from another thread m_statusMutex.WaitOne(); if (m_running && !m_paused) { m_pauseAsked = true; m_statusMutex.ReleaseMutex(); m_pauseSyncEvent.WaitOne(); m_statusMutex.WaitOne(); m_paused = true; m_fileUpdater.Pause(); } m_statusMutex.ReleaseMutex(); } } /// /// Resume if paused /// public void Resume() { if (Thread.CurrentThread.Name == m_threadName) { // Resume from library thread if (m_running && m_paused) { m_pauseAsked = false; m_unpauseEvent.Set(); m_fileUpdater.Resume(); m_paused = false; } } else { // Resume from another thread m_statusMutex.WaitOne(); if (m_running && m_paused) { m_pauseAsked = false; m_unpauseEvent.Set(); m_fileUpdater.Resume(); m_paused = false; } m_statusMutex.ReleaseMutex(); } } #endregion /// /// The update function /// private void Update(object sender, EventArgs e) { if (Running && !Paused) { if (PausedAsked) { m_pauseSyncEvent.Set(); m_unpauseEvent.WaitOne(); } m_updatingFiles = true; m_fileUpdater.UpdateLibrary(); m_updatingFiles = false; if (m_updateAddedFiles) ResolveDependencies(); if (m_updateModifiedFiles) OnChanged(); m_updateModifiedFiles = false; m_updateAddedFiles = false; } } /// /// Remove all the files from the library. /// public void Clear() { m_keyManager.Clear(); m_files = new Dictionary(); } /// /// Create a new DataFile /// true if operation succeeded /// public bool CreateNewDataFile(string _path, out DataFile _newFile) { DataFileKey newKey; bool isFullPath = PathUtils.IsRelative(DataFolder, _path); string normalizedPath = PathUtils.Normalize(_path, DataFolder); if (m_keyManager.AddFile(normalizedPath, out newKey)) { _newFile = new DataFile(); _newFile.FilePath = isFullPath ? PathUtils.MakeRelative(_path, DataFolder) : _path; _newFile.NormalizedPath = normalizedPath; _newFile.Key = newKey; _newFile.AddWarning(DataFileError.Codes.NotInPerforce); m_files.Add(newKey, _newFile); if (m_updatingFiles) m_updateAddedFiles = true; // the resolving will be done at the end of the update else ResolveDependencies(); OnChanged(); return true; } _newFile = null; return false; } /// /// Is this file in the library? /// public bool FileExists(string _path) { string normalizedPath = PathUtils.Normalize(_path); DataFileKey key; return m_keyManager.GetFileKey(normalizedPath, out key); } /// /// Get a file from its key /// The DataFile or null if this key is unknown /// public DataFile GetFile(DataFileKey _key) { DataFile dataFile = null; if (!m_files.TryGetValue(_key, out dataFile)) dataFile = null; return dataFile; } /// /// Get a file from its NORMALIZED path /// The DataFile or null if this file is not in the library /// public DataFile GetFile(string _path) { DataFileKey key; if (m_keyManager.GetFileKey(_path, out key)) { return GetFile(key); } return null; } /// /// Remove this file from the library /// public void RemoveFile(DataFileKey _key) { DataFile dataFile; if (m_files.TryGetValue(_key, out dataFile)) { // Clean the dependencies to this file foreach (DataFileDependency dependency in dataFile.Parents) { DataFile parent; if (m_files.TryGetValue(dependency.Key, out parent)) parent.OnChildDeleted(_key); } // Delete dependencies to the children foreach (DataFileDependency dependency in dataFile.Children) { DataFile child; if (m_files.TryGetValue(dependency.Key, out child)) { var platformIds = Enum.GetValues(typeof (PlatformId)).Cast(); var allPlatforms = new HashSet(platformIds.Select(x => new Platform(x))); child.RemoveParent(_key, allPlatforms); if (child.Parents.Count == 0 && child.IsMissing) RemoveFile(child.Key); } } if (dataFile.Parents.Count == 0) { // No more references to this file, it can be property deleted. m_files.Remove(_key); m_keyManager.RemoveKey(_key); } else { // References to this file still exists, transform it to a missing file. dataFile.AddError(DataFileError.Codes.MissingFile); dataFile.Children.Clear(); } OnChanged(); } } /// /// Get the file updater /// public FileUpdater Updater { get { return m_fileUpdater; } } #region Data file dependencies /// /// Set the dependencies of a data file for a given platform /// true if success /// public bool SetDataFileDependencies(DataFile _dataFile, IEnumerable _dependenciesPaths, Platform _platform) { bool result = true; var platforms = new HashSet(); platforms.Add(_platform); // Remove the old dependencies of this file for this platform List dependenciesToRemove = new List(_dataFile.Children); foreach (DataFileDependency dependency in dependenciesToRemove) { RemoveDataFileDependency(_dataFile, dependency, platforms); } // Add the new dependencies foreach (string dependencyPath in _dependenciesPaths) { AddDataFileDependency(_dataFile, dependencyPath, platforms); } // Update the unresolved dependencies flag _dataFile.RemoveError(DataFileError.Codes.UnresolvedDependencies); foreach (DataFileDependency dependency in _dataFile.Children) { if (!dependency.IsValid) { _dataFile.AddError(DataFileError.Codes.UnresolvedDependencies); break; } } return result; } /// /// Add one dependency to a data file on a given platform /// private void AddDataFileDependency(DataFile _dataFile, string _dependencyPath, HashSet _platforms) { DataFileKey dependencyKey; bool missing = false; if (m_keyManager.GetFileKey(_dependencyPath, out dependencyKey)) { // Add the reverse dependency to the child DataFile childFile = GetFile(dependencyKey); if (childFile != null) { childFile.AddParent(_dataFile.NormalizedPath, _dataFile.Key, _platforms); if (childFile.IsMissing) missing = true; } } else { // Create a "phantom" file with the missing file state. DataFile missingFile; CreateNewDataFile(_dependencyPath, out missingFile); missingFile.AddError(DataFileError.Codes.MissingFile); missing = true; dependencyKey = missingFile.Key; DataFile childFile = GetFile(dependencyKey); if (childFile != null) childFile.AddParent(_dataFile.NormalizedPath, _dataFile.Key, _platforms); } _dataFile.AddChild(_dependencyPath, dependencyKey, _platforms, missing); } /// /// Remove one dependency from a data file on a given platform /// private void RemoveDataFileDependency(DataFile _dataFile, DataFileDependency _dependency, HashSet _platforms) { // Remove the reverse dependency from the child DataFile childFile = GetFile(_dependency.Key); if (childFile != null) { childFile.RemoveParent(_dataFile.Key, _platforms); if (childFile.Parents.Count == 0 && childFile.Errors.Any(error => error.Code == DataFileError.Codes.MissingFile)) { // This missing file is referenced by no one anymore, remove it. RemoveFile(childFile.Key); } } _dataFile.RemoveChild(_dependency, _platforms); } /// /// Some dependencies may be invalid (pointing to a non-existing file). Try to fix them. /// private void ResolveDependencies() { m_resolveRefStopwatch.Start(); DataFileKey newKey; foreach (KeyValuePair kvp in m_files) { DataFile dataFile = kvp.Value; bool hasUnresolvedDependencies = false; foreach (DataFileDependency dependency in dataFile.Children) { if (!dependency.IsValid) { if (m_keyManager.GetFileKey(dependency.Path, out newKey)) { dependency.Key = newKey; DataFile child = GetFile(newKey); if (child != null) child.AddParent(dataFile.NormalizedPath, dataFile.Key, dependency.Platforms); } else { hasUnresolvedDependencies = true; } } } if (hasUnresolvedDependencies) dataFile.AddError(DataFileError.Codes.UnresolvedDependencies); } m_resolveRefStopwatch.Stop(); } /// /// Launch an "Update Dependencies" operation when needed /// private void DependenciesCheck() { foreach (KeyValuePair pair in m_files) { DataFile dataFile = pair.Value; if (!dataFile.DependenciesUpToDate) { m_fileUpdater.AskForFileDependenciesUpdate(dataFile.FilePath); } } } #endregion } }