using System; using System.Collections.Generic; using System.Threading; using UAF; using UAF.Interfaces.Logging; using System.Diagnostics; using System.Xml.Linq; namespace DataFileAnalyzer { /// /// Manages the update of the data files. /// public class FileUpdater : IDisposable { public enum OperationPriority { High = 0, Medium = 1, Low = 2, Count } // Maximum time spent in one call to the UpdateLibrary function private static readonly long maxTimePerUpdate = 16; // Number of file treated by the UpdateLibrary function before // it can be interrupted private static readonly int updateLibraryBatchSize = 100; private readonly FileLibrary m_library; private DataFolderWatcher m_watcher; private ILog m_logger = new ConsoleLog(); // Operations to launch (by priority) private List> m_pendingOperations; // Operations that have been executed and are waiting for the library to ask for an update (by priority) private List> m_waitingForUpdateOperations; // Operations that have been treated by the update thread private Int32 m_finishedOperationCount = 0; // The update thread private Thread m_thread = null; // Mutex to lock the members of the class (bool, int, etc...) private Mutex m_internalMutex = new Mutex(false, "DataFileAnalyzer.FileUpdater.m_internalMutex"); // Event set when the update thread has been started private readonly AutoResetEvent m_startedSyncEvent = new AutoResetEvent(false); // Event set when the update thread has been stopped private readonly AutoResetEvent m_stopSyncEvent = new AutoResetEvent(false); // Event to notify the update thread that there is new operations to treat private readonly AutoResetEvent m_pendingOperationsEvent = new AutoResetEvent(false); // Event to set when we exit the pause private readonly AutoResetEvent m_unpauseEvent = new AutoResetEvent(false); // Event to set when the update thread is paused private readonly AutoResetEvent m_pauseSyncEvent = new AutoResetEvent(false); // Stop as soon as possible private bool m_exitAsked = false; // Pause as soon as possible private bool m_pauseAsked = false; // The update thread is running private bool m_running = false; // The update thread is paused private bool m_paused = false; // Operations currently treated by the update thread (by priority) private List> m_runningOperations; private engineWrapper m_engineWrapper = null; private cookerWrapper m_cookerWrapper = null; private Stopwatch m_runStopwatch = new Stopwatch(); private Stopwatch m_updateStopwatch = new Stopwatch(); /// /// Constructor /// public FileUpdater(FileLibrary _library) { m_pendingOperations = new List>(); m_runningOperations = new List>(); m_waitingForUpdateOperations = new List>(); for (int priority = 0; priority < (int)OperationPriority.Count; ++priority) { m_pendingOperations.Add(new Queue()); m_runningOperations.Add(new Queue()); m_waitingForUpdateOperations.Add(new Queue()); } m_library = _library; m_logger = _library.Logger; m_watcher = new DataFolderWatcher(this); } public void Dispose() { if(m_watcher != null) m_watcher.Dispose(); } /// /// The logger. Cannot be set to a null value. /// public ILog Logger { get { return m_logger; } set { if (value != null) { m_logger = value; m_watcher.Logger = value; } } } /// /// Accessor to the FileLibrary. /// public FileLibrary Library { get { return m_library; } } /// /// The cooker wrapper /// public cookerWrapper CookerWrapper { get { return m_cookerWrapper; } } /// /// The number of update operations that have been done since Start was called /// public Int32 FinishedOperationsCount { get { m_internalMutex.WaitOne(); Int32 result = m_finishedOperationCount; m_internalMutex.ReleaseMutex(); return result; } } /// /// The data folder of the library has changed /// public void OnDataFolderChanged(string _folder, List _excludedFolders) { m_watcher.OnDataFolderChanged(_folder, _excludedFolders); } #region Start/Stop/Pause/Resume management /// /// Start updating. /// public void Start() { if (!Running) { m_thread = new Thread(MainThread); m_thread.Name = "DataFileAnalyzer.FileUpdater.MainThread"; m_thread.Start(); m_startedSyncEvent.WaitOne(); m_running = true; m_paused = false; m_watcher.Start(m_library); } } /// /// Stop updating. /// public void Stop() { if (Running) { Resume(); m_watcher.Stop(); ExitAsked = true; m_pendingOperationsEvent.Set(); m_stopSyncEvent.WaitOne(); m_exitAsked = false; m_thread = null; for (int priority = 0; priority < (int)OperationPriority.Count; ++priority) { m_pendingOperations[priority].Clear(); m_runningOperations[priority].Clear(); m_waitingForUpdateOperations[priority].Clear(); } m_finishedOperationCount = 0; m_runStopwatch.Reset(); m_updateStopwatch.Reset(); m_running = false; } } /// /// Pause all the threads /// public void Pause() { if (Running && !Paused) { PausedAsked = true; m_pendingOperationsEvent.Set(); m_pauseSyncEvent.WaitOne(); Paused = true; } } /// /// Resume the threads /// public void Resume() { if (Running && Paused) { PausedAsked = false; m_unpauseEvent.Set(); Paused = false; } } #region Multithread-safe properties /// /// Ask for the other thread(s) to exit. /// protected bool ExitAsked { get { m_internalMutex.WaitOne(); bool result = m_exitAsked; m_internalMutex.ReleaseMutex(); return result; } set { m_internalMutex.WaitOne(); m_exitAsked = value; m_internalMutex.ReleaseMutex(); } } /// /// Ask for a pause and wait for the threads to pause. /// protected bool PausedAsked { get { m_internalMutex.WaitOne(); bool result = m_pauseAsked; m_internalMutex.ReleaseMutex(); return result; } set { m_internalMutex.WaitOne(); m_pauseAsked = value; m_internalMutex.ReleaseMutex(); } } /// /// Running? /// protected bool Running { get { m_internalMutex.WaitOne(); bool result = m_running; m_internalMutex.ReleaseMutex(); return result; } set { m_internalMutex.WaitOne(); m_running = value; m_internalMutex.ReleaseMutex(); } } /// /// Paused? /// protected bool Paused { get { m_internalMutex.WaitOne(); bool result = m_paused; m_internalMutex.ReleaseMutex(); return result; } set { m_internalMutex.WaitOne(); m_paused = value; m_internalMutex.ReleaseMutex(); } } #endregion #endregion /// /// Clear the file library and cancel all the pending and running update operations. /// public void ClearLibrary() { m_logger.Message("[Updater] Canceling all operation and asking to clear the library."); // This function is called from the watcher thread. // We should modify the library the library and updater threads are running. m_library.Pause(); // Clear the file library m_library.Clear(); // Cancel all operations lock (m_pendingOperations) { for (int priority = 0; priority < (int)OperationPriority.Count; ++priority) m_pendingOperations[priority].Clear(); } lock (m_runningOperations) { for (int priority = 0; priority < (int)OperationPriority.Count; ++priority) m_runningOperations[priority].Clear(); } lock (m_waitingForUpdateOperations) { for (int priority = 0; priority < (int)OperationPriority.Count; ++priority) m_waitingForUpdateOperations[priority].Clear(); } m_library.Resume(); } /// /// Main thread of the updater. /// protected void MainThread() { m_engineWrapper = new engineWrapper("DataFileAnalyzer.UpdateThread"); m_cookerWrapper = new cookerWrapper(); m_cookerWrapper.init(PathUtils.NormalizeDataRoot(m_library.DataFolder)); m_cookerWrapper.disableCache(); // Unlock parent thread m_startedSyncEvent.Set(); // Main loop while(!ExitAsked) { m_pendingOperationsEvent.WaitOne(); if (PausedAsked) { m_pauseSyncEvent.Set(); m_unpauseEvent.WaitOne(); } if (ExitAsked) break; lock (m_runStopwatch) { m_runStopwatch.Start(); } for (int priority = 0; priority < (int)OperationPriority.Count && !ExitAsked && !PausedAsked; ++priority) { Queue pendingOperations; lock (m_pendingOperations) { pendingOperations = new Queue(m_pendingOperations[priority]); m_pendingOperations[priority].Clear(); } lock (m_runningOperations) { while (pendingOperations.Count != 0) m_runningOperations[priority].Enqueue(pendingOperations.Dequeue()); } while (!ExitAsked && !PausedAsked) { UpdateOperation operation; lock (m_runningOperations) { if (m_runningOperations[priority].Count != 0) operation = m_runningOperations[priority].Dequeue(); else break; } operation.Run(); lock (m_waitingForUpdateOperations) { m_waitingForUpdateOperations[priority].Enqueue(operation); } } } lock (m_runStopwatch) { m_runStopwatch.Stop(); } } m_cookerWrapper.destroy(); m_cookerWrapper = null; m_engineWrapper.destroy(); m_engineWrapper = null; // Unlock parent thread m_stopSyncEvent.Set(); } /// /// Notify the updater that a data file should be fully updated. /// The added operation or the existing operation that /// will do the job. /// public UpdateOperation AskForFileUpdate(string _filePath) { return AskForFileUpdate(_filePath, false); } /// /// Notify the updater that a data file should be fully updated. /// The added operation or the existing operation that /// will do the job. /// public UpdateOperation AskForFileUpdate(string _filePath, bool isRelative) { string dataFilePath; if (isRelative) dataFilePath = _filePath; else dataFilePath = PathUtils.MakeRelative(_filePath, Library.DataFolder); UpdateOperation newOperation = new BasicFileUpdate(this, dataFilePath); return AddPendingOperation(newOperation, OperationPriority.High); } /// /// Notify the updater that the dependencies of a data file should be recomputed. /// The added operation or the existing operation that /// will do the job. /// public UpdateOperation AskForFileDependenciesUpdate(string _dataFilePath) { UpdateOperation newOperation = new DependenciesUpdateOperation(this, _dataFilePath); return AddPendingOperation(newOperation, OperationPriority.Medium); } /// /// Notify the updater that the dependencies of a data file should be recomputed. /// The added operation or the existing operation that /// will do the job. /// public UpdateOperation AskForOrphanDetectionUpdate(string _dataFilePath) { UpdateOperation newOperation = new OrphanDetectionOperation(this, _dataFilePath); return AddPendingOperation(newOperation, OperationPriority.Low); } /// /// Notify the updater that Perforce information of a data file should be recomputed. /// The added operation or the existing operation that /// will do the job. /// public UpdateOperation AskForPerforceUpdate(string _dataFilePath, Ubisoft.Perforce.Data.DepotFileInfo _p4info) { UpdateOperation newOperation = new PerforceUpdateOperation(this, _dataFilePath, _p4info); return AddPendingOperation(newOperation, OperationPriority.Low); } /// /// Add an update operation to the pending list. Do not add the operation /// if there is already another pending operation updating the same /// thing. /// The added operation or of the existing operation that /// will do the job. /// protected UpdateOperation AddPendingOperation(UpdateOperation _newOperation, OperationPriority _priority) { lock (m_pendingOperations) { Queue queue = m_pendingOperations[(int)_priority]; foreach (UpdateOperation operation in queue) { if (operation.CanSkip(_newOperation)) return operation; } queue.Enqueue(_newOperation); m_pendingOperationsEvent.Set(); return _newOperation; } } /// /// Get some info about the current state (number of operations pending, running, etc...) /// Called from the library thread. /// public void GetCurrentState(FileUpdaterState _state) { _state.Initializing = m_running && m_watcher.Initializing; lock (m_pendingOperations) { for (int priority = 0; priority < (int)OperationPriority.Count; ++priority) _state.PendingCount = m_pendingOperations[priority].Count; } lock (m_runningOperations) { for (int priority = 0; priority < (int)OperationPriority.Count; ++priority) _state.RunningCount = m_runningOperations[priority].Count; } lock (m_waitingForUpdateOperations) { for (int priority = 0; priority < (int)OperationPriority.Count; ++priority) _state.WaitingUpdateCount = m_waitingForUpdateOperations[priority].Count; } lock (m_runStopwatch) { _state.RunningDuration = m_runStopwatch.Elapsed; } _state.FinishedCount = FinishedOperationsCount; _state.UpdateDuration = m_updateStopwatch.Elapsed; } /// /// Update the library from the finished operations. Called from the library thread. /// public void UpdateLibrary() { // Now treat operations that are waiting for update until the time is up Queue operationList = new Queue(); Int64 initialElapsed = m_updateStopwatch.ElapsedMilliseconds; m_updateStopwatch.Start(); for (int priority = 0; priority < (int)OperationPriority.Count && m_updateStopwatch.ElapsedMilliseconds - initialElapsed < maxTimePerUpdate; ++priority) { while (m_updateStopwatch.ElapsedMilliseconds - initialElapsed < maxTimePerUpdate) { lock (m_waitingForUpdateOperations) { Int32 fileCount = m_waitingForUpdateOperations[priority].Count; if (fileCount != 0) { int batchSize = (fileCount < updateLibraryBatchSize) ? fileCount : updateLibraryBatchSize; for (int i = 0; i < batchSize; ++i) operationList.Enqueue(m_waitingForUpdateOperations[priority].Dequeue()); } else { break; } } foreach (UpdateOperation operation in operationList) { operation.UpdateLibrary(); } m_internalMutex.WaitOne(); m_finishedOperationCount += operationList.Count; m_internalMutex.ReleaseMutex(); operationList.Clear(); } } m_updateStopwatch.Stop(); } /// /// Load the state of the updater that was saved in a library XML /// /// true if no error occurred public bool Load(XElement _libraryElement) { if (m_running) return false; return m_watcher.Load(_libraryElement); } /// /// Save the state of the updater in a library XML /// /// true if no error occurred public bool Save(XElement _libraryElement) { return m_watcher.Save(_libraryElement); } } }