#pragma once #include "dtcore/inc/dtdefinitions.h" #include "dtcore/inc/dtbytebuffer.h" #include "dtcore/inc/dtmemory.h" #include "dtcore/inc/dterror.h" #include "dtcore/inc/dtxmlparser.h" #include "dtcore/inc/dtusertoagentmapping.h" #include "dtcore/inc/dttestdescriptor.h" #pragma warning(push) #include #pragma warning(pop) namespace dt { // Macro used by test to create a failure // Note that the optional additional text *must* start with double quotes, if you need to use a function output do something like: // DT_FAIL(my_desc, "" << my_func()); #define DT_FAIL(header, ...) \ do { \ dt::String _header; \ _header << header; \ if (*(""#__VA_ARGS__) != '\0') \ { \ _header << " (" << "" __VA_ARGS__ << ")"; \ } \ dt::Test::AssertionFailed(dt::AssertionDetails(_header, \ dt::String::Blank(), \ dt::String::Blank(), \ __FILE__, \ __LINE__)); \ } while (0) #define DT_FAIL_WITH_CATEGORY(header, category, ...) \ do { \ dt::String _header; \ _header << header; \ if (*(""#__VA_ARGS__) != '\0') \ { \ _header << " (" << "" __VA_ARGS__ << ")"; \ } \ dt::String _category; \ _category << category; \ dt::Test::AssertionFailed(dt::AssertionDetails(_header, \ dt::String::Blank(), \ _category, \ __FILE__, \ __LINE__)); \ } while (0) #define DT_FAIL_WITH_DETAILS(header, extended, ...) \ do { \ dt::String _header; \ _header << header; \ if (*(""#__VA_ARGS__) != '\0') \ { \ _header << " (" << "" __VA_ARGS__ << ")"; \ } \ dt::String _extended; \ _extended << extended; \ dt::Test::AssertionFailed(dt::AssertionDetails(_header, \ _extended, \ dt::String::Blank(), \ __FILE__, \ __LINE__)); \ } while (0) #define DT_FAIL_WITH_DETAILS_CATEGORY(header, extended, category, ...) \ do { \ dt::String _header; \ _header << header; \ if (*(""#__VA_ARGS__) != '\0') \ { \ _header << " (" << "" __VA_ARGS__ << ")"; \ } \ dt::String _extended; \ _extended << extended; \ dt::String _category; \ _category << category; \ dt::Test::AssertionFailed(dt::AssertionDetails(_header, \ _extended, \ _category, \ __FILE__, \ __LINE__)); \ } while (0) #define DT_FAIL_WITH_DETAILS_FILE_LINE(header, extended, file, line, ...) \ do { \ dt::String _header; \ _header << header; \ if (*(""#__VA_ARGS__) != '\0') \ { \ _header << " (" << "" __VA_ARGS__ << ")"; \ } \ dt::String _extended; \ _extended << extended; \ dt::Test::AssertionFailed(dt::AssertionDetails(_header, \ _extended, \ dt::String::Blank(), \ file, \ line)); \ } while (0) #define DT_FAIL_WITH_DETAILS_CATEGORY_FILE_LINE(header, extended, category, file, line, ...) \ do { \ dt::String _header; \ _header << header; \ if (*(""#__VA_ARGS__) != '\0') \ { \ _header << " (" << "" __VA_ARGS__ << ")"; \ } \ dt::String _extended; \ _extended << extended; \ dt::String _category; \ _category << category; \ dt::Test::AssertionFailed(dt::AssertionDetails(_header, \ _extended, \ _category, \ file, \ line)); \ } while (0) // Macro used by test to create a warning #define DT_WARNING(header, ...) \ do { \ dt::String _header; \ _header << header; \ if (*(""#__VA_ARGS__) != '\0') \ { \ _header << " (" << "" __VA_ARGS__ << ")"; \ } \ dt::Test::AssertionWarning(dt::AssertionDetails(_header, \ dt::String::Blank(), \ dt::String::Blank(), \ __FILE__, \ __LINE__)); \ } while (0) #define DT_WARNING_WITH_DETAILS(header, extended, ...) \ do { \ dt::String _header; \ _header << header; \ if (*(""#__VA_ARGS__) != '\0') \ { \ _header << " (" << "" __VA_ARGS__ << ")"; \ } \ dt::String _extended; \ _extended << extended; \ dt::Test::AssertionWarning(dt::AssertionDetails(_header, \ _extended, \ dt::String::Blank(), \ __FILE__, \ __LINE__)); \ } while (0) // Macro used by test to verify equality #define DT_EXPECT_EQUAL(expression, expectation, ...) \ do { \ if ((expression) == (expectation)) \ { \ dt::Test::AssertionSucceeded(); \ } \ else \ { \ DT_FAIL_WITH_DETAILS("Given value should match expectation", \ "Given value (" << (expression) << ") should match expectation (" << \ (expectation) << ") in expression: " << #expression << " == " << \ #expectation, __VA_ARGS__); \ } \ } while (0) // Macro used by test to verify > #define DT_EXPECT_GREATER(expression, expectation, ...) \ do { \ if ((expression) > (expectation)) \ { \ dt::Test::AssertionSucceeded(); \ } \ else \ { \ DT_FAIL_WITH_DETAILS("Given value should be greater than expectation", \ "Given value (" << (expression) << ") should be greater than expectation (" << \ (expectation) << ") in expression: " << #expression << " > " << \ #expectation, __VA_ARGS__); \ } \ } while (0) // Macro used by test to verify >= #define DT_EXPECT_GREATER_OR_EQUAL(expression, expectation, ...) \ do { \ if ((expression) >= (expectation)) \ { \ dt::Test::AssertionSucceeded(); \ } \ else \ { \ DT_FAIL_WITH_DETAILS("Given value should be greater or equal to expectation", \ "Given value (" << (expression) << ") should be greater or equal to expectation (" << \ (expectation) << ") in expression: " << #expression << " >= " << \ #expectation, __VA_ARGS__); \ } \ } while (0) // Macro used by test to verify < #define DT_EXPECT_SMALLER(expression, expectation, ...) \ do { \ if ((expression) < (expectation)) \ { \ dt::Test::AssertionSucceeded(); \ } \ else \ { \ DT_FAIL_WITH_DETAILS("Given value should be smaller than expectation", \ "Given value (" << (expression) << ") should be smaller than expectation (" << \ (expectation) << ") in expression: " << #expression << " < " << \ #expectation, __VA_ARGS__); \ } \ } while (0) // Macro used by test to verify >= #define DT_EXPECT_SMALLER_OR_EQUAL(expression, expectation, ...) \ do { \ if ((expression) <= (expectation)) \ { \ dt::Test::AssertionSucceeded(); \ } \ else \ { \ DT_FAIL_WITH_DETAILS("Given value should be smaller or equal to expectation", \ "Given value (" << (expression) << ") should be smaller or equal to expectation (" << \ (expectation) << ") in expression: " << #expression << " <= " << \ #expectation, __VA_ARGS__); \ } \ } while (0) // Macro used by test to verify equality #define DT_EXPECT_UNEQUAL(expression, expectation, ...) \ do { \ if ((expression) != (expectation)) \ { \ dt::Test::AssertionSucceeded(); \ } \ else \ { \ DT_FAIL_WITH_DETAILS("Given value should not match expectation", \ "Given value (" << (expression) << ") should not match expectation (" << \ (expectation) << ") in expression: " << #expression << " != " << \ #expectation, __VA_ARGS__); \ } \ } while (0) // Macro used by test to verify truthfulness #define DT_EXPECT_TRUE(expression, ...) \ do { \ if ((expression)) \ { \ dt::Test::AssertionSucceeded(); \ } \ else \ { \ DT_FAIL_WITH_DETAILS("Given value should match expectation", \ "Given value should evaluate to TRUE in expression: " << \ #expression, __VA_ARGS__); \ } \ } while (0) // Macro used by test to verify falseness #define DT_EXPECT_FALSE(expression, ...) \ do { \ if (!(expression)) \ { \ dt::Test::AssertionSucceeded(); \ } \ else \ { \ DT_FAIL_WITH_DETAILS("Given value should match expectation", \ "Given value should evaluate to FALSE in expression: " << \ #expression, __VA_ARGS__); \ } \ } while (0) // Macro used by test to report progress #define DT_REPORT_PROGRESS(description) \ do { \ dt::String _msg; \ _msg << description; \ dt::Test::ReportProgress(_msg, dt::ProgressionType::User); \ } while (0) // Use this macro to register your test to the TestFactory #define DT_REGISTER_TEST(className) \ do { \ extern dt::Test* dtCreate##className(); \ dt::TestFactory::Register(#className, &dtCreate##className); \ } while (0) // Use this macro to declare a test. The base class should be or derive from dt::Test // example: DT_DECLARE_TEST(yourtestname) : public dt::Test #define DT_DECLARE_TEST(className, ...) \ class className; \ __VA_ARGS__ dt::Test* dtCreate##className() \ { \ return dt::TestFactory::CreateTemplate(); \ } \ class __VA_ARGS__ className ////////////////////////////////////////////////////////////////////////// enum class ProgressionType { User, System }; ////////////////////////////////////////////////////////////////////////// // The progression listener interface class can be used to listen to dt::Test::ReportProgress calls. class TestProgressionListener { public: DT_CORE_DLL_ENTRY virtual ~TestProgressionListener() {} DT_CORE_DLL_ENTRY virtual void OnProgression(const String& message, ProgressionType progressionType) = 0; }; ////////////////////////////////////////////////////////////////////////// class AssertionDetails { public: DT_CORE_DLL_ENTRY AssertionDetails() = default; DT_CORE_DLL_ENTRY AssertionDetails(const String& header, const String& extended, const String& category, const char* file, int line); DT_CORE_DLL_ENTRY const String& GetHeader() const { return m_Header; } DT_CORE_DLL_ENTRY const String& GetExtended() const { return m_Extended; } DT_CORE_DLL_ENTRY const String& GetCategory() const { return m_Category; } DT_CORE_DLL_ENTRY const char* GetFile() const { return m_File; } DT_CORE_DLL_ENTRY int GetLine() const { return m_Line; } private: String m_Header; String m_Extended; String m_Category; const char* m_File = nullptr; int m_Line = 0; }; ////////////////////////////////////////////////////////////////////////// class Test { public: DT_CORE_DLL_ENTRY virtual ~Test() {} // Functions to be overridden in your test // override this method in your test if needed, this base function must no be called from your test DT_CORE_DLL_ENTRY virtual void Setup(); // override this method in your test if needed, this base function must no be called from your test DT_CORE_DLL_ENTRY virtual void Execute(); // override this method in your test if needed, this base function must no be called from your test DT_CORE_DLL_ENTRY virtual void TearDown(); // override this method to add a custom failure when you test timeout. Useful to report what you where exactly doing in the failure reason. DT_CORE_DLL_ENTRY virtual void OnTimeout(); // Utility functions // this function will be called by the test assertion macros: DT_EXPECT_EQUAL, DT_EXPECT_TRUE, DT_EXPECT_FALSE, etc // when a success is found DT_CORE_DLL_ENTRY static void AssertionSucceeded(); // this function will be called by the test assertion macros: DT_EXPECT_EQUAL, DT_EXPECT_TRUE, DT_EXPECT_FALSE, etc // when a failure is found DT_CORE_DLL_ENTRY static void AssertionFailed(const AssertionDetails& details); DT_CORE_DLL_ENTRY static void AssertionWarning(const AssertionDetails& details); // this function is to be called when a crash is detected by external tools, this will trigger a message to the farm to inform user if in interactive mode. DT_CORE_DLL_ENTRY static void InteractiveTrigger(bool trap); // call this function if your test is in asynchronous mode (see dt::Test::SetExecutionMode()) DT_CORE_DLL_ENTRY static void SetCompleted(); // returns true if Test::SetCompleted() was called DT_CORE_DLL_ENTRY static bool IsCompleted(); // function used to override or create a DTEST user variable DT_CORE_DLL_ENTRY static void SetUserVariable(const String& name, const String& value); // function used to get a DTEST user variable DT_CORE_DLL_ENTRY static const Variable& GetUserVariable(const String& name); // function used to get a DTEST user variable DT_CORE_DLL_ENTRY static bool TryGetUserVariable(const String& name, const Variable** out); // Test Descriptor // get the TestDescriptor of current running test // The descriptor will only be valid after TestDescriptorCallback, set when creating ServiceTest_Agent, is called. DT_CORE_DLL_ENTRY static const TestDescriptor& GetDescriptor(); // get the current agent index running. The indexes starts at 0. If you have 3 agents in your test the indexes will be 0, 1 and 2 DT_CORE_DLL_ENTRY static int GetAgentIndex(); DT_CORE_DLL_ENTRY static AgentIndex GetStrongTypeAgentIndex(); // get the total number of agents necessary to run this test; DT_CORE_DLL_ENTRY static int GetTotalAgentCount(); // get a token of the specified type and return a reservation id; Set only in you test constructor. Your test will fail if you get an unknown token DT_CORE_DLL_ENTRY static const String& GetToken(const String& tokenName); // Additional test information // get the batch ID DT_CORE_DLL_ENTRY static int GetBatchId(); DT_CORE_DLL_ENTRY static BatchId GetStrongTypeBatchId(); // get the test ID (unique in the current database) DT_CORE_DLL_ENTRY static int GetTestId(); DT_CORE_DLL_ENTRY static TestId GetStrongTypeTestId(); // get the test GUID (globally unique) DT_CORE_DLL_ENTRY static const String& GetTestGUID(); // get the current try of the test DT_CORE_DLL_ENTRY static int GetTryId(); DT_CORE_DLL_ENTRY static TryId GetStrongTypeTryId(); // get the resource ID DT_CORE_DLL_ENTRY static int GetResourceId(); DT_CORE_DLL_ENTRY static ResourceReservationId GetStrongTypeResourceId(); // Returns the ID of the associated TestManagementTool test, if one was associated with this test, INVALID_ID otherwise. DT_CORE_DLL_ENTRY static int GetTestManagementToolTestId(); DT_CORE_DLL_ENTRY static TestManagementToolTestId GetTestManagementToolStrongTypeTestId(); // get the shell command ID. The shell command ID is a unique ID of the command that started the executable of this agent. DT_CORE_DLL_ENTRY static int GetShellCommandId(); DT_CORE_DLL_ENTRY static CommandId GetStrongTypeCommandId(); // get if timeouts should be enabled (this is to ease debugging) DT_CORE_DLL_ENTRY static bool IsTimeoutEnabled(); // get if the script is current paused on a breakpoint DT_CORE_DLL_ENTRY static bool IsPausedOnBreakpoint(); // get if the agent is waiting for user interaction on test end DT_CORE_DLL_ENTRY static bool IsWaitInteractionOnEnd(); // get if the given agent has failed DT_CORE_DLL_ENTRY static bool HasAgentFailed(int agentIndex); // get the maintainer's name DT_CORE_DLL_ENTRY static const String& GetMaintainerName(); // get the maintainer's email DT_CORE_DLL_ENTRY static const String& GetMaintainerEmail(); // get the current platform name as specified in the DTEST project DT_CORE_DLL_ENTRY static const String& GetPlatform(); // get the current config name as specified in the DTEST project DT_CORE_DLL_ENTRY static const String& GetConfig(); // get the current profile used to run the test DT_CORE_DLL_ENTRY static const String& GetProfile(); // get the current configuration file name (dtproj) DT_CORE_DLL_ENTRY static const String& GetProjectFileName(); // get the current executing script line DT_CORE_DLL_ENTRY static bool GetExecutingScriptLine(const ScriptLine** curLine); // get the maximum artefact size allowed to be uploaded to the farm DT_CORE_DLL_ENTRY static SizeT GetMaxArtefactSizeInBytes(); // return true if this subtest is the first to run in the agent. // return false if the current test is not a subtest. // This can be used to run setup code that means to be executed before the test can run DT_CORE_DLL_ENTRY static bool IsFirstSubTest(); // Events // Wait for a specific event // eventId: The event type to wait for // payload: Optionally pass a pointer to an instance of a ByteBuffer to retrieve a payload // returns true if within maxWaitTimeMs an event of type eventId was received DT_CORE_DLL_ENTRY static bool WaitForEvent(const String& eventId, ByteBuffer* payload, int maxWaitTimeMs = 0); // Wait for any event // eventId: Optionally pass a pointer to an instance of a String to retrieve the type of event received // payload: Optionally pass a pointer to an instance of a ByteBuffer to retrieve a payload // returns true if within maxWaitTimeMs an event was received DT_CORE_DLL_ENTRY static bool WaitForAnyEvent(String* eventId, ByteBuffer* payload, int maxWaitTimeMs = 0); // Sends an event to all agents within the test // eventId: The event type to send // payload: Optional custom data to send with the event DT_CORE_DLL_ENTRY static void BroadcastEvent(const String& eventId, const ByteBuffer& payload = ByteBuffer()); // Send an event to the specified agent within the test // AgentIndex: The destination agent index to send to event to // eventId: The event type to send // payload: Optional custom data to send with the event DT_CORE_DLL_ENTRY static void UnicastEvent(int AgentIndex, const String& eventId, const ByteBuffer& payload = ByteBuffer()); // Threads // Invoke a call on the foreground thread of the application. This is useful if the tests runs in a // background thread (ServiceTest_Agent created with runTestOnForegroundUpdate=false). The call will // be performed when ServiceTest_Agent::ForegroundUpdate is called. Invoke will return immediatly and // the call will be performed in background. InvokeAndWait will return only afterward the call has been // down in the foreground thread. DT_CORE_DLL_ENTRY static void Invoke(const std::function& function); DT_CORE_DLL_ENTRY static void InvokeAndWait(const std::function& function); // Progression // A test can add progression string to show the big steps of a test as they happen during execution. // The progression information will be shown in the DTEST graphical application, in the dtTerminal // standard output and on the web site test details. The goal of this is to provide a brief overview // of what is happening in your test. It should no be too verbose and is not there to replace logs. // It shall be user friendly message that every one can understand like: // Player spawned // Waiting for friend to join // Starting playing with friend // Etc. // These progression message will also show up in logs, so no need to duplicate them. DT_CORE_DLL_ENTRY static void ReportProgress(const String& progression, ProgressionType progressionType); // A progression listener can be registered to intercept any calls to dt::Test::ReportProgress. // You are responsible to remove the listener before its deletion. DT_CORE_DLL_ENTRY static void AddProgressListener(TestProgressionListener* listener); // Remove a previously registered listener. If the listener is not found, the call does nothing. DT_CORE_DLL_ENTRY static void RemoveProgressListener(TestProgressionListener* listener); // Shell Commands // Starts a global shell command already declared in DTest's config files. // globalCommandName: The global command name to be used // params: The parameters of the shell command // returns the local command ID of the command that was started DT_CORE_DLL_ENTRY static int StartGlobalShellCommand(const String& globalCommandName, const ShellCommandParams& params); // Starts a custom shell command. // command: A string which represents the actual instructions to be executed. // params: The parameters of the shell command // returns the local command ID of the command that was started DT_CORE_DLL_ENTRY static int StartCustomShellCommand(const String& command, const ShellCommandParams& params); // Waits for a previously started shell command to be finished. // commandId: The ID of the command to wait for // exitCode: Returns the exit code of the command that executed // output: The string that will receive the output of the shell command. Pass null if you don't care about output // maxWaitTimeSec: Maximum wait time for the command to complete, in seconds. DT_CORE_DLL_ENTRY static bool WaitForRuntimeShellCommand(int commandId, int* exitCode, String* output = nullptr, int maxWaitTimeSec = 0); // Returns the output of the shell command where the ShellCommandParams::SetOutputRequested was defined true // commandId: The ID of the command // output: The string that will receive the output of the shell command // returns True if the shell command output was copied in the string. False if the command has not yet completed or was not found. DT_CORE_DLL_ENTRY static bool GetShellCommandOutput(int commandId, String* output); // Clears the output of the shell command // commandId: The ID of the command DT_CORE_DLL_ENTRY static void ClearShellCommandOutput(int commandId); // Deprecated API // DEPRECATED - This will be removed in the next versions. Please use GetDescriptor() instead. DT_DEPRECATED("This will be removed in the next versions. Please use GetDescriptor() instead.") DT_CORE_DLL_ENTRY static const String& GetName(); // DEPRECATED - This will be removed in the next versions. Please use GetDescriptor() instead. DT_DEPRECATED("This will be removed in the next versions. Please use GetDescriptor() instead.") DT_CORE_DLL_ENTRY static const String& GetClass(); // DEPRECATED - This will be removed in the next versions. Please use GetDescriptor() instead. DT_DEPRECATED("This will be removed in the next versions. Please use GetDescriptor() instead.") DT_CORE_DLL_ENTRY static TestExecutionMode GetExecutionMode(); // DEPRECATED - This will be removed in the next versions. Please use GetDescriptor() instead. DT_DEPRECATED("This will be removed in the next versions. Please use GetDescriptor() instead.") DT_CORE_DLL_ENTRY static const XmlNode* GetCustomData(); // DEPRECATED - This will be removed in the next versions. Please use alternate form of StartGlobalShellCommand() instead. DT_DEPRECATED("This will be removed in the next versions. Please use alternate form of StartGlobalShellCommand() instead.") DT_CORE_DLL_ENTRY static int StartGlobalShellCommand(const String& globalCommandName, const ShellCommandArguments& commandArguments, const String& commandName); // DEPRECATED - This will be removed in the next versions. Please use alternate form of StartCustomShellCommand() instead. DT_DEPRECATED("This will be removed in the next versions. Please use alternate form of StartCustomShellCommand() instead.") DT_CORE_DLL_ENTRY static int StartCustomShellCommand(const String& command, const ShellCommandArguments& commandArguments, const String& commandName, int commandTimeout = INVALID_VALUE); DT_DEPRECATED("This will be removed in the next versions. Please use the GetTestManagementToolTestId() version instead.") DT_CORE_DLL_ENTRY static int GetTestRailTestId(); }; ////////////////////////////////////////////////////////////////////////// // The test factory call is used by the DT_DECLARE_TEST macro to register your test and let the system create // it. It shall not be necessary to use it from the test perspective class TestFactory { public: typedef Test* (*CreateFunction)(); // Register a create function for the specified class name DT_CORE_DLL_ENTRY static void Register(const char* name, CreateFunction createFunc); // Try a create a test class of the specific type name // returns a new instance of the specified class // returns true if the class was known and test class created DT_CORE_DLL_ENTRY static bool Create(const String& typeName, Test** out); // Helper function to create a test template static Test* CreateTemplate(); // Returns the complete list of known tests DT_CORE_DLL_ENTRY static void GetRegisteredTests(Strings* out); }; template Test* TestFactory::CreateTemplate() { return DT_NEW(T); } }