325 lines
No EOL
15 KiB
Python
325 lines
No EOL
15 KiB
Python
import os
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
import os.path
|
|
import time
|
|
import argparse
|
|
import subprocess
|
|
import email
|
|
import smtplib
|
|
import shutil
|
|
|
|
argParser = argparse.ArgumentParser(description="Get the arguments : the name of the files to analyse and compare")
|
|
argParser.add_argument( '--perftestFile',
|
|
type=str,
|
|
default='perftests_results.xml',
|
|
help="Xml file containing the performance tests results.")
|
|
argParser.add_argument( '--perftestHavok',
|
|
action='store_true',
|
|
default=False,
|
|
help="Enable Havok performance tests.")
|
|
argParser.add_argument( '--buildmachine',
|
|
action='store_true',
|
|
default=False,
|
|
help="Generate xml file needed from ${LOGS_DIR}/everything.log")
|
|
argParser.add_argument( '--minimumPerformanceVariation',
|
|
type=float,
|
|
default=-0.1,
|
|
help="Set the minimum performance variation. [-1:0]")
|
|
argParser.add_argument('--tracker',
|
|
type=str,
|
|
default='Gear',
|
|
help="Set the tracker database to use. Usually, we choose between 'Gear' and 'TG_Test_Tracker'")
|
|
argParser.add_argument('--logsOutput',
|
|
type=str,
|
|
help="Set the location of log files. Mandatory when running from buildmachine: we need everything.log to generate perftests results.")
|
|
args = argParser.parse_args()
|
|
|
|
# files management
|
|
TRACKER_FILE = "tracker.txt"
|
|
PERFTEST_FILE = args.perftestFile
|
|
PERFTEST_HAVOK = args.perftestHavok
|
|
PREVIOUS_PERFTEST_FILE = "previous_perftests_result.xml"
|
|
IS_ON_BUILDMACHINE = args.buildmachine
|
|
TRACKER = args.tracker
|
|
LOGS_DIR = args.logsOutput
|
|
TRACKING_FILE1 = "gear_core_tracker.txt"
|
|
TRACKING_FILE2 = "STL_tracker.txt"
|
|
OUTPUT_TRACKING_FILE1_WITH_RATIO = "gear_core_tracker2.txt"
|
|
OUTPUT_TRACKING_FILE2_WITH_RATIO = "stl_tracker2.txt"
|
|
|
|
# alerts management
|
|
MINIMUM_PERFORMANCE_DIFF = max(min(args.minimumPerformanceVariation, 0.0), -1.0)
|
|
|
|
def SendEmail(previousRootNode,appName, appVersion, moduleName, className, perftestName, perftestResult,previousPerftestResult):
|
|
# Import the email modules we'll need
|
|
from email.mime.text import MIMEText
|
|
# Create a text/plain message
|
|
message = '\n'+'Application='+str(appName)+'\n'+ 'Platform='+str(appVersion)+'\n'+'moduleName='+ str(moduleName)+'\n'+'className='+ str(className)+'\n'+'Function='+ str(perftestName)+'\n'+'Current perftestResult='+str(perftestResult)+'\n'+'previous perftestResult='+str(previousPerftestResult)+'\n'+'http://tableau01/views/Gear4/PerformanceGear4?'+'platform='+appVersion+'&application='+appName+'&class='+className+'&functionName='+perftestName
|
|
msg = MIMEText(message)
|
|
msg['Subject'] = 'Performance tests results has dropped'
|
|
msg['From'] = "PerformanceBot"
|
|
msg['To'] = 'gearsupport@ubisoft.com'
|
|
# Send the message via our own SMTP server, but don't include the
|
|
# envelope header.
|
|
s = smtplib.SMTP('relay.ubisoft.org', 25)
|
|
s.sendmail("gearsupport@ubisoft.com", "gearsupport@ubisoft.com", msg.as_string())
|
|
s.quit()
|
|
|
|
# this function takes a specific result and search for it in the previous perftests results
|
|
# Comparison results are printed in the report and moved with the log files after execution
|
|
# After comparing, the old result is replaced by the new one
|
|
def CompareAndReplacePreviousResults(previousRootNode, appName, appVersion, moduleName, className, perftestName, perftestResult):
|
|
# find the appNode
|
|
for previousAppNode in previousRootNode.iterfind('application'):
|
|
if ( previousAppNode.get('name') == appName and previousAppNode.get('version') == appVersion ):
|
|
# find the module node
|
|
for previousModuleNode in previousAppNode.findall('module'):
|
|
if ( previousModuleNode.get('name') == moduleName ):
|
|
# find the class node
|
|
for previousClassNode in previousModuleNode.findall('class'):
|
|
if ( previousClassNode.get('name') == className ):
|
|
#find the preftest node
|
|
for previousPerftestNode in previousClassNode.findall('perftest'):
|
|
if ( previousPerftestNode.get('name') == perftestName ):
|
|
#perftest found!
|
|
previousPerftestResult = float(previousPerftestNode.get('result'))
|
|
if (previousPerftestResult < perftestResult and previousPerftestResult-perftestResult<=MINIMUM_PERFORMANCE_DIFF):
|
|
SendEmail(previousRootNode,appName, appVersion, moduleName, className, perftestName, perftestResult,previousPerftestResult)
|
|
|
|
def CalculateRatio(file1,file2,outputFile):
|
|
|
|
file1_File_Open = True
|
|
loopCount = 0
|
|
|
|
with open(file1, 'r') as file1_Text_File:
|
|
outputFile_tracker_file = open(outputFile, 'w')
|
|
|
|
#skip the first line for both file since it's useless for what we want to do
|
|
firstLineFile1 = file1_Text_File.readline()
|
|
#Enable this line if you want to make test locally otherwise it will submit the data in the Gear database instead of the test db
|
|
#firstLineFile1 = firstLineFile1.replace("http://TGTracker01:9118/Tracker/Gear","http://localhost:9118/Tracker/TG_Test_Tracker")
|
|
outputFile_tracker_file.write(firstLineFile1)
|
|
|
|
while file1_File_Open == True:
|
|
|
|
line1 = file1_Text_File.readline()
|
|
if line1 =="" or line1 ==" ":
|
|
file1_File_Open = False
|
|
file2_File_Open = False
|
|
else:
|
|
file2_File_Open=True
|
|
|
|
#retrieve moduleName
|
|
line1 = line1.replace('"performance test" ','')
|
|
index = line1.find(" ")
|
|
moduleNameFile1 = line1[:index]
|
|
moduleNameFile1 = moduleNameFile1.replace('"',"")
|
|
|
|
#retrieve className
|
|
index = index+1
|
|
line1 = line1[index:]
|
|
index = line1.find(" ")
|
|
classNameFile1 = line1[:index]
|
|
classNameFile1 = classNameFile1.replace('"',"")
|
|
|
|
#retrieve perftestName
|
|
index = index+1
|
|
line1 = line1[index:]
|
|
index = line1.find(" ")
|
|
perftestNameFile1 = line1[:index]
|
|
perftestNameFile1 = perftestNameFile1.replace('"',"")
|
|
|
|
#retrieve stats
|
|
index = line1.rfind(' ')
|
|
perftestResultLine1 = line1[index:]
|
|
perftestResultLine1 = perftestResultLine1.replace('"','')
|
|
perftestResultLine1 = perftestResultLine1.replace(' ','')
|
|
perftestResultLine1 = perftestResultLine1.replace('\n','')
|
|
|
|
|
|
with open(file2, 'r') as stl_File:
|
|
if loopCount == 0 :
|
|
firstLineFile2= stl_File.readline()
|
|
#Enable this line if you want to make test locally otherwise it will submit the data in the Gear database instead of the test db
|
|
#firstLineFile2= firstLineFile2.replace("http://TGTracker01:9118/Tracker/Gear","http://localhost:9118/Tracker/TG_Test_Tracker")
|
|
#gear_core_stl_file.write(firstLineFile2)
|
|
loopCount=loopCount+1
|
|
|
|
while file2_File_Open == True:
|
|
|
|
line2 = stl_File.readline()
|
|
if line2 =="" or line2==" ":
|
|
file2_File_Open = False
|
|
|
|
#retrieve moduleName
|
|
line2 = line2.replace('"performance test" ','')
|
|
index = line2.find(" ")
|
|
moduleNameFile2 = line2[:index]
|
|
moduleNameFile2 = moduleNameFile2.replace('"',"")
|
|
|
|
#retrieve className
|
|
index = index+1
|
|
line2 = line2[index:]
|
|
index = line2.find(" ")
|
|
classNameFile2 = line2[:index]
|
|
classNameFile2 = classNameFile2.replace('"',"")
|
|
|
|
#retrieve perftestName
|
|
index = index+1
|
|
line2 = line2[index:]
|
|
index = line2.find(" ")
|
|
perftestNameFile2 = line2[:index]
|
|
perftestNameFile2 = perftestNameFile2.replace('"',"")
|
|
|
|
#retrieve stats
|
|
index = line2.rfind(' ')
|
|
perftestResultFile2 = line2[index:]
|
|
perftestResultFile2 = perftestResultFile2.replace('"','')
|
|
perftestResultFile2 = perftestResultFile2.replace(' ','')
|
|
perftestResultFile2 = perftestResultFile2.replace('\n','')
|
|
|
|
ratioCore="0"
|
|
ratioStl="0"
|
|
|
|
if moduleNameFile1==moduleNameFile2:
|
|
if classNameFile1==classNameFile2:
|
|
if perftestNameFile1==perftestNameFile2:
|
|
if (perftestResultLine1 > perftestResultFile2):
|
|
ratioStl = float(perftestResultLine1)/float(perftestResultFile2)*100
|
|
ratioCore = -ratioStl
|
|
else:
|
|
ratioCore = float(perftestResultFile2)/float(perftestResultLine1)*100
|
|
ratioStl = -ratioCore
|
|
|
|
if moduleNameFile1 != "":
|
|
outputFile_tracker_file.write("\"performance test\" \""+moduleNameFile1+"\" \""+classNameFile1+"\" \""+perftestNameFile1 +"\" \""+perftestResultLine1+"\"\n")
|
|
outputFile_tracker_file.write("\"performance test ratio\" \""+moduleNameFile1+"\" \""+classNameFile1+"\" \""+perftestNameFile1 +"\" \""+str(ratioCore)+"\"\n")
|
|
|
|
# Main
|
|
#The fields available in Tracker are:
|
|
# - action ("Performance Testing")
|
|
# - name (appName, eg: Gear, Havok, gear_core)
|
|
# - version (eg: xbox360, ps3)
|
|
# - stat (performance test result: unit execution time [usec], eg: 45.13)
|
|
# - message (moduleName, eg: math, container)
|
|
# - message2 (className, eg: Vector2f, Array<T, N>)
|
|
# - data (function signature, eg: "G4::Vector3d::Cross(Vector3d&, const Vector3d&, const Vector3d&)" )
|
|
def main():
|
|
print("Launching PerformanceMonitor Tool...\n")
|
|
|
|
if not os.path.exists(PREVIOUS_PERFTEST_FILE) and os.path.exists(PERFTEST_FILE):
|
|
shutil.copy2(PERFTEST_FILE, PREVIOUS_PERFTEST_FILE)
|
|
|
|
if IS_ON_BUILDMACHINE:
|
|
perftestFileHasToBeenRegenerated = True
|
|
try:
|
|
with open(PERFTEST_FILE): pass #PERFTEST_FILE already exists
|
|
perftestFileHasToBeenRegenerated = False
|
|
#check if PERFTEST_FILE has been generated during the last hour
|
|
if abs(os.path.getctime(PERFTEST_FILE) - time.time()) > 3600: #PERFTEST_FILE is old : delete and regenerate it
|
|
os.rename(PERFTEST_FILE, PREVIOUS_PERFTEST_FILE)
|
|
os.remove(PERFTEST_FILE)
|
|
perftestFileHasToBeenRegenerated = True
|
|
print(PERFTEST_FILE + " is obsolete : try to regenerate it from 'Test Results.log'.")
|
|
else:
|
|
print(PERFTEST_FILE + " is up to date.")
|
|
except IOError: pass
|
|
|
|
if perftestFileHasToBeenRegenerated:
|
|
print("Generating "+PERFTEST_FILE+" from 'Test Results.log'...\n")
|
|
rootFound = False
|
|
with open(LOGS_DIR+"/Test Results.log", 'r') as everything:
|
|
with open(PERFTEST_FILE, 'w') as perfFile:
|
|
for line in everything:
|
|
if rootFound:
|
|
perfFile.write(line[line.find('<'):])
|
|
if "</PerformanceTestsRoot>" in line:
|
|
break
|
|
if "<PerformanceTestsRoot>" in line:
|
|
rootFound = True
|
|
perfFile.write(line[line.find('<'):])
|
|
if not rootFound:
|
|
print("Error! Can't generate xml file from 'Test Results.log'\n")
|
|
return -1
|
|
|
|
# get the root node
|
|
try:
|
|
print("XML Parsing")
|
|
xmlTree = ET.parse(PERFTEST_FILE)
|
|
rootNode = xmlTree.getroot()
|
|
|
|
except IOError:
|
|
print("XML Parsing - Exception")
|
|
return -1
|
|
|
|
# open preivous results to compare with
|
|
cantFindPreviousResults = False
|
|
try:
|
|
xmlTree = ET.parse(PREVIOUS_PERFTEST_FILE)
|
|
previousRootNode = xmlTree.getroot()
|
|
except IOError:
|
|
cantFindPreviousResults = True
|
|
|
|
if cantFindPreviousResults:
|
|
xmlTree = ET.parse(PERFTEST_FILE)
|
|
previousRootNode = xmlTree.getroot()
|
|
|
|
totalApp = 0;
|
|
print("Analyzing application(s) in root node : "+ str(rootNode)+ " ...\n")
|
|
for appNode in rootNode.findall('application'):
|
|
appName = appNode.attrib['name']
|
|
appVersion = appNode.attrib['version']
|
|
print("Application found : "+appName+". ")
|
|
if appName.lower() == "havok" and not PERFTEST_HAVOK:
|
|
print("/!\\ Option 'perftestHavok' has not been set. Skipping application...\n")
|
|
continue
|
|
totalApp += 1
|
|
print("Compiling perftests results...\n")
|
|
with open(TRACKER_FILE, 'w') as trackerFile:
|
|
trackerFile.write("\""+appName+"\" \""+appVersion+"\" \"http://TGTracker01:9118/Tracker/"+TRACKER+"\"\n")
|
|
for moduleNode in appNode.findall('module'):
|
|
moduleName = moduleNode.attrib['name']
|
|
for classNode in moduleNode.findall('class'):
|
|
className = classNode.attrib['name']
|
|
for perftestNode in classNode.findall('perftest'):
|
|
perftestName = perftestNode.attrib['name']
|
|
perftestResult = perftestNode.attrib['result']
|
|
trackerFile.write("\"performance test\" \""+moduleName+"\" \""+className+"\" \""+perftestName+"\" \""+perftestResult+"\"\n")
|
|
|
|
shutil.copy2(TRACKER_FILE, appName+"_tracker.txt")
|
|
|
|
CalculateRatio(TRACKING_FILE1,TRACKING_FILE2,OUTPUT_TRACKING_FILE1_WITH_RATIO)
|
|
CalculateRatio(TRACKING_FILE2,TRACKING_FILE1,OUTPUT_TRACKING_FILE2_WITH_RATIO)
|
|
|
|
CompareAndReplacePreviousResults(previousRootNode, appName, appVersion, moduleName, className, perftestName, float(perftestResult))
|
|
print("Adding "+appName+" perftests results to Tracker '"+"gear_core_tracker.txt"+"'...\n")
|
|
trackerCmdLine = os.path.dirname(os.getcwd()).replace('\\', '/') + "/private/tools/Tracker/Tracker.exe -v --datafile " + OUTPUT_TRACKING_FILE1_WITH_RATIO;
|
|
if sys.platform != "win32":
|
|
trackerCmdLine = "mono " + trackerCmdLine
|
|
|
|
print("Calling Tracker: " + trackerCmdLine + "\n")
|
|
os.system(trackerCmdLine)
|
|
|
|
print("Adding "+appName+" perftests results to Tracker '"+"stl_tracker.txt"+"'...\n")
|
|
trackerCmdLine = os.path.dirname(os.getcwd()).replace('\\', '/') + "/private/tools/Tracker/Tracker.exe -v --datafile " + OUTPUT_TRACKING_FILE2_WITH_RATIO;
|
|
if sys.platform != "win32":
|
|
trackerCmdLine = "mono " + trackerCmdLine
|
|
|
|
print("Calling Tracker: " + trackerCmdLine + "\n")
|
|
os.system(trackerCmdLine)
|
|
|
|
os.remove(TRACKER_FILE)
|
|
os.remove(TRACKING_FILE1)
|
|
os.remove(TRACKING_FILE2)
|
|
os.remove(OUTPUT_TRACKING_FILE1_WITH_RATIO)
|
|
os.remove(OUTPUT_TRACKING_FILE2_WITH_RATIO)
|
|
|
|
|
|
os.remove(PERFTEST_FILE)
|
|
print("Monitoring done. "+str(totalApp)+" application(s) analyzed. Have fun.\n")
|
|
return 0
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit( main() ) |