define("doh/runner", ["dojo/_base/lang"], function(lang){ var doh = { // summary: // Functions for registering and running automated tests. }; // intentionally define global tests and global doh symbols // TODO: scrub these globals from tests and remove this pollution for 2.0 tests = doh; this.doh = doh; doh._line = "------------------------------------------------------------"; doh.debug = function(){ // summary: // takes any number of arguments and sends them to whatever debugging // or logging facility is available in this environment // YOUR TEST RUNNER NEEDS TO IMPLEMENT THIS }; doh.error = function(){ // summary: // logging method to be used to send Error objects, so that // whatever debugging or logging facility you have can decide to treat it // as an Error object and show additional information - such as stack trace // YOUR TEST RUNNER NEEDS TO IMPLEMENT THIS }; doh._AssertFailure = function(msg, hint){ if (doh.breakOnError) { //>>excludeStart("debuggerCrashesRhino", /^shrinksafe.comments/.test(kwArgs.optimize)); debugger; //>>excludeEnd("debuggerCrashesRhino") } if(!(this instanceof doh._AssertFailure)){ return new doh._AssertFailure(msg, hint); } if(hint){ msg = (new String(msg||""))+" with hint: \n\t\t"+(new String(hint)+"\n"); } this.message = new String(msg||""); return this; }; doh._AssertFailure.prototype = new Error(); doh._AssertFailure.prototype.constructor = doh._AssertFailure; doh._AssertFailure.prototype.name = "doh._AssertFailure"; doh.Deferred = function(canceller){ this.chain = []; this.id = this._nextId(); this.fired = -1; this.paused = 0; this.results = [null, null]; this.canceller = canceller; this.silentlyCancelled = false; }; lang.extend(doh.Deferred, { getTestErrback: function(cb, scope){ // summary: // Replaces outer getTextCallback's in nested situations to avoid multiple callback(true)'s var _this = this; return function(){ try{ cb.apply(scope||doh.global||_this, arguments); }catch(e){ _this.reject(e); } }; }, getTestCallback: function(cb, scope){ var _this = this; return function(){ try{ cb.apply(scope||doh.global||_this, arguments); }catch(e){ _this.reject(e); return; } _this.resolve(true); }; }, _nextId: (function(){ var n = 1; return function(){ return n++; }; })(), cancel: function(){ if(this.fired == -1){ if (this.canceller){ this.canceller(this); }else{ this.silentlyCancelled = true; } if(this.fired == -1){ this.reject(new Error("Deferred(unfired)")); } }else if(this.fired == 0 && this.results[0] && this.results[0].cancel){ this.results[0].cancel(); } }, _pause: function(){ this.paused++; }, _unpause: function(){ this.paused--; if ((this.paused == 0) && (this.fired >= 0)) { this._fire(); } }, _continue: function(res){ this._resback(res); this._unpause(); }, _resback: function(res){ this.fired = ((res instanceof Error) ? 1 : 0); this.results[this.fired] = res; this._fire(); }, _check: function(){ if(this.fired != -1){ if(!this.silentlyCancelled){ throw new Error("already called!"); } this.silentlyCancelled = false; } }, resolve: function(res){ this._check(); this._resback(res); }, reject: function(res){ this._check(); if(!(res instanceof Error)){ res = new Error(res); } this._resback(res); }, then: function(cb, eb){ this.chain.push([cb, eb]); if(this.fired >= 0){ this._fire(); } return this; }, always: function(cb){ this.then(cb, cb); }, otherwise: function(eb){ this.then(null, eb); }, isFulfilled: function(){ return this.fired >= 0; }, isResolved: function(){ return this.fired == 0; }, isRejected: function(){ return this.fired == 1; }, _fire: function(){ var chain = this.chain; var fired = this.fired; var res = this.results[fired]; var self = this; var cb = null; while(chain.length > 0 && this.paused == 0){ // Array var pair = chain.shift(); var f = pair[fired]; if(f == null){ continue; } try { res = f(res); fired = ((res instanceof Error) ? 1 : 0); if(res && res.then){ cb = function(res){ self._continue(res); }; this._pause(); } }catch(err){ fired = 1; res = err; } } this.fired = fired; this.results[fired] = res; if((cb)&&(this.paused)){ res.always(cb); } } }); lang.extend(doh.Deferred, { // Back compat methods, remove for 2.0 getFunctionFromArgs: function(){ // Like lang.hitch but first arg (context) is optional var a = arguments; if((a[0])&&(!a[1])){ if(typeof a[0] == "function"){ return a[0]; }else if(typeof a[0] == "string"){ return doh.global[a[0]]; } }else if((a[0])&&(a[1])){ return lang.hitch(a[0], a[1]); } return null; }, addCallbacks: function(cb, eb){ this.then(cb, eb); }, addCallback: function(cb, cbfn){ var enclosed = this.getFunctionFromArgs(cb, cbfn); if(arguments.length > 2){ enclosed = lang.hitch(null, enclosed, arguments, 2); } return this.then(enclosed); }, addErrback: function(cb, cbfn){ var enclosed = this.getFunctionFromArgs(cb, cbfn); if(arguments.length > 2){ enclosed = lang.hitch(null, enclosed, arguments, 2); } return this.otherwise(enclosed); }, addBoth: function(cb, cbfn){ var enclosed = this.getFunctionFromArgs(cb, cbfn); if(arguments.length > 2){ enclosed = lang.hitch(null, enclosed, arguments, 2); } return this.always(enclosed); }, callback: function(val){ this.resolve(val); }, errback: function(val){ this.reject(val); } }); // // State Keeping and Reporting // doh._testCount = 0; doh._groupCount = 0; doh._errorCount = 0; doh._failureCount = 0; doh._currentGroup = null; doh._currentTest = null; doh._paused = true; doh._init = function(){ this._currentGroup = null; this._currentTest = null; this._errorCount = 0; this._failureCount = 0; this.debug(this._testCount, "tests to run in", this._groupCount, "groups"); }; doh._groups = {}; // // Test Types // doh._testTypes= {}; doh.registerTestType= function(name, initProc){ // summary: // Adds a test type and associates a function used to initialize each test of the given type // name: String // The name of the type. // initProc: Function // Type specific test initializer; called after the test object is created. doh._testTypes[name]= initProc; }; doh.registerTestType("perf", function(group, tObj, type){ // Augment the test with some specific options to make it identifiable as a // particular type of test so it can be executed properly. if(type === "perf" || tObj.testType === "perf"){ tObj.testType = "perf"; // Build an object on the root DOH class to contain all the test results. // Cache it on the test object for quick lookup later for results storage. if(!doh.perfTestResults){ doh.perfTestResults = {}; doh.perfTestResults[group] = {}; } if(!doh.perfTestResults[group]){ doh.perfTestResults[group] = {}; } if(!doh.perfTestResults[group][tObj.name]){ doh.perfTestResults[group][tObj.name] = {}; } tObj.results = doh.perfTestResults[group][tObj.name]; // If it's not set, then set the trial duration; default to 100ms. if(!("trialDuration" in tObj)){ tObj.trialDuration = 100; } // If it's not set, then set the delay between trial runs to 100ms // default to 100ms to allow for GC and to make IE happy. if(!("trialDelay" in tObj)){ tObj.trialDelay = 100; } // If it's not set, then set number of times a trial is run to 10. if(!("trialIterations" in tObj)){ tObj.trialIterations = 10; } } }); // // Test Registration // var createFixture= function(group, test, type){ // test is a function, string, or fixture object var tObj = test; if(lang.isString(test)){ tObj = { name: test.replace("/\s/g", "_"), // FIXME: bad escapement runTest: new Function("t", test) }; }else if(lang.isFunction(test)){ // if we didn't get a fixture, wrap the function tObj = { "runTest": test }; if(test["name"]){ tObj.name = test.name; }else{ try{ var fStr = "function "; var ts = tObj.runTest+""; if(0 <= ts.indexOf(fStr)){ tObj.name = ts.split(fStr)[1].split("(", 1)[0]; } // doh.debug(tObj.runTest.toSource()); }catch(e){ } } // FIXME: try harder to get the test name here }else if(lang.isString(tObj.runTest)){ tObj.runTest= new Function("t", tObj.runTest); } if(!tObj.runTest){ return 0; } // if the test is designated as a particular type, do type-specific initialization var testType= doh._testTypes[type] || doh._testTypes[tObj.testType]; if(testType){ testType(group, tObj); } // add the test to this group doh._groups[group].push(tObj); doh._testCount++; doh._testRegistered(group, tObj); return tObj; }, dumpArg= function(arg){ if(lang.isString(arg)){ return "string(" + arg + ")"; } else { return typeof arg; } }, illegalRegister= function(args, testArgPosition){ var hint= "\targuments: "; for(var i= 0; i<5; i++){ hint+= dumpArg(args[i]); } doh.debug("ERROR:"); if(testArgPosition){ doh.debug("\tillegal arguments provided to doh.register; the test at argument " + testArgPosition + " wasn't a test."); }else{ doh.debug("\tillegal arguments provided to doh.register"); } doh.debug(hint); }; doh._testRegistered = function(group, fixture){ // slot to be filled in }; doh._groupStarted = function(group){ // slot to be filled in }; doh._groupFinished = function(group, success){ // slot to be filled in }; doh._testStarted = function(group, fixture){ // slot to be filled in }; doh._testFinished = function(group, fixture, success){ // slot to be filled in }; doh._registerTest = function(group, test, type){ // summary: // add the provided test function or fixture object to the specified // test group. // group: String // string name of the group to add the test to // test: Function||String||Object // TODOC // type: String? // An identifier denoting the type of testing that the test performs, such // as a performance test. If falsy, defaults to test.type. // get, possibly create, the group object var groupObj = this._groups[group]; if(!groupObj){ this._groupCount++; groupObj = this._groups[group] = []; groupObj.inFlight = 0; } if(!test){ return groupObj; } // create the test fixture var tObj; if(lang.isFunction(test) || lang.isString(test) || "runTest" in test){ return createFixture(group, test, type) ? groupObj : 0; }else if(lang.isArray(test)){ // a vector of tests... for(var i=0; i 1) ? "s" : "")+" to run"); doh._groupStarted(groupName); }; doh._handleFailure = function(groupName, fixture, e){ // this.debug("FAILED test:", fixture.name); // mostly borrowed from JUM this._groups[groupName].failures++; var out = ""; if(e instanceof this._AssertFailure){ this._failureCount++; if(e["fileName"]){ out += e.fileName + ':'; } if(e["lineNumber"]){ out += e.lineNumber + ' '; } out += e.message; this.error("\t_AssertFailure:", out); }else{ this._errorCount++; this.error("\tError:", e.message || e); // printing Error on IE9 (and other browsers?) yields "[Object Error]" } if(fixture.runTest["toSource"]){ var ss = fixture.runTest.toSource(); this.debug("\tERROR IN:\n\t\t", ss); }else{ this.debug("\tERROR IN:\n\t\t", fixture.runTest); } if(e.rhinoException){ e.rhinoException.printStackTrace(); }else if(e.javaException){ e.javaException.printStackTrace(); } }; doh._runPerfFixture = function(/*String*/ groupName, /*Object*/ fixture){ // summary: // This function handles how to execute a 'performance' test // which is different from a straight UT style test. These // will often do numerous iterations of the same operation and // gather execution statistics about it, like max, min, average, // etc. It makes use of the already in place DOH deferred test // handling since it is a good idea to put a pause in between each // iteration to allow for GC cleanup and the like. // groupName: // The test group that contains this performance test. // fixture: // The performance test fixture. var tg = this._groups[groupName]; fixture.startTime = new Date(); // Perf tests always need to act in an async manner as there is a // number of iterations to flow through. var def = new doh.Deferred(); tg.inFlight++; def.groupName = groupName; def.fixture = fixture; var threw = false; def.otherwise(function(err){ doh._handleFailure(groupName, fixture, err); threw = true; }); // Set up the finalizer. var fulfilled; var retEnd = function(){ fulfilled = true; if(fixture["tearDown"]){ fixture.tearDown(doh); } tg.inFlight--; if((!tg.inFlight)&&(tg.iterated)){ doh._groupFinished(groupName, !tg.failures); } doh._testFinished(groupName, fixture, !threw); if(doh._paused){ doh.run(); } }; // Since these can take who knows how long, we don't want to timeout // unless explicitly set var timer; var to = fixture.timeout; if(to > 0) { timer = setTimeout(function(){ def.reject(new Error("test timeout in "+fixture.name.toString())); }, to); } // Set up the end calls to the test into the deferred we'll return. def.always(function(){ if(timer){ clearTimeout(timer); } retEnd(); }); // Okay, now set up the timing loop for the actual test. // This is down as an async type test where there is a delay // between each execution to allow for GC time, etc, so the GC // has less impact on the tests. var res = fixture.results; res.trials = []; // Try to figure out how many calls are needed to hit a particular threshold. var itrDef = doh._calcTrialIterations(groupName, fixture); // Blah, since tests can be deferred, the actual run has to be deferred until after // we know how many iterations to run. This is just plain ugly. itrDef.then( function(iterations){ if(iterations){ var countdown = fixture.trialIterations; doh.debug("TIMING TEST: [" + fixture.name + "]\n\t\tITERATIONS PER TRIAL: " + iterations + "\n\tTRIALS: " + countdown); // Figure out how many times we want to run our 'trial'. // Where each trial consists of 'iterations' of the test. var trialRunner = function() { // Set up our function to execute a block of tests var start = new Date(); var tTimer = new doh.Deferred(); var tState = { countdown: iterations }; var testRunner = function(state){ while(state){ try{ state.countdown--; if(state.countdown){ var ret = fixture.runTest(doh); if(ret && ret.then){ // Deferreds have to be handled async, // otherwise we just keep looping. var atState = { countdown: state.countdown }; ret.then( function(){ testRunner(atState) }, function(err){ doh._handleFailure(groupName, fixture, err); fixture.endTime = new Date(); def.reject(err); } ); state = null; } }else{ tTimer.resolve(new Date()); state = null; } }catch(err){ fixture.endTime = new Date(); tTimer.reject(err); } } }; tTimer.then( function(end){ // Figure out the results and try to factor out function call costs. var tResults = { trial: (fixture.trialIterations - countdown), testIterations: iterations, executionTime: (end.getTime() - start.getTime()), average: (end.getTime() - start.getTime())/iterations }; res.trials.push(tResults); doh.debug("\n\t\tTRIAL #: " + tResults.trial + "\n\tTIME: " + tResults.executionTime + "ms.\n\tAVG TEST TIME: " + (tResults.executionTime/tResults.testIterations) + "ms."); // Okay, have we run all the trials yet? countdown--; if(countdown){ setTimeout(trialRunner, fixture.trialDelay); }else{ // Okay, we're done, let's compute some final performance results. var t = res.trials; // We're done. fixture.endTime = new Date(); def.resolve(true); } }, // Handler if tTimer gets an error function(err){ fixture.endTime = new Date(); def.reject(err); } ); testRunner(tState); }; trialRunner(); } }, // Handler if itrDef gets an error function(err){ fixture.endTime = new Date(); def.reject(err); } ); // Set for a pause, returned the deferred. if(!fulfilled){ doh.pause(); } return def; }; doh._calcTrialIterations = function(/*String*/ groupName, /*Object*/ fixture){ // summary: // This function determines the rough number of iterations to // use to reach a particular MS threshold. This returns a deferred // since tests can theoretically by async. Async tests aren't going to // give great perf #s, though. // The callback is passed the # of iterations to hit the requested // threshold. // fixture: // The test fixture we want to calculate iterations for. var def = new doh.Deferred(); var calibrate = function () { var testFunc = lang.hitch(fixture, fixture.runTest); // Set the initial state. We have to do this as a loop instead // of a recursive function. Otherwise, it blows the call stack // on some browsers. var iState = { start: new Date(), curIter: 0, iterations: 5 }; var handleIteration = function(state){ while(state){ if(state.curIter < state.iterations){ try{ var ret = testFunc(doh); if(ret && ret.then){ var aState = { start: state.start, curIter: state.curIter + 1, iterations: state.iterations }; ret.then( function(){ handleIteration(aState); }, function(err) { fixture.endTime = new Date(); def.reject(err); } ); state = null; }else{ state.curIter++; } }catch(err){ fixture.endTime = new Date(); def.reject(err); return; } }else{ var end = new Date(); var totalTime = (end.getTime() - state.start.getTime()); if(totalTime < fixture.trialDuration){ var nState = { iterations: state.iterations * 2, curIter: 0 }; state = null; setTimeout(function(){ nState.start = new Date(); handleIteration(nState); }, 50); }else{ var itrs = state.iterations; setTimeout(function(){def.resolve(itrs)}, 50); state = null; } } } }; handleIteration(iState); }; setTimeout(calibrate, 10); return def; }; doh._runRegFixture = function(/*String*/ groupName, /*Object*/ fixture){ // summary: // Function to help run a generic doh test. Called from _runFixture(). These are not // specialized tests, like performance groups and such. // groupName: // The groupName of the test. // fixture: // The test fixture to execute. var tg = this._groups[groupName]; fixture.startTime = new Date(); var ret = fixture.runTest(this); // if we get a deferred back from the test runner, we know we're // gonna wait for an async result. It's up to the test code to trap // errors and give us an errback or callback. if(ret && ret.then){ // If ret is a dojo/Deferred, get the corresponding Promise; it has some additional methods we need. if(ret.promise){ ret = ret.promise; } tg.inFlight++; ret.groupName = groupName; ret.fixture = fixture; // Setup handler for when test fails. var threw = false; ret.otherwise(function(err){ if(threw){ // the fixture timeout (below) must have already fired return; } doh._handleFailure(groupName, fixture, err); threw = true; }); var fulfilled; var retEnd = function(){ // summary: // Called when tests finishes successfully, fails, or times out if(fulfilled){ // retEnd() has already executed; probably the timeout above fired and then later ret completed. return; } fulfilled = true; fixture.endTime = new Date(); if(fixture.tearDown){ try { fixture.tearDown(doh); }catch(e){ this.debug("Error tearing down test: "+e.message); } } tg.inFlight--; doh._testFinished(groupName, fixture, !threw); if((!tg.inFlight)&&(tg.iterated)){ doh._groupFinished(groupName, !tg.failures); } // Go on to next test if(doh._paused){ doh.run(); } }; var timer = setTimeout(function(){ if(!timer){ // we already called clearTimeout(), but it fired anyway, due to IE bug; just ignore. return; } // Note: cannot call ret.reject() because ret may be a readonly promise doh._handleFailure(groupName, fixture, new Error("test timeout in " + fixture.name.toString())); threw = true; retEnd(); }, fixture["timeout"]||1000); ret.always(function(){ clearTimeout(timer); timer = null; retEnd(); }); if(!fulfilled){ doh.pause(); } return ret; }else{ // Synchronous test; tearDown etc. handled in _runFixture(), the function that called me } }; doh._runFixture = function(groupName, fixture){ var tg = this._groups[groupName]; this._testStarted(groupName, fixture); var threw = false; var err = null; // run it, catching exceptions and reporting them try{ // let doh reference "this.group.thinger..." which can be set by // another test or group-level setUp function fixture.group = tg; // only execute the parts of the fixture we've got if(fixture["setUp"]){ fixture.setUp(this); } if(fixture["runTest"]){ // should we error out of a fixture doesn't have a runTest? if(fixture.testType === "perf"){ // Always async deferred, so return it. return doh._runPerfFixture(groupName, fixture); }else{ // May or may not by async. var ret = doh._runRegFixture(groupName, fixture); if(ret){ // this design is ridiculous, but tearDown etc. is handled in _runRegFixture iff fixture is async; // likewise with runPerfFixture return ret; } } } }catch(e){ threw = true; err = e; } // The rest of the code in this function executes only if test returns synchronously... fixture.endTime = new Date(); // should try to tear down regardless whether test passed or failed... try{ if(fixture["tearDown"]){ fixture.tearDown(this); } }catch(e){ this.debug("Error tearing down test: "+e.message); } var d = new doh.Deferred(); setTimeout(lang.hitch(this, function(){ if(threw){ this._handleFailure(groupName, fixture, err); } this._testFinished(groupName, fixture, !threw); if((!tg.inFlight)&&(tg.iterated)){ doh._groupFinished(groupName, !tg.failures); }else if(tg.inFlight > 0){ setTimeout(lang.hitch(this, function(){ doh.runGroup(groupName); }), 100); this._paused = true; } if(doh._paused){ doh.run(); } }), 30); doh.pause(); return d; }; doh.runGroup = function(/*String*/ groupName, /*Integer*/ idx){ // summary: // runs the specified test group // the general structure of the algorithm is to run through the group's // list of doh, checking before and after each of them to see if we're in // a paused state. This can be caused by the test returning a deferred or // the user hitting the pause button. In either case, we want to halt // execution of the test until something external to us restarts it. This // means we need to pickle off enough state to pick up where we left off. // FIXME: need to make fixture execution async!! idx = idx || 0; var tg = this._groups[groupName]; if(tg.skip === true){ return; } if(lang.isArray(tg)){ if(tg.iterated===undefined){ tg.iterated = false; tg.inFlight = 0; tg.failures = 0; this._setupGroupForRun(groupName); if(tg["setUp"]){ tg.setUp(this); } } for(var y=idx; y