Merge pull request #636 from mathquill/ci.screenshots
Take screenshots of visual.html during unit tests
diff --git a/circle.yml b/circle.yml
index fcb7b97..5611ba0 100644
--- a/circle.yml
+++ b/circle.yml
@@ -35,6 +35,9 @@
cache_directories:
- ~/sauce-connect
pre:
+ # imagemagick is installed to give us access to the
+ # `convert` tool to stitch together the screenshots.
+ - sudo apt-get update; sudo apt-get install imagemagick
- ? |-
mkdir -p ~/sauce-connect
cd ~/sauce-connect
@@ -49,7 +52,9 @@
test:
override:
- - make server:
+ # Sauce cant connect to Safari unless the server is ran on port 3000 or 4000
+ # https://david263a.wordpress.com/2015/04/18/fixing-safari-cant-connect-to-localhost-issue-when-using-sauce-labs-connect-tunnel/
+ - PORT=3000 make server:
background: true
# CircleCI expects test results to be reported in an JUnit/xUnit-style XML
@@ -76,6 +81,10 @@
# are much faster, no need to wait for them)
- while [ ! -e ~/sauce_is_ready ]; do sleep 1; done
+ # Start taking screenshots in the background while the unit tests are running
+ - npm install wd && node script/screenshots.js http://localhost:3000/test/visual.html && touch ~/screenshots_are_ready:
+ background: true
+
# Run in-browser unit tests, based on:
# https://wiki.saucelabs.com/display/DOCS/JavaScript+Unit+Testing+Methods
# "build" and "tag" parameters from:
@@ -92,7 +101,7 @@
"circle-ci"
],
"framework": "mocha",
- "url": "http://localhost:9292/test/unit.html?post_xunit_to=http://localhost:9000",
+ "url": "http://localhost:3000/test/unit.html?post_xunit_to=http://localhost:9000",
"platforms": [["", "Chrome", ""]]
}' | tee js-tests
@@ -118,6 +127,17 @@
[ "$(node -p 'require("./status.json").completed')" != false ] && break
done
+ # Wait for screenshots to be ready
+ - while [ ! -e ~/screenshots_are_ready ]; do sleep 1; done:
+ timeout: 300
+
+ - |-
+ dir=$CIRCLE_ARTIFACTS/imgs
+ for x in $(ls $dir)
+ do
+ convert $dir/$x/*.png -append $dir/$x.png
+ done
+
# finally, complain to Circle CI if there were nonzero test failures
- |-
[ "$(node -p 'require("./status.json")["js tests"][0].result.failures')" == 0 ]
diff --git a/script/screenshots.js b/script/screenshots.js
new file mode 100644
index 0000000..4ff63d3
--- /dev/null
+++ b/script/screenshots.js
@@ -0,0 +1,162 @@
+// This script assumes the following:
+// 1. You've installed wd with `npm install wd'.
+// 2. You've set the environment variables $SAUCE_USERNAME and $SAUCE_ACCESS_KEY.
+// 3. If the environment variable $CIRCLE_ARTIFACTS is not set images will be saved in /tmp
+//
+// This scripts creates following files for each browser in browserVersions:
+// $CIRCLE_ARTIFACTS/imgs/{browser_version_platform}/#.png
+//
+// The intention of this script is that it will be ran from CircleCI
+//
+// Example usage:
+// node screenshots.js http://localhost:9292/test/visual.html
+// node screenshots.js http://google.com
+
+var wd = require('wd');
+var fs = require('fs');
+var username = process.env['SAUCE_USERNAME'];
+var accessKey = process.env['SAUCE_ACCESS_KEY'];
+var baseDir = process.env['CIRCLE_ARTIFACTS'] || '/tmp';
+var url = process.argv[2];
+var allImgsDir = baseDir+'/imgs';
+fs.mkdirSync(allImgsDir);
+
+var browserVersions = [
+ {
+ // Expecting IE 8
+ 'browserName': 'Internet Explorer',
+ 'platform': 'Windows XP'
+ },
+ {
+ // Expecting IE 11
+ 'browserName': 'Internet Explorer',
+ 'platform': 'Windows 7'
+ },
+ {
+ 'browserName': 'MicrosoftEdge',
+ 'platform': 'Windows 10'
+ },
+ {
+ 'browserName': 'Firefox',
+ 'platform': 'OS X 10.11'
+ },
+ {
+ 'browserName': 'Safari',
+ 'platform': 'OS X 10.11'
+ },
+ {
+ 'browserName': 'Chrome',
+ 'platform': 'OS X 10.11'
+ },
+ {
+ 'browserName': 'Firefox',
+ 'platform': 'Linux'
+ },
+];
+
+
+browserVersions.forEach(function(cfg) {
+ var browserDriver = wd.remote('ondemand.saucelabs.com', 80, username, accessKey);
+ // The following is in the style of
+ // https://github.com/admc/wd/blob/62f2b0060d36a402de5634477b26a5ed4c051967/examples/async/chrome.js#L25-L40
+ browserDriver.init(cfg, function(err, _, capabilities) {
+ if (err) console.log(err);
+
+ var browser = cfg.browserName.replace(/\s/g, '_')+(capabilities ? '_'+capabilities.version : '');
+ var platform = (capabilities || cfg).platform.replace(/\s/g, '_');
+ var piecesDir = allImgsDir+'/'+browser+'_'+platform;
+ fs.mkdirSync(piecesDir);
+
+ browserDriver.get(url, function(err) {
+ if (err) console.log(err);
+ browserDriver.safeExecute('document.documentElement.scrollHeight', function(err,scrollHeight) {
+ if (err) console.log(err);
+ browserDriver.safeExecute('document.documentElement.clientHeight', function(err,viewportHeight) {
+ if (err) console.log(err);
+
+ // Firefox and Internet Explorer will take a screenshot of the entire webpage,
+ if (cfg.browserName != 'Safari' && cfg.browserName != 'Chrome' && cfg.browserName != 'MicrosoftEdge') {
+ // saves file in the file `piecesDir/browser_version_platform/*.png`
+ var filename = piecesDir+'/'+browser+'_'+platform+'.png';
+ browserDriver.saveScreenshot(filename, function(err) {
+ if (err) console.log(err);
+
+ browserDriver.log('browser', function(err,logs) {
+ if (err) console.log(err);
+
+ var logfile = baseDir+'/'+browser+'_'+platform+'.log'
+ logs = logs || [];
+ fs.writeFile(logfile,logs.join('\n'), function(err) {
+ if (err) console.log(err);
+
+ browserDriver.quit();
+ });
+ });
+
+ });
+ } else {
+ var scrollTop = 0;
+
+ // loop generates the images. Firefox and Internet Explorer will take
+ // a screenshot of the entire webpage, but Opera, Safari, and Chrome
+ // do not. For those browsers we scroll through the page and take
+ // incremental screenshots.
+ (function loop() {
+ var index = (scrollTop/viewportHeight) + 1;
+
+ // Use `window.scrollTo` because thats what jQuery does
+ // https://github.com/jquery/jquery/blob/1.12.3/src/offset.js#L186
+ // `window.scrollTo` was used instead of jQuery because jQuery was
+ // causing a stackoverflow in Safari.
+ browserDriver.safeEval('window.scrollTo(0,'+scrollTop+');', function(err) {
+ if (err) console.log(JSON.stringify(err));
+
+ // saves file in the file `piecesDir/browser_version_platform/#.png`
+ var filename = piecesDir+'/'+index+'.png';
+ browserDriver.saveScreenshot(filename, function(err) {
+ if (err) console.log(err);
+
+ scrollTop += viewportHeight;
+ if (scrollTop + viewportHeight > scrollHeight) {
+ browserDriver.getWindowSize(function(err,size) {
+ // account for the viewport offset
+ var extra = size.height - viewportHeight;
+ browserDriver.setWindowSize(size.width, (scrollHeight-scrollTop)+extra, function(err) {
+ if (err) console.log(err);
+
+ browserDriver.safeEval('window.scrollTo(0,'+scrollHeight+');', function(err) {
+ if (err) console.log(JSON.stringify(err));
+
+ index++;
+ var filename = piecesDir+'/'+index+'.png';
+ browserDriver.saveScreenshot(filename, function(err) {
+ if (err) console.log(err);
+
+ browserDriver.log('browser', function(err,logs) {
+ if (err) console.log(err);
+
+ var logfile = baseDir+'/'+browser+'_'+platform+'.log'
+ logs = logs || [];
+ fs.writeFile(logfile,logs.join('\n'), function(err) {
+ if (err) console.log(err);
+
+ browserDriver.quit();
+ });
+ });
+ });
+ });
+
+ });
+ });
+ } else {
+ loop();
+ }
+ });
+ });
+ })();
+ }
+ });
+ });
+ });
+ });
+});
diff --git a/script/test_server.js b/script/test_server.js
index a551bae..17a7420 100644
--- a/script/test_server.js
+++ b/script/test_server.js
@@ -35,6 +35,8 @@
}
}
else {
+ var ext = filepath.match(/\.[^.]+$/);
+ if (ext) res.setHeader('Content-Type', 'text/' + ext[0].slice(1));
res.end(data);
}
diff --git a/test/unit/focusBlur.test.js b/test/unit/focusBlur.test.js
index 27180ae..20b4fbd 100644
--- a/test/unit/focusBlur.test.js
+++ b/test/unit/focusBlur.test.js
@@ -80,7 +80,7 @@
$(mq.el()).remove();
done();
});
- }, 10);
+ }, 100);
});
});
});
diff --git a/test/visual.html b/test/visual.html
index 0ccda79..acfa902 100644
--- a/test/visual.html
+++ b/test/visual.html
@@ -280,7 +280,8 @@
</div>
<script type="text/javascript">
-window.onerror = function() {
+window.onerror = function(err) {
+ console.log(err); // to show up in Selenium WebDriver logs
$('html').css('background', 'red');
};
</script>
