| # Okay so maybe everyone else already knows all this, but it took some time |
| # for Michael and I [Han] to really see how everything fits together. |
| # |
| # Basically, what we're doing here is automated browser testing, so CircleCI |
| # handles the automation, and Sauce Labs handles the browser testing. |
| # Specifically, Sauce Labs offers a REST API to run tests in browsers in VMs, |
| # and CircleCI can be configured to listen for git pushes and run local |
| # servers and call out to REST APIs to test against these local servers. |
| # |
| # The flow goes like this: |
| # - CircleCI notices/is notified of a git push |
| # - they pull and checkout and magically know to install dependencies and shit |
| # + https://circleci.com/docs/manually/ |
| # - their magic works fine for MathQuill's dependencies but to run the tests, |
| # it foolishly runs `make test`, what an inconceivable mistake |
| # - that's where we come in: `circle.yml` lets us override the test script. |
| # + https://circleci.com/docs/configuration/ |
| # - our `circle.yml` first installs and runs a tunnel to Sauce Labs |
| # - and runs `make server` |
| # - then it calls out to Sauce Labs' REST API to open browsers that reach |
| # back through the tunnel to access test pages on the local server |
| # + > Sauce Connect allows you to run a test server within the CircleCI |
| # > build container and expose it it (using a URL like `localhost:8080`) |
| # > to Sauce Labs’ browsers. |
| # |
| # https://circleci.com/docs/browser-testing-with-sauce-labs/ |
| # |
| # - boom testing boom |
| |
| |
| # this file is based on https://github.com/circleci/sauce-connect/blob/a65e41c91e02550ce56c75740a422bebc4acbf6f/circle.yml |
| # via https://circleci.com/docs/browser-testing-with-sauce-labs/ |
| |
| machine: |
| environment: |
| json: node_modules/.bin/json |
| |
| dependencies: |
| cache_directories: |
| - ~/sauce-connect |
| pre: |
| - "test $SAUCE_USERNAME && test $SAUCE_ACCESS_KEY |
| # Sauce Labs credentials required. Sign up here: https://saucelabs.com/opensauce/" |
| - ? |- |
| { |
| mkdir -p ~/sauce-connect |
| cd ~/sauce-connect |
| if [ -x sc-*-linux/bin/sc ]; then |
| echo Using cached sc-*-linux/bin/sc |
| else |
| time wget https://saucelabs.com/downloads/sc-latest-linux.tar.gz |
| time tar -xzf sc-latest-linux.tar.gz |
| fi |
| # Sauce Connect randomly fails so try twice https://git.io/vPN8v |
| time sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY \ |
| --readyfile ~/sauce_is_ready |
| test -e ~/sauce_was_ready && exit |
| time sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY \ |
| --readyfile ~/sauce_is_ready |
| test -e ~/sauce_was_ready && exit |
| echo 'ERROR: Exited twice without creating readyfile' \ |
| | tee /dev/stderr > ~/sauce_is_ready |
| exit 1 |
| } >$CIRCLE_ARTIFACTS/sauce-connect.log 2>&1 |
| : |
| background: true |
| - test -x $json || npm install json |
| |
| test: |
| override: |
| # Safari on Sauce can only connect to port 3000, 4000, 7000, or 8000. Edge needs port 7000 or 8000. |
| # https://david263a.wordpress.com/2015/04/18/fixing-safari-cant-connect-to-localhost-issue-when-using-sauce-labs-connect-tunnel/ |
| # https://support.saucelabs.com/customer/portal/questions/14368823-requests-to-localhost-on-microsoft-edge-are-failing-over-sauce-connect |
| - PORT=8000 make server >$CIRCLE_ARTIFACTS/make_server.log 2>&1: |
| background: true |
| |
| # CircleCI expects test results to be reported in an JUnit/xUnit-style XML file: |
| # https://circleci.com/docs/test-metadata/ |
| # Our unit tests are in a browser, so they can't write to a file, and Sauce |
| # apparently truncates custom data in their test result reports, so instead we |
| # POST to this trivial Node server on localhost:9000 that writes the body of |
| # any POST request to $CIRCLE_TEST_REPORTS/mocha/xunit.xml |
| - mkdir -p $CIRCLE_TEST_REPORTS/mocha |
| - ? |- |
| node << 'EOF' >$CIRCLE_TEST_REPORTS/mocha/xunit.xml \ |
| 2>$CIRCLE_ARTIFACTS/mocha-test-report.log |
| require('http').createServer(function(req, res) { |
| res.setHeader('Access-Control-Allow-Origin', '*'); |
| req.pipe(process.stdout); |
| req.on('end', res.end.bind(res)); |
| }) |
| .listen(9000); |
| console.error('listening on http://0.0.0.0:9000/'); |
| EOF |
| : |
| background: true |
| |
| # Wait for tunnel to be ready (`make server` and the trivial Node server |
| # are much faster, no need to wait for them) |
| - while [ ! -e ~/sauce_is_ready ]; do sleep 1; done; touch ~/sauce_was_ready; test -z "$(<~/sauce_is_ready)" |
| |
| # Start taking screenshots in the background while the unit tests are running |
| - ? |- |
| { |
| build_name="CircleCI build #$CIRCLE_BUILD_NUM" |
| if [ $CIRCLE_PR_NUMBER ]; then |
| build_name="$build_name: PR #$CIRCLE_PR_NUMBER" |
| [ "$CIRCLE_BRANCH" ] && build_name="$build_name ($CIRCLE_BRANCH)" |
| else |
| build_name="$build_name: $CIRCLE_BRANCH" |
| fi |
| build_name="$build_name @ ${CIRCLE_SHA1:0:7}" |
| export MQ_CI_BUILD_NAME="$build_name" |
| |
| time { test -d node_modules/wd || npm install wd; } |
| time node script/screenshots.js http://localhost:8000/test/visual.html \ |
| && touch ~/screenshots_are_ready || echo EXIT STATUS $? | tee /dev/stderr > ~/screenshots_are_ready: |
| } >$CIRCLE_ARTIFACTS/screenshots.log 2>&1 |
| : |
| background: true |
| |
| # Run in-browser unit tests, based on: |
| # https://wiki.saucelabs.com/display/DOCS/JavaScript+Unit+Testing+Methods |
| # "build" and "customData" parameters from: |
| # https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options#TestConfigurationOptions-TestAnnotation |
| # CircleCI environment variables from: |
| # https://circleci.com/docs/environment-variables/ |
| - |- |
| build_name="CircleCI build #$CIRCLE_BUILD_NUM" |
| if [ $CIRCLE_PR_NUMBER ]; then |
| build_name="$build_name: PR #$CIRCLE_PR_NUMBER" |
| [ "$CIRCLE_BRANCH" ] && build_name="$build_name ($CIRCLE_BRANCH)" |
| else |
| build_name="$build_name: $CIRCLE_BRANCH" |
| fi |
| build_name="$build_name @ ${CIRCLE_SHA1:0:7}" |
| |
| set -o pipefail |
| curl -i -X POST https://saucelabs.com/rest/v1/$SAUCE_USERNAME/js-tests \ |
| -u $SAUCE_USERNAME:$SAUCE_ACCESS_KEY \ |
| -H 'Content-Type: application/json' \ |
| -d '{ |
| "build": "'"$build_name"'", |
| "customData": {"build_url": "'"$CIRCLE_BUILD_URL"'"}, |
| "framework": "mocha", |
| "url": "http://localhost:8000/test/unit.html?post_xunit_to=http://localhost:9000", |
| "platforms": [["", "Chrome", ""]] |
| }' | tee /dev/stderr | tail -1 > js-tests.json |
| |
| # Wait for tests to finish: |
| # |
| # > Make the request multiple times as the tests run until the response |
| # > contains `completed: true` to the get the final results. |
| # |
| # https://wiki.saucelabs.com/display/DOCS/JavaScript+Unit+Testing+Methods |
| - |- |
| while true # Bash has no do...while >:( |
| do |
| sleep 5 |
| curl -i -X POST https://saucelabs.com/rest/v1/$SAUCE_USERNAME/js-tests/status \ |
| -u $SAUCE_USERNAME:$SAUCE_ACCESS_KEY \ |
| -H 'Content-Type: application/json' \ |
| -d @js-tests.json \ |
| | tee /dev/stderr | tail -1 > status.json |
| |
| # deliberately do `... != false` rather than `... == true` |
| # because unexpected values should break rather than infinite loop |
| [ "$($json completed <status.json)" != false ] && break |
| done |
| |
| # Complain to Circle CI if any unit tests failed |
| - test "$($json 'js tests'.0.result.failures <status.json)" = 0 |
| |
| # Wait for screenshots to be ready |
| - while [ ! -e ~/screenshots_are_ready ]; do sleep 1; done; test -z "$(<~/screenshots_are_ready)": |
| timeout: 300 |
| |
| # Stitch together images |
| - |- |
| for img in $(ls $CIRCLE_ARTIFACTS/imgs/pieces/); do |
| convert $(ls -1 $CIRCLE_ARTIFACTS/imgs/pieces/$img/*.png | sort -n) -append $CIRCLE_ARTIFACTS/imgs/$img.png |
| done |
| |
| # Download the latest screenshots from master. |
| - |- |
| artifacts_json="$(curl https://circleci.com/api/v1/project/mathquill/mathquill/latest/artifacts?branch=ci.cleanup)" |
| #when done testing, restore: artifacts_json="$(curl https://circleci.com/api/v1/project/mathquill/mathquill/latest/artifacts?branch=master)" |
| echo |
| echo '/latest/artifacts?branch=master:' |
| echo |
| echo "$artifacts_json" |
| echo |
| |
| mkdir $CIRCLE_ARTIFACTS/imgs/baseline/ |
| baseline_imgs="$(echo "$artifacts_json" \ |
| | $json -a url pretty_path -d '\n\t' \ |
| | grep '\.png$' \ |
| | grep -v '/pieces/' \ |
| | grep -v '/baseline/' \ |
| | sed "s:\$CIRCLE_ARTIFACTS/imgs/:-o $CIRCLE_ARTIFACTS/imgs/baseline/:")" |
| echo 'Baseline image URLs and files:' |
| echo |
| echo "$baseline_imgs" |
| echo |
| |
| test -z "$baseline_imgs" || curl $baseline_imgs |
| |
| # Generate image diffs. |
| - |- |
| cd $CIRCLE_ARTIFACTS/imgs/ |
| for file in $(ls *.png); do |
| # if evergreen browser, browser version of previous screenshot may not match, |
| # so replace previous browser version with glob |
| baseline=baseline/"$(sh -c "ls $(echo $file | sed 's/[^_]*_(evergreen)/*/')")" |
| echo "Number of different pixels from baseline in $file:" |
| compare -metric AE $baseline $file ${file/%.png/_DIFF.png} |
| done |
| post: |
| - killall --wait sc; true # wait for Sauce Connect to close the tunnel; ignore errors since it's just cleanup |