CI: Consolidate CircleCI commands for readability
This way there are only 3 "commands" (really, entire scripts) that
CircleCI runs, each which begins with a descriptive comment.
This commit also reduces the number of backgrounded commands, and most
backgrounded commands start with a word in the comment that describes it
well, so the 'stdout_*.txt' and 'stderr_*.txt' log outputs auto-captured
by CircleCI will have more readable names and we don't need those *.log
files.
Recommend -w to view diff.
diff --git a/circle.yml b/circle.yml
index 89c56a4..e38fd7e 100644
--- a/circle.yml
+++ b/circle.yml
@@ -39,69 +39,74 @@
cache_directories:
- ~/sauce-connect
pre:
- - "test $SAUCE_USERNAME && test $SAUCE_ACCESS_KEY
- # Sauce Labs credentials required. Sign up here: https://saucelabs.com/opensauce/"
- ? |-
- {
- mkdir -p ~/sauce-connect
- cd ~/sauce-connect
- if [ -x sc-*-linux/bin/sc ]; then
- echo Using cached sc-*-linux/bin/sc
- else
- time wget https://saucelabs.com/downloads/sc-latest-linux.tar.gz
- time tar -xzf sc-latest-linux.tar.gz
- fi
- # Sauce Connect randomly fails so try twice https://git.io/vPN8v
- time sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY \
- --readyfile ~/sauce_is_ready
- test -e ~/sauce_was_ready && exit
- time sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY \
- --readyfile ~/sauce_is_ready
- test -e ~/sauce_was_ready && exit
- echo 'ERROR: Exited twice without creating readyfile' \
- | tee /dev/stderr > ~/sauce_is_ready
+ # SauceConnect: download if not cached, and launch with retry
+ test $SAUCE_USERNAME && test $SAUCE_ACCESS_KEY || {
+ echo 'Sauce Labs credentials required. Sign up here: https://saucelabs.com/opensauce/'
exit 1
- } >$CIRCLE_ARTIFACTS/sauce-connect.log 2>&1
+ }
+
+ mkdir -p ~/sauce-connect
+ cd ~/sauce-connect
+ if [ -x sc-*-linux/bin/sc ]; then
+ echo Using cached sc-*-linux/bin/sc
+ else
+ time wget https://saucelabs.com/downloads/sc-latest-linux.tar.gz
+ time tar -xzf sc-latest-linux.tar.gz
+ fi
+ # Sauce Connect randomly fails so try twice https://git.io/vPN8v
+ time sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY \
+ --readyfile ~/sauce_is_ready
+ test -e ~/sauce_was_ready && exit
+ time sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY \
+ --readyfile ~/sauce_is_ready
+ test -e ~/sauce_was_ready && exit
+ echo 'ERROR: Exited twice without creating readyfile' \
+ | tee /dev/stderr > ~/sauce_is_ready
+ exit 1
:
background: true
- test -x $json || npm install json
test:
- override:
+ pre:
# Safari on Sauce can only connect to port 3000, 4000, 7000, or 8000. Edge needs port 7000 or 8000.
# https://david263a.wordpress.com/2015/04/18/fixing-safari-cant-connect-to-localhost-issue-when-using-sauce-labs-connect-tunnel/
# https://support.saucelabs.com/customer/portal/questions/14368823-requests-to-localhost-on-microsoft-edge-are-failing-over-sauce-connect
- - PORT=8000 make server >$CIRCLE_ARTIFACTS/make_server.log 2>&1:
+ - PORT=8000 make server:
background: true
- # CircleCI expects test results to be reported in an JUnit/xUnit-style XML file:
- # https://circleci.com/docs/test-metadata/
- # Our unit tests are in a browser, so they can't write to a file, and Sauce
- # apparently truncates custom data in their test result reports, so instead we
- # POST to this trivial Node server on localhost:9000 that writes the body of
- # any POST request to $CIRCLE_TEST_REPORTS/mocha/xunit.xml
- - mkdir -p $CIRCLE_TEST_REPORTS/mocha
- - ? |-
- node << 'EOF' >$CIRCLE_TEST_REPORTS/mocha/xunit.xml \
- 2>$CIRCLE_ARTIFACTS/mocha-test-report.log
- require('http').createServer(function(req, res) {
- res.setHeader('Access-Control-Allow-Origin', '*');
- req.pipe(process.stdout);
- req.on('end', res.end.bind(res));
- })
- .listen(9000);
- console.error('listening on http://0.0.0.0:9000/');
- EOF
- :
- background: true
-
- # Wait for tunnel to be ready (`make server` and the trivial Node server
- # are much faster, no need to wait for them)
+ # Wait for tunnel to be ready (`make server` is much faster, no need to wait for it)
- while [ ! -e ~/sauce_is_ready ]; do sleep 1; done; touch ~/sauce_was_ready; test -z "$(<~/sauce_is_ready)"
- # Start taking screenshots in the background while the unit tests are running
+ override:
- ? |-
- {
+ # Screenshots: capture in the background while running unit tests
+ mkdir -p $CIRCLE_TEST_REPORTS/mocha
+
+ # CircleCI expects test results to be reported in an JUnit/xUnit-style XML file:
+ # https://circleci.com/docs/test-metadata/
+ # Our unit tests are in a browser, so they can't write to a file, and Sauce
+ # apparently truncates custom data in their test result reports, so instead we
+ # POST to this trivial Node server on localhost:9000 that writes the body of
+ # any POST request to $CIRCLE_TEST_REPORTS/mocha/xunit.xml
+ node -e '
+ require("http").createServer(function(req, res) {
+ res.setHeader("Access-Control-Allow-Origin", "*");
+ req.pipe(process.stdout);
+ req.on("end", res.end.bind(res));
+ })
+ .listen(9000);
+ console.error("listening on http://0.0.0.0:9000/");
+ ' 2>&1 >$CIRCLE_TEST_REPORTS/mocha/xunit.xml | {
+ # ^ note: `2>&1` must precede `>$CIRCLE_TEST_REPORTS/...` because
+ # shell redirect is like assignment; if it came after, then both
+ # stdout and stderr would be written to `xunit.xml` and nothing
+ # would be piped into here
+
+ head -1 # wait for "listening on ..." to be logged
+
+ # https://circleci.com/docs/environment-variables/
build_name="CircleCI build #$CIRCLE_BUILD_NUM"
if [ $CIRCLE_PR_NUMBER ]; then
build_name="$build_name: PR #$CIRCLE_PR_NUMBER"
@@ -115,17 +120,17 @@
time { test -d node_modules/wd || npm install wd; }
time node script/screenshots.js http://localhost:8000/test/visual.html \
&& touch ~/screenshots_are_ready || echo EXIT STATUS $? | tee /dev/stderr > ~/screenshots_are_ready:
- } >$CIRCLE_ARTIFACTS/screenshots.log 2>&1
+ }
:
background: true
- # Run in-browser unit tests, based on:
- # https://wiki.saucelabs.com/display/DOCS/JavaScript+Unit+Testing+Methods
- # "build" and "customData" parameters from:
- # https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options#TestConfigurationOptions-TestAnnotation
- # CircleCI environment variables from:
- # https://circleci.com/docs/environment-variables/
- |-
+ # Unit tests in the browser
+
+ echo '1. Launch tests'
+ echo
+
+ # https://circleci.com/docs/environment-variables/
build_name="CircleCI build #$CIRCLE_BUILD_NUM"
if [ $CIRCLE_PR_NUMBER ]; then
build_name="$build_name: PR #$CIRCLE_PR_NUMBER"
@@ -135,6 +140,8 @@
fi
build_name="$build_name @ ${CIRCLE_SHA1:0:7}"
+ # "build" and "customData" parameters from:
+ # https://wiki.saucelabs.com/display/DOCS/Test+Configuration+Options#TestConfigurationOptions-TestAnnotation
set -o pipefail
curl -i -X POST https://saucelabs.com/rest/v1/$SAUCE_USERNAME/js-tests \
-u $SAUCE_USERNAME:$SAUCE_ACCESS_KEY \
@@ -148,13 +155,11 @@
"platforms": [["", "Chrome", ""]]
}' | tee /dev/stderr | tail -1 > js-tests.json
- # Wait for tests to finish:
- #
- # > Make the request multiple times as the tests run until the response
- # > contains `completed: true` to the get the final results.
- #
- # https://wiki.saucelabs.com/display/DOCS/JavaScript+Unit+Testing+Methods
- - |-
+ echo '2. Wait for tests to finish:'
+ echo
+ # > Make the request multiple times as the tests run until the response
+ # > contains `completed: true` to the get the final results.
+ # https://wiki.saucelabs.com/display/DOCS/JavaScript+Unit+Testing+Methods
while true # Bash has no do...while >:(
do
sleep 5
@@ -169,21 +174,26 @@
[ "$($json completed <status.json)" != false ] && break
done
- # Complain to Circle CI if any unit tests failed
- - test "$($json 'js tests'.0.result.failures <status.json)" = 0
+ echo '3. Exit with non-zero status code if any unit tests failed'
+ exit "$($json 'js tests'.0.result.failures <status.json)"
- # Wait for screenshots to be ready
- - while [ ! -e ~/screenshots_are_ready ]; do sleep 1; done; test -z "$(<~/screenshots_are_ready)":
- timeout: 300
-
- # Stitch together images
- |-
+ # Stitch together screenshots and diff against master
+
+ echo '0. Wait for screenshots to be ready'
+ echo
+ while [ ! -e ~/screenshots_are_ready ]; do sleep 1; done
+ test -z "$(<~/screenshots_are_ready)" || exit 1
+
+ echo '1. Stitch together pieces'
+ echo
for img in $(ls $CIRCLE_ARTIFACTS/imgs/pieces/); do
convert $(ls -1 $CIRCLE_ARTIFACTS/imgs/pieces/$img/*.png | sort -n) -append $CIRCLE_ARTIFACTS/imgs/$img.png
done
- # Download the latest screenshots from master.
- - |-
+ echo '2. Download the latest screenshots from master'
+ echo
+
artifacts_json="$(curl https://circleci.com/api/v1/project/mathquill/mathquill/latest/artifacts?branch=ci.cleanup)"
#when done testing, restore: artifacts_json="$(curl https://circleci.com/api/v1/project/mathquill/mathquill/latest/artifacts?branch=master)"
echo
@@ -205,10 +215,11 @@
echo "$baseline_imgs"
echo
- test -z "$baseline_imgs" || curl $baseline_imgs
+ test -z "$baseline_imgs" && { echo 'No baseline images to download'; exit; }
+ curl $baseline_imgs
- # Generate image diffs.
- - |-
+ echo '3. Generate image diffs'
+ echo
cd $CIRCLE_ARTIFACTS/imgs/
for file in $(ls *.png); do
# if evergreen browser, browser version of previous screenshot may not match,
@@ -218,5 +229,6 @@
compare -metric AE $baseline $file ${file/%.png/_DIFF.png}
done
true # ignore errors like "image widths or heights differ"
+
post:
- killall --wait sc; true # wait for Sauce Connect to close the tunnel; ignore errors since it's just cleanup
