Merge pull request #650 from mathquill/ci.fix-timeouts

CI: Fix timeouts, logfiles
diff --git a/circle.yml b/circle.yml
index e7ec805..0a162f0 100644
--- a/circle.yml
+++ b/circle.yml
@@ -38,15 +38,23 @@
     # imagemagick is installed to give us access to the
     # `convert` tool to stitch together the screenshots.
     - sudo apt-get update; sudo apt-get install imagemagick
+    - "test $SAUCE_USERNAME && test $SAUCE_ACCESS_KEY
+       # Sauce Labs credentials required. Sign up here: https://saucelabs.com/opensauce/"
     - ? |-
-        mkdir -p ~/sauce-connect
-        cd ~/sauce-connect
-        if [ ! -x sc-*-linux/bin/sc ]
-        then
-          wget https://saucelabs.com/downloads/sc-latest-linux.tar.gz
-          tar -xzf sc-latest-linux.tar.gz
-        fi
-        sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY --readyfile ~/sauce_is_ready
+        {
+          mkdir -p ~/sauce-connect
+          cd ~/sauce-connect
+          if [ -x sc-*-linux/bin/sc ]; then
+            echo Using cached sc-*-linux/bin/sc
+          else
+            time wget https://saucelabs.com/downloads/sc-latest-linux.tar.gz
+            time tar -xzf sc-latest-linux.tar.gz
+          fi
+          # Sauce Connect randomly fails so try twice
+          time sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY --readyfile ~/sauce_is_ready \
+            || time sc-*-linux/bin/sc --user $SAUCE_USERNAME --api-key $SAUCE_ACCESS_KEY --readyfile ~/sauce_is_ready \
+            || echo ERROR > ~/sauce_is_ready
+        } >$CIRCLE_ARTIFACTS/sauce-connect.log 2>&1
       :
         background: true
 
@@ -55,7 +63,7 @@
     # Sauce can connect to Safari on ports 3000, 4000, 7000, and 8000. Edge needs port 7000 or 8000.
     # https://david263a.wordpress.com/2015/04/18/fixing-safari-cant-connect-to-localhost-issue-when-using-sauce-labs-connect-tunnel/
     # https://support.saucelabs.com/customer/portal/questions/14368823-requests-to-localhost-on-microsoft-edge-are-failing-over-sauce-connect
-    - PORT=8000 make server:
+    - PORT=8000 make server >$CIRCLE_ARTIFACTS/make_server.log 2>&1:
         background: true
 
     # CircleCI expects test results to be reported in an JUnit/xUnit-style XML
@@ -65,25 +73,33 @@
     # apparently truncates custom data in their test result reports, so instead
     # we POST to this trivial Node server on localhost:9000 that writes the
     # body of any POST request to $CIRCLE_TEST_REPORTS/mocha/xunit.xml
+    - mkdir -p $CIRCLE_TEST_REPORTS/mocha
     - ? |-
-        mkdir -p $CIRCLE_TEST_REPORTS/mocha
-        node << 'EOF' | tee $CIRCLE_TEST_REPORTS/mocha/xunit.xml
-        require('http').createServer(function(req, res) {
-          res.setHeader('Access-Control-Allow-Origin', '*');
-          req.pipe(process.stdout);
-          req.on('end', res.end.bind(res));
-        })
-        .listen(9000);
+        node << 'EOF' >$CIRCLE_TEST_REPORTS/mocha/xunit.xml \
+                      2>$CIRCLE_ARTIFACTS/mocha-test-report.log
+          require('http').createServer(function(req, res) {
+            res.setHeader('Access-Control-Allow-Origin', '*');
+            req.pipe(process.stdout);
+            req.on('end', res.end.bind(res));
+          })
+          .listen(9000);
+          console.error('listening on http://0.0.0.0:9000/');
         EOF
       :
         background: true
 
     # Wait for tunnel to be ready (`make server` and the trivial Node server
     # are much faster, no need to wait for them)
-    - while [ ! -e ~/sauce_is_ready ]; do sleep 1; done
+    - while [ ! -e ~/sauce_is_ready ]; do sleep 1; done; test "$(<~/sauce_is_ready)" != ERROR
 
     # Start taking screenshots in the background while the unit tests are running
-    - npm install wd && node script/screenshots.js http://localhost:8000/test/visual.html && touch ~/screenshots_are_ready:
+    - ? |-
+        {
+          time { test -d node_modules/wd || npm install wd; }
+          time node script/screenshots.js http://localhost:8000/test/visual.html \
+            && touch ~/screenshots_are_ready || echo ERROR > ~/screenshots_are_ready:
+        } >$CIRCLE_ARTIFACTS/screenshots.log 2>&1
+      :
         background: true
 
     # Run in-browser unit tests, based on:
@@ -129,7 +145,7 @@
       done
 
     # Wait for screenshots to be ready
-    - while [ ! -e ~/screenshots_are_ready ]; do sleep 1; done:
+    - while [ ! -e ~/screenshots_are_ready ]; do sleep 1; done; test "$(<~/screenshots_are_ready)" != ERROR:
         timeout: 300
 
     # Stitch together images
diff --git a/test/unit.html b/test/unit.html
index 04fb3b6..601820c 100644
--- a/test/unit.html
+++ b/test/unit.html
@@ -78,7 +78,7 @@
 
         runner.on('end', function() {
           setTimeout(function() {
-            $.post(post_xunit_to, xunit, function() {
+            $.post(post_xunit_to, xunit).always(function() {
               window.mochaResults = runner.stats;
               window.mochaResults.reports = failedTests;
             });