etc/ci/performance/test_runner.py

#!/usr/bin/env python3

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

import runner
import pytest


def test_log_parser():
    mock_url = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
    mock_log = b"""
[PERF] perf block start
[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/index.html
[PERF],navigationStart,1460358376
[PERF],unloadEventStart,undefined
[PERF],unloadEventEnd,undefined
[PERF],redirectStart,undefined
[PERF],redirectEnd,undefined
[PERF],fetchStart,undefined
[PERF],domainLookupStart,undefined
[PERF],domainLookupEnd,undefined
[PERF],connectStart,undefined
[PERF],connectEnd,undefined
[PERF],secureConnectionStart,undefined
[PERF],requestStart,undefined
[PERF],responseStart,undefined
[PERF],responseEnd,undefined
[PERF],domLoading,1460358376000
[PERF],domInteractive,1460358388000
[PERF],domContentLoadedEventStart,1460358388000
[PERF],domContentLoadedEventEnd,1460358388000
[PERF],domComplete,1460358389000
[PERF],loadEventStart,undefined
[PERF],loadEventEnd,undefined
[PERF] perf block end
Shutting down the Constellation after generating an output file or exit flag specified
"""

    expected = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "navigationStart": 1460358376,
            "unloadEventStart": None,
            "unloadEventEnd": None,
            "redirectStart": None,
            "redirectEnd": None,
            "fetchStart": None,
            "domainLookupStart": None,
            "domainLookupEnd": None,
            "connectStart": None,
            "connectEnd": None,
            "secureConnectionStart": None,
            "requestStart": None,
            "responseStart": None,
            "responseEnd": None,
            "domLoading": 1460358376000,
            "domInteractive": 1460358388000,
            "domContentLoadedEventStart": 1460358388000,
            "domContentLoadedEventEnd": 1460358388000,
            "domComplete": 1460358389000,
            "loadEventStart": None,
            "loadEventEnd": None,
        }
    ]
    result = runner.parse_log(mock_log, mock_url)
    assert expected == list(result)


def test_log_parser_complex():
    mock_log = b"""
[PERF] perf block start
[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/content.html
[PERF],navigationStart,1460358300
[PERF],unloadEventStart,undefined
[PERF],unloadEventEnd,undefined
[PERF],redirectStart,undefined
[PERF],redirectEnd,undefined
[PERF],fetchStart,undefined
[PERF],domainLookupStart,undefined
[PERF],domainLookupEnd,undefined
[PERF],connectStart,undefined
[PERF],connectEnd,undefined
[PERF],secureConnectionStart,undefined
[PERF],requestStart,undefined
[PERF],responseStart,undefined
[PERF],responseEnd,undefined
[PERF],domLoading,1460358376000
[PERF],domInteractive,1460358388000
[PERF],domContentLoadedEventStart,1460358388000
[PERF],domContentLoadedEventEnd,1460358388000
[PERF],domComplete,1460358389000
[PERF],loadEventStart,undefined
[PERF],loadEventEnd,undefined
[PERF] perf block end
Some other js error logs here

[PERF] perf block start
[PERF],testcase,http://localhost:8000/page_load_test/56.com/www.56.com/index.html
[PERF],navigationStart,1460358376
[PERF],unloadEventStart,undefined
[PERF],unloadEventEnd,undefined
[PERF],redirectStart,undefined
[PERF],redirectEnd,undefined
[PERF],fetchStart,undefined
[PERF],domainLookupStart,undefined
[PERF],domainLookupEnd,undefined
[PERF],connectStart,undefined
[PERF],connectEnd,undefined
[PERF],secureConnectionStart,undefined
[PERF],requestStart,undefined
[PERF],responseStart,undefined
[PERF],responseEnd,undefined
[PERF],domLoading,1460358376000
[PERF],domInteractive,1460358388000
[PERF],domContentLoadedEventStart,1460358388000
[PERF],domContentLoadedEventEnd,1460358388000
[PERF],domComplete,1460358389000
[PERF],loadEventStart,undefined
[PERF],loadEventEnd,undefined
[PERF] perf block end
Shutting down the Constellation after generating an output file or exit flag specified
"""
    mock_url = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
    expected = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "navigationStart": 1460358376,
            "unloadEventStart": None,
            "unloadEventEnd": None,
            "redirectStart": None,
            "redirectEnd": None,
            "fetchStart": None,
            "domainLookupStart": None,
            "domainLookupEnd": None,
            "connectStart": None,
            "connectEnd": None,
            "secureConnectionStart": None,
            "requestStart": None,
            "responseStart": None,
            "responseEnd": None,
            "domLoading": 1460358376000,
            "domInteractive": 1460358388000,
            "domContentLoadedEventStart": 1460358388000,
            "domContentLoadedEventEnd": 1460358388000,
            "domComplete": 1460358389000,
            "loadEventStart": None,
            "loadEventEnd": None,
        }
    ]
    result = runner.parse_log(mock_log, mock_url)
    assert expected == list(result)


def test_log_parser_empty():
    mock_log = b"""
[PERF] perf block start
[PERF]BROKEN!!!!!!!!!1
[PERF]BROKEN!!!!!!!!!1
[PERF]BROKEN!!!!!!!!!1
[PERF]BROKEN!!!!!!!!!1
[PERF]BROKEN!!!!!!!!!1
[PERF] perf block end
"""
    mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"

    expected = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "title": "",
            "navigationStart": 0,
            "unloadEventStart": -1,
            "unloadEventEnd": -1,
            "redirectStart": -1,
            "redirectEnd": -1,
            "fetchStart": -1,
            "domainLookupStart": -1,
            "domainLookupEnd": -1,
            "connectStart": -1,
            "connectEnd": -1,
            "secureConnectionStart": -1,
            "requestStart": -1,
            "responseStart": -1,
            "responseEnd": -1,
            "domLoading": -1,
            "domInteractive": -1,
            "domContentLoadedEventStart": -1,
            "domContentLoadedEventEnd": -1,
            "domComplete": -1,
            "loadEventStart": -1,
            "loadEventEnd": -1,
        }
    ]
    result = runner.parse_log(mock_log, mock_testcase)
    assert expected == list(result)


def test_log_parser_error():
    mock_log = b"Nothing here! Test failed!"
    mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"

    expected = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "title": "",
            "navigationStart": 0,
            "unloadEventStart": -1,
            "unloadEventEnd": -1,
            "redirectStart": -1,
            "redirectEnd": -1,
            "fetchStart": -1,
            "domainLookupStart": -1,
            "domainLookupEnd": -1,
            "connectStart": -1,
            "connectEnd": -1,
            "secureConnectionStart": -1,
            "requestStart": -1,
            "responseStart": -1,
            "responseEnd": -1,
            "domLoading": -1,
            "domInteractive": -1,
            "domContentLoadedEventStart": -1,
            "domContentLoadedEventEnd": -1,
            "domComplete": -1,
            "loadEventStart": -1,
            "loadEventEnd": -1,
        }
    ]
    result = runner.parse_log(mock_log, mock_testcase)
    assert expected == list(result)


def test_log_parser_bad_testcase_name():
    mock_testcase = "http://localhost:8000/page_load_test/56.com/www.56.com/index.html"
    # Notice the testcase is about:blank, servo crashed
    mock_log = b"""
[PERF] perf block start
[PERF],testcase,about:blank
[PERF],navigationStart,1460358376
[PERF],unloadEventStart,undefined
[PERF],unloadEventEnd,undefined
[PERF],redirectStart,undefined
[PERF],redirectEnd,undefined
[PERF],fetchStart,undefined
[PERF],domainLookupStart,undefined
[PERF],domainLookupEnd,undefined
[PERF],connectStart,undefined
[PERF],connectEnd,undefined
[PERF],secureConnectionStart,undefined
[PERF],requestStart,undefined
[PERF],responseStart,undefined
[PERF],responseEnd,undefined
[PERF],domLoading,1460358376000
[PERF],domInteractive,1460358388000
[PERF],domContentLoadedEventStart,1460358388000
[PERF],domContentLoadedEventEnd,1460358388000
[PERF],domComplete,1460358389000
[PERF],loadEventStart,undefined
[PERF],loadEventEnd,undefined
[PERF] perf block end
Shutting down the Constellation after generating an output file or exit flag specified
"""

    expected = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "title": "",
            "navigationStart": 0,
            "unloadEventStart": -1,
            "unloadEventEnd": -1,
            "redirectStart": -1,
            "redirectEnd": -1,
            "fetchStart": -1,
            "domainLookupStart": -1,
            "domainLookupEnd": -1,
            "connectStart": -1,
            "connectEnd": -1,
            "secureConnectionStart": -1,
            "requestStart": -1,
            "responseStart": -1,
            "responseEnd": -1,
            "domLoading": -1,
            "domInteractive": -1,
            "domContentLoadedEventStart": -1,
            "domContentLoadedEventEnd": -1,
            "domComplete": -1,
            "loadEventStart": -1,
            "loadEventEnd": -1,
        }
    ]
    result = runner.parse_log(mock_log, mock_testcase)
    assert expected == list(result)


def test_manifest_loader():
    text = """
http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html
http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html

http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html
# Disabled! http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html
"""
    expected = [
        ("http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html", False),
        ("http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html", False),
        ("http://localhost/page_load_test/tp5n/aljazeera.net/aljazeera.net/portal.html", False),
    ]
    assert expected == list(runner.parse_manifest(text))


def test_manifest_loader_async():
    text = """
http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html
async http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html
"""
    expected = [
        ("http://localhost/page_load_test/tp5n/163.com/www.163.com/index.html", False),
        ("http://localhost/page_load_test/tp5n/56.com/www.56.com/index.html", True),
    ]
    assert expected == list(runner.parse_manifest(text))


def test_filter_result_by_manifest():
    input_json = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/content.html",
            "domComplete": 1460358389000,
        },
        {
            "testcase": "non-existing-html",
            "domComplete": 1460358389000,
        },
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "domComplete": 1460358389000,
        },
    ]

    expected = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "domComplete": 1460358389000,
        }
    ]

    manifest = [("http://localhost:8000/page_load_test/56.com/www.56.com/index.html", False)]

    assert expected == runner.filter_result_by_manifest(input_json, manifest)


def test_filter_result_by_manifest_error():
    input_json = [
        {
            "testcase": "1.html",
            "domComplete": 1460358389000,
        }
    ]

    manifest = [("1.html", False), ("2.html", False)]

    with pytest.raises(Exception) as execinfo:
        runner.filter_result_by_manifest(input_json, manifest)
    assert "Missing test result" in str(execinfo.value)


def test_take_result_median_odd():
    input_json = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "domComplete": 1460358389001,
            "domLoading": 1460358380002,
        },
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "domComplete": 1460358389002,
            "domLoading": 1460358380001,
        },
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "domComplete": 1460358389003,
            "domLoading": 1460358380003,
        },
    ]

    expected = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "domComplete": 1460358389002,
            "domLoading": 1460358380002,
        }
    ]

    assert expected == runner.take_result_median(input_json, len(input_json))


def test_take_result_median_even():
    input_json = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "domComplete": 1460358389001,
            "domLoading": 1460358380002,
        },
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "domComplete": 1460358389002,
            "domLoading": 1460358380001,
        },
    ]

    expected = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "domComplete": 1460358389001.5,
            "domLoading": 1460358380001.5,
        }
    ]

    assert expected == runner.take_result_median(input_json, len(input_json))


def test_take_result_median_error():
    input_json = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "domComplete": None,
            "domLoading": 1460358380002,
        },
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "domComplete": 1460358389002,
            "domLoading": 1460358380001,
        },
    ]

    expected = [
        {
            "testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html",
            "domComplete": 1460358389002,
            "domLoading": 1460358380001.5,
        }
    ]

    assert expected == runner.take_result_median(input_json, len(input_json))


def test_log_result():
    results = [
        {"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", "domComplete": -1},
        {"testcase": "http://localhost:8000/page_load_test/56.com/www.56.com/index.html", "domComplete": -1},
        {"testcase": "http://localhost:8000/page_load_test/104.com/www.104.com/index.html", "domComplete": 123456789},
    ]

    expected = """
========================================
Total 3 tests; 1 succeeded, 2 failed.

Failure summary:
 - http://localhost:8000/page_load_test/56.com/www.56.com/index.html
========================================
"""
    assert expected == runner.format_result_summary(results)