foundation/org.eclipse.persistence.core.test.framework/src/main/java/org/eclipse/persistence/testing/framework/PerformanceRegressionTestCase.java

/*
 * Copyright (c) 1998, 2021 Oracle and/or its affiliates. All rights reserved.
 *
 * This program and the accompanying materials are made available under the
 * terms of the Eclipse Public License v. 2.0 which is available at
 * http://www.eclipse.org/legal/epl-2.0,
 * or the Eclipse Distribution License v. 1.0 which is available at
 * http://www.eclipse.org/org/documents/edl-v10.php.
 *
 * SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause
 */

// Contributors:
//     Oracle - initial API and implementation from Oracle TopLink
package org.eclipse.persistence.testing.framework;

import org.eclipse.persistence.sessions.*;
import org.eclipse.persistence.expressions.*;
import org.eclipse.persistence.queries.*;

/**
 * Used to ensure the performance of an operation or fine grained use case does not
 * regress / become slower than the previous successful run of the test.
 * This accesses the test result database to query the previous test result.
 */
public abstract class PerformanceRegressionTestCase extends PerformanceComparisonTestCase implements PerformanceRegressionTest {

    /** Switch reset to true to reset the baseline comparison. */
    public static boolean reset = false;

    public PerformanceRegressionTestCase() {
        this.allowableDecrease = DEFAULT_ALLOWABLE_DECREASE;
    }

    /**
     * Load the last test result from the test result database.
     * Find only the results run on the same machine and database.
     * Compare the current test run result with the previous results
     * do determine if the test passes or fails.
     */
    @Override
    public void verify() {
        super.verify();
        verify(this);
    }

    /**
     * Load the last test result from the test result database.
     * Find only the results run on the same machine and database.
     * Compare the current test run result with the previous results
     * do determine if the test passes or fails.
     */
    public static void verify(PerformanceRegressionTest test) {
        // Ensures all tests pass to reset baseline,
        // Required when tests or environment change to be slower to avoid failures.
        if (reset) {
            throw new TestWarningException("Reseting baseline.");
        }
        Session session = LoadBuildSystem.getSystem().getSession();

        // Query all previous successful test results for this test on the same machine and database.
        // Find only the baseline version, or last version run different than current version.
        // If you need to compare results against the current version you must change the TopLink version string.
        ReadAllQuery query = new ReadAllQuery(TestResult.class);
        ExpressionBuilder result = new ExpressionBuilder();
        query.setSelectionCriteria(result.get("name").equal(test.getName()).and(result.get("loadBuildSummary").get("machine").equal(LoadBuildSystem.getSummary().getMachine())).and(result.get("loadBuildSummary").get("loginChoice").equal(LoadBuildSystem.getSummary().getLoginChoice())));
        // Allow comparing to a set version through a system property.
        String currentVersion = LoadBuildSystem.getSummary().getToplinkVersion();
        String baselineVersion = null;
        if (System.getProperties().containsKey("toplink.loadbuild.baseline-version")) {
            baselineVersion = System.getProperties().getProperty("toplink.loadbuild.baseline-version");
            // System properties cannot store spaces so need to replace them from \b.
            baselineVersion = baselineVersion.replace('_', ' ');
            ((PerformanceComparisonTestResult)((TestCase)test).getTestResult()).baselineVersion = baselineVersion;
        } else {
            // Compare against the last successful version.
            ReportQuery reportQuery = new ReportQuery(TestResult.class, query.getExpressionBuilder());
            reportQuery.useDistinct();
            reportQuery.returnSingleValue();
            reportQuery.addAttribute("version", result.get("loadBuildSummary").get("toplinkVersion"));
            reportQuery.setSelectionCriteria(query.getSelectionCriteria().and((result.get("outcome").equal(TestResult.PASSED)).or(result.get("outcome").equal(TestResult.WARNING))).and(result.get("loadBuildSummary").get("toplinkVersion").notEqual(currentVersion)));
            reportQuery.addOrdering(result.get("loadBuildSummary").get("timestamp").descending());
            baselineVersion = (String) session.executeQuery(reportQuery);
        }
        query.setSelectionCriteria(query.getSelectionCriteria().and(result.get("loadBuildSummary").get("toplinkVersion").equal(baselineVersion)));
        query.addOrdering(result.get("loadBuildSummary").get("timestamp").descending());
        query.setMaxRows(10);
        query.useCursoredStream(1, 1);
        CursoredStream stream = (CursoredStream)session.executeQuery(query);
        if (!stream.hasMoreElements()) {
            throw new TestWarningException("No previous test result to compare performance with.");
        }
        TestResult lastResult = (TestResult)stream.nextElement();
        double lastCount = lastResult.getTestTime();
        PerformanceComparisonTestResult testResult = (PerformanceComparisonTestResult)((TestCase)test).getTestResult();
        testResult.getBaselineVersionResults().add(Double.valueOf(lastCount));
        // Average last 5 runs.
        int numberOfRuns = 0;
        while (stream.hasMoreElements() && (numberOfRuns < 4)) {
            TestResult nextResult = (TestResult)stream.nextElement();
            testResult.getBaselineVersionResults().add(Double.valueOf(nextResult.getTestTime()));
            numberOfRuns++;
        }
        stream.close();
        double baselineAverage = PerformanceComparisonTestResult.averageResults(testResult.getBaselineVersionResults());
        double currentCount = ((TestCase)test).getTestResult().getTestTime();
        testResult.baselineVersion = lastResult.getLoadBuildSummary().getToplinkVersion();
        testResult.percentageDifferenceLastRun =
                PerformanceComparisonTestResult.percentageDifference(currentCount, lastCount);

        // Query the current version last 5 runs for averaging.
        query = new ReadAllQuery(TestResult.class);
        result = new ExpressionBuilder();
        query.setSelectionCriteria(result.get("name").equal(test.getName()).and(result.get("loadBuildSummary").get("machine").equal(LoadBuildSystem.getSummary().getMachine())).and(result.get("loadBuildSummary").get("loginChoice").equal(LoadBuildSystem.getSummary().getLoginChoice())).and(result.get("loadBuildSummary").get("toplinkVersion").equal(currentVersion)));
        query.addOrdering(result.get("loadBuildSummary").get("timestamp").descending());
        query.useCursoredStream(1, 1);
        stream = (CursoredStream)session.executeQuery(query);
        // Average last 5 runs.
        testResult.getCurrentVersionResults().add(Double.valueOf(currentCount));
        numberOfRuns = 0;
        while (stream.hasMoreElements() && (numberOfRuns < 4)) {
            TestResult nextResult = (TestResult)stream.nextElement();
            testResult.getCurrentVersionResults().add(Double.valueOf(nextResult.getTestTime()));
            numberOfRuns++;
        }
        stream.close();
        double currentAverage = PerformanceComparisonTestResult.averageResults(testResult.getCurrentVersionResults());
        testResult.percentageDifferenceAverage =
                PerformanceComparisonTestResult.percentageDifference(currentAverage, baselineAverage);

        testResult.baselineStandardDeviation =
                PerformanceComparisonTestResult.standardDeviationResults(testResult.getBaselineVersionResults());
        testResult.currentStandardDeviation =
                PerformanceComparisonTestResult.standardDeviationResults(testResult.getCurrentVersionResults());

        if (testResult.percentageDifferenceAverage < test.getAllowableDecrease()) {
            throw new TestErrorException("Test is " + ((long)testResult.percentageDifferenceAverage) +
                                         "% slower than last successful execution.");
        }
    }
}