| /* |
| * Copyright (c) 1998, 2021 Oracle and/or its affiliates. All rights reserved. |
| * |
| * This program and the accompanying materials are made available under the |
| * terms of the Eclipse Public License v. 2.0 which is available at |
| * http://www.eclipse.org/legal/epl-2.0, |
| * or the Eclipse Distribution License v. 1.0 which is available at |
| * http://www.eclipse.org/org/documents/edl-v10.php. |
| * |
| * SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause |
| */ |
| |
| // Contributors: |
| // Oracle - initial API and implementation from Oracle TopLink |
| package org.eclipse.persistence.testing.framework; |
| |
| import org.eclipse.persistence.sessions.*; |
| import org.eclipse.persistence.expressions.*; |
| import org.eclipse.persistence.queries.*; |
| |
| /** |
| * Used to ensure the performance of an operation or fine grained use case does not |
| * regress / become slower than the previous successful run of the test. |
| * This accesses the test result database to query the previous test result. |
| */ |
| public abstract class PerformanceRegressionTestCase extends PerformanceComparisonTestCase implements PerformanceRegressionTest { |
| |
| /** Switch reset to true to reset the baseline comparison. */ |
| public static boolean reset = false; |
| |
| public PerformanceRegressionTestCase() { |
| this.allowableDecrease = DEFAULT_ALLOWABLE_DECREASE; |
| } |
| |
| /** |
| * Load the last test result from the test result database. |
| * Find only the results run on the same machine and database. |
| * Compare the current test run result with the previous results |
| * do determine if the test passes or fails. |
| */ |
| @Override |
| public void verify() { |
| super.verify(); |
| verify(this); |
| } |
| |
| /** |
| * Load the last test result from the test result database. |
| * Find only the results run on the same machine and database. |
| * Compare the current test run result with the previous results |
| * do determine if the test passes or fails. |
| */ |
| public static void verify(PerformanceRegressionTest test) { |
| // Ensures all tests pass to reset baseline, |
| // Required when tests or environment change to be slower to avoid failures. |
| if (reset) { |
| throw new TestWarningException("Reseting baseline."); |
| } |
| Session session = LoadBuildSystem.getSystem().getSession(); |
| |
| // Query all previous successful test results for this test on the same machine and database. |
| // Find only the baseline version, or last version run different than current version. |
| // If you need to compare results against the current version you must change the TopLink version string. |
| ReadAllQuery query = new ReadAllQuery(TestResult.class); |
| ExpressionBuilder result = new ExpressionBuilder(); |
| query.setSelectionCriteria(result.get("name").equal(test.getName()).and(result.get("loadBuildSummary").get("machine").equal(LoadBuildSystem.getSummary().getMachine())).and(result.get("loadBuildSummary").get("loginChoice").equal(LoadBuildSystem.getSummary().getLoginChoice()))); |
| // Allow comparing to a set version through a system property. |
| String currentVersion = LoadBuildSystem.getSummary().getToplinkVersion(); |
| String baselineVersion = null; |
| if (System.getProperties().containsKey("toplink.loadbuild.baseline-version")) { |
| baselineVersion = System.getProperties().getProperty("toplink.loadbuild.baseline-version"); |
| // System properties cannot store spaces so need to replace them from \b. |
| baselineVersion = baselineVersion.replace('_', ' '); |
| ((PerformanceComparisonTestResult)((TestCase)test).getTestResult()).baselineVersion = baselineVersion; |
| } else { |
| // Compare against the last successful version. |
| ReportQuery reportQuery = new ReportQuery(TestResult.class, query.getExpressionBuilder()); |
| reportQuery.useDistinct(); |
| reportQuery.returnSingleValue(); |
| reportQuery.addAttribute("version", result.get("loadBuildSummary").get("toplinkVersion")); |
| reportQuery.setSelectionCriteria(query.getSelectionCriteria().and((result.get("outcome").equal(TestResult.PASSED)).or(result.get("outcome").equal(TestResult.WARNING))).and(result.get("loadBuildSummary").get("toplinkVersion").notEqual(currentVersion))); |
| reportQuery.addOrdering(result.get("loadBuildSummary").get("timestamp").descending()); |
| baselineVersion = (String) session.executeQuery(reportQuery); |
| } |
| query.setSelectionCriteria(query.getSelectionCriteria().and(result.get("loadBuildSummary").get("toplinkVersion").equal(baselineVersion))); |
| query.addOrdering(result.get("loadBuildSummary").get("timestamp").descending()); |
| query.setMaxRows(10); |
| query.useCursoredStream(1, 1); |
| CursoredStream stream = (CursoredStream)session.executeQuery(query); |
| if (!stream.hasMoreElements()) { |
| throw new TestWarningException("No previous test result to compare performance with."); |
| } |
| TestResult lastResult = (TestResult)stream.nextElement(); |
| double lastCount = lastResult.getTestTime(); |
| PerformanceComparisonTestResult testResult = (PerformanceComparisonTestResult)((TestCase)test).getTestResult(); |
| testResult.getBaselineVersionResults().add(Double.valueOf(lastCount)); |
| // Average last 5 runs. |
| int numberOfRuns = 0; |
| while (stream.hasMoreElements() && (numberOfRuns < 4)) { |
| TestResult nextResult = (TestResult)stream.nextElement(); |
| testResult.getBaselineVersionResults().add(Double.valueOf(nextResult.getTestTime())); |
| numberOfRuns++; |
| } |
| stream.close(); |
| double baselineAverage = PerformanceComparisonTestResult.averageResults(testResult.getBaselineVersionResults()); |
| double currentCount = ((TestCase)test).getTestResult().getTestTime(); |
| testResult.baselineVersion = lastResult.getLoadBuildSummary().getToplinkVersion(); |
| testResult.percentageDifferenceLastRun = |
| PerformanceComparisonTestResult.percentageDifference(currentCount, lastCount); |
| |
| // Query the current version last 5 runs for averaging. |
| query = new ReadAllQuery(TestResult.class); |
| result = new ExpressionBuilder(); |
| query.setSelectionCriteria(result.get("name").equal(test.getName()).and(result.get("loadBuildSummary").get("machine").equal(LoadBuildSystem.getSummary().getMachine())).and(result.get("loadBuildSummary").get("loginChoice").equal(LoadBuildSystem.getSummary().getLoginChoice())).and(result.get("loadBuildSummary").get("toplinkVersion").equal(currentVersion))); |
| query.addOrdering(result.get("loadBuildSummary").get("timestamp").descending()); |
| query.useCursoredStream(1, 1); |
| stream = (CursoredStream)session.executeQuery(query); |
| // Average last 5 runs. |
| testResult.getCurrentVersionResults().add(Double.valueOf(currentCount)); |
| numberOfRuns = 0; |
| while (stream.hasMoreElements() && (numberOfRuns < 4)) { |
| TestResult nextResult = (TestResult)stream.nextElement(); |
| testResult.getCurrentVersionResults().add(Double.valueOf(nextResult.getTestTime())); |
| numberOfRuns++; |
| } |
| stream.close(); |
| double currentAverage = PerformanceComparisonTestResult.averageResults(testResult.getCurrentVersionResults()); |
| testResult.percentageDifferenceAverage = |
| PerformanceComparisonTestResult.percentageDifference(currentAverage, baselineAverage); |
| |
| testResult.baselineStandardDeviation = |
| PerformanceComparisonTestResult.standardDeviationResults(testResult.getBaselineVersionResults()); |
| testResult.currentStandardDeviation = |
| PerformanceComparisonTestResult.standardDeviationResults(testResult.getCurrentVersionResults()); |
| |
| if (testResult.percentageDifferenceAverage < test.getAllowableDecrease()) { |
| throw new TestErrorException("Test is " + ((long)testResult.percentageDifferenceAverage) + |
| "% slower than last successful execution."); |
| } |
| } |
| } |