From 8e34202abf92a70c444d8abba935f6f2a5df8eee Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 11:59:07 +0200 Subject: [PATCH 01/10] test: run the existing browser CI groups on Endform --- .github/workflows/playwright.yml | 133 +++--------------- .gitignore | 1 + endform.config.ts | 1 + package.json | 4 + playwright.endform.config.js | 21 +++ tests/ui-testing/endform-global-setup.js | 63 +++++++++ tests/ui-testing/endform.config.ts | 39 +++++ tests/ui-testing/package-lock.json | 99 ++++++++++++- tests/ui-testing/package.json | 3 +- .../playwright-tests/utils/global-setup.js | 3 +- tests/ui-testing/playwright.config.js | 17 ++- 11 files changed, 265 insertions(+), 119 deletions(-) create mode 100644 endform.config.ts create mode 100644 package.json create mode 100644 playwright.endform.config.js create mode 100644 tests/ui-testing/endform-global-setup.js create mode 100644 tests/ui-testing/endform.config.ts diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 85f0f4f345e..2496df030a1 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -363,6 +363,7 @@ jobs: permissions: contents: read actions: read + id-token: write # container: # image: mcr.microsoft.com/playwright:v1.50.0-jammy # options: --user root @@ -497,122 +498,30 @@ jobs: done echo "::warning::DL relay did not start; the fan-out case will skip" - - name: Cache Playwright browser - id: playwright-cache - uses: actions/cache@v6 - with: - path: ~/.cache/ms-playwright - key: playwright-chromium-${{ runner.os }}-${{ hashFiles('tests/ui-testing/package-lock.json') }} + - name: Install UI test dependencies + working-directory: tests/ui-testing + run: npm ci - - name: Install dependencies and run ui-tests - run: | - touch .env - echo "ZO_ROOT_USER_EMAIL=${ZO_ROOT_USER_EMAIL}" >> .env - echo "ZO_ROOT_USER_PASSWORD=${ZO_ROOT_USER_PASSWORD}" >> .env - echo "ZO_BASE_URL=${ZO_BASE_URL}" >> .env - echo "WS_ZO_BASE_URL=${WS_ZO_BASE_URL}" >> .env - echo "ZO_BASE_URL_SC=${ZO_BASE_URL_SC}" >> .env - echo "ZO_BASE_URL_SC_UI=${ZO_BASE_URL_SC_UI}" >> .env - echo "INGESTION_URL=${INGESTION_URL}" >> .env - echo "ORGNAME=${ORGNAME}" >> .env - echo "ZO_SMTP_ENABLED=${ZO_SMTP_ENABLED}" >> .env - echo "MAILPIT_URL=${MAILPIT_URL}" >> .env - echo "DL_ADDRESS=${DL_ADDRESS}" >> .env - echo "DL_MEMBERS=${DL_MEMBERS}" >> .env - mv .env tests/ui-testing - cd tests/ui-testing && npm ci - - # Install apt system dependencies (always needed). - # azure.archive.ubuntu.com intermittently times out fetching font/lib - # packages, which makes `install-deps` exit 100 and fails the whole job. - # Give apt its own per-fetch retries and wrap the call in a backoff loop. - echo 'Acquire::Retries "5";' | sudo tee /etc/apt/apt.conf.d/99retries >/dev/null - for attempt in 1 2 3; do - if npx playwright install-deps chromium; then - echo "install-deps succeeded on attempt ${attempt}" - break - fi - if [ "${attempt}" = 3 ]; then - echo "::error::playwright install-deps chromium failed after 3 attempts" - exit 1 - fi - echo "install-deps attempt ${attempt} failed; retrying in $((attempt * 20))s" - sleep $((attempt * 20)) - done + - name: Link the suite dependencies + run: ln -s tests/ui-testing/node_modules node_modules - # Get revision from playwright's own metadata - CHROMIUM_REV=$(node -pe " - require('./node_modules/playwright-core/browsers.json') - .browsers.find(b => b.name === 'chromium').revision - ") - - install_browser() { - local NAME="$1" - local BINARY="$2" - local ZIP_PATH="$3" - local DIR="$HOME/.cache/ms-playwright/${NAME}-${CHROMIUM_REV}" - if [ -f "${DIR}/INSTALLATION_COMPLETE" ]; then - echo "${NAME} already installed (cache hit)"; return 0 - fi - rm -rf "${DIR}" - echo "Cache miss — downloading ${NAME} r${CHROMIUM_REV} via curl + system unzip" - curl -fL --progress-bar \ - "https://cdn.playwright.dev/dbazure/download/playwright/builds/${ZIP_PATH}" \ - -o "/tmp/${NAME}.zip" - mkdir -p "${DIR}" - unzip -q "/tmp/${NAME}.zip" -d "${DIR}" - rm -f "/tmp/${NAME}.zip" - find "${DIR}" -name "${BINARY}" -type f -exec chmod 755 {} \; - find "${DIR}" -name 'chrome_sandbox' -type f -exec chmod 4755 {} \; 2>/dev/null || true - touch "${DIR}/INSTALLATION_COMPLETE" - echo "${NAME} installed: $(ls ${DIR}/)" - } - - install_browser "chromium" "chrome" \ - "chromium/${CHROMIUM_REV}/chromium-linux.zip" - install_browser "chromium_headless_shell" "headless_shell" \ - "chromium/${CHROMIUM_REV}/chromium-headless-shell-linux.zip" - - # ffmpeg is required for video recording (video: retain-on-failure in playwright.config.js) - # The custom install_browser uses CHROMIUM_REV which doesn't match ffmpeg's own revision, - # so use the local playwright binary (pinned via package-lock.json) to install ffmpeg only. - ./node_modules/.bin/playwright install ffmpeg - - # Get list of files to run using join function + - name: Run UI tests on Endform + env: + ENDFORM_TEST_CONCURRENCY: ${{ matrix.workers || 5 }} + run: | FILE_LIST="${{ join(matrix.run_files, ' ') }}" - echo "DEBUG: FILE_LIST = $FILE_LIST" - echo "DEBUG: matrix.testfolder = ${{ matrix.testfolder }}" - - if [ -n "$FILE_LIST" ]; then - # actual_folder (the real directory under playwright-tests/) comes straight - # from the matrix now — the logical->directory mapping lives in ci-matrix/ci_matrix.json. - ACTUAL_FOLDER="${{ matrix.actual_folder }}" - echo "DEBUG: ACTUAL_FOLDER = $ACTUAL_FOLDER" - - # Build file paths - FILE_PATHS="" - for file in $FILE_LIST; do - FILE_PATH="./playwright-tests/$ACTUAL_FOLDER/$file" - echo "DEBUG: Will run file: $FILE_PATH" - FILE_PATHS="$FILE_PATHS $FILE_PATH" - done - - # Optional per-shard worker pin. Empty keeps playwright.config.js's - # default; a shard whose specs contend for a shared server-side - # resource sets "workers": 1 in ci_matrix.json, because fullyParallel - # races separate spec FILES and mode:'serial' only orders tests - # within one file. - WORKERS_ARG="" - if [ -n "${{ matrix.workers }}" ]; then - WORKERS_ARG="--workers=${{ matrix.workers }}" - echo "DEBUG: pinning workers to ${{ matrix.workers }}" - fi - - echo "DEBUG: Final command: npx playwright test $FILE_PATHS $WORKERS_ARG" - npx playwright test $FILE_PATHS $WORKERS_ARG - else - echo "No files specified to run for ${{ matrix.testfolder }} folder" + ACTUAL_FOLDER="${{ matrix.actual_folder }}" + FILE_PATHS="" + for file in $FILE_LIST; do + FILE_PATHS="$FILE_PATHS ./tests/ui-testing/playwright-tests/$ACTUAL_FOLDER/$file" + done + if [ -z "$FILE_PATHS" ]; then + echo "::error::No files specified for ${{ matrix.testfolder }}" + exit 1 fi + npx endform@latest --version + echo "TEST_STAGE_STARTED_AT=$(date -u +%FT%TZ)" + npx endform@latest test --organization-id 2G1ZCj7X --config=playwright.endform.config.js $FILE_PATHS - name: Upload blob report to GitHub Actions Artifacts if: ${{ !cancelled() }} diff --git a/.gitignore b/.gitignore index 9087b3b9108..5a57bb9d2d0 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ # ---------- node / web ---------- node_modules/ +/node_modules .vite/ .vitest/ /web/dist diff --git a/endform.config.ts b/endform.config.ts new file mode 100644 index 00000000000..b9e5c906d5b --- /dev/null +++ b/endform.config.ts @@ -0,0 +1 @@ +export { default } from './tests/ui-testing/endform.config'; diff --git a/package.json b/package.json new file mode 100644 index 00000000000..49d9b6355ba --- /dev/null +++ b/package.json @@ -0,0 +1,4 @@ +{ + "name": "openobserve-browser-ci", + "private": true +} diff --git a/playwright.endform.config.js b/playwright.endform.config.js new file mode 100644 index 00000000000..c294636f5f1 --- /dev/null +++ b/playwright.endform.config.js @@ -0,0 +1,21 @@ +const path = require('path'); +const original = require('./tests/ui-testing/playwright.config.js'); + +const suiteDir = path.join(__dirname, 'tests/ui-testing'); + +module.exports = { + ...original, + testDir: path.resolve(suiteDir, original.testDir), + outputDir: path.resolve(suiteDir, original.outputDir), + globalSetup: path.resolve(suiteDir, original.globalSetup), + globalTeardown: path.resolve(suiteDir, original.globalTeardown), + reporter: original.reporter.map(([name, options]) => [ + name.startsWith('.') ? path.resolve(suiteDir, name) : name, + options && { + ...options, + ...(options.outputDir && { outputDir: path.resolve(suiteDir, options.outputDir) }), + ...(options.outputFolder && { outputFolder: path.resolve(suiteDir, options.outputFolder) }), + ...(options.outputFile && { outputFile: path.resolve(suiteDir, options.outputFile) }), + }, + ]), +}; diff --git a/tests/ui-testing/endform-global-setup.js b/tests/ui-testing/endform-global-setup.js new file mode 100644 index 00000000000..07c303dcc6e --- /dev/null +++ b/tests/ui-testing/endform-global-setup.js @@ -0,0 +1,63 @@ +const fs = require('fs'); +const path = require('path'); +const { request } = require('@playwright/test'); +const { performGlobalIngestion } = require('./playwright-tests/utils/global-setup.js'); +const { ingestTraces } = require('./playwright-tests/utils/trace-ingestion.js'); +const { ingestRumErrors } = require('./playwright-tests/utils/rum-error-ingestion.js'); + +async function globalSetup() { + const baseURL = process.env.ZO_BASE_URL; + const email = process.env.ZO_ROOT_USER_EMAIL; + const password = process.env.ZO_ROOT_USER_PASSWORD; + if (!baseURL || !email || !password) { + throw new Error('ZO_BASE_URL, ZO_ROOT_USER_EMAIL and ZO_ROOT_USER_PASSWORD are required'); + } + + const context = await request.newContext(); + try { + const response = await context.post(`${baseURL}/auth/login`, { + data: { name: email, password }, + }); + if (!response.ok()) { + throw new Error(`Global authentication failed with HTTP ${response.status()}`); + } + const login = await response.json(); + if (login.status !== true) { + throw new Error('Global authentication was rejected'); + } + + const now = Math.floor(Date.now() / 1000); + const userInfo = { + given_name: email, + auth_time: now, + name: email, + exp: now + 30 * 24 * 60 * 60, + family_name: '', + email, + role: login.role, + }; + const state = await context.storageState(); + state.origins = [{ + origin: new URL(baseURL).origin, + localStorage: [ + { name: 'userInfo', value: Buffer.from(JSON.stringify(userInfo)).toString('base64') }, + { name: 'currentuser', value: JSON.stringify(JSON.stringify(userInfo)) }, + ], + }]; + const authFile = path.join(__dirname, 'playwright-tests/utils/auth/user.json'); + fs.mkdirSync(path.dirname(authFile), { recursive: true }); + fs.writeFileSync(authFile, JSON.stringify(state), { mode: 0o600 }); + + const cleanupOnly = process.argv.some(arg => /cleanup\.spec\.(js|ts)$/.test(arg)); + if (!cleanupOnly && process.env.SKIP_INGESTION !== 'true') { + const page = { request: context }; + await performGlobalIngestion(page); + await ingestTraces(page, 20); + await ingestRumErrors(page, 3); + } + } finally { + await context.dispose(); + } +} + +module.exports = globalSetup; diff --git a/tests/ui-testing/endform.config.ts b/tests/ui-testing/endform.config.ts new file mode 100644 index 00000000000..297a1cd56ca --- /dev/null +++ b/tests/ui-testing/endform.config.ts @@ -0,0 +1,39 @@ +import { resolve } from 'node:path'; +import { defineEndformConfig } from 'endform'; + +export default defineEndformConfig({ + proxyNetworkHosts: [''], + concurrentTestLimits: [{ + scope: 'within-suite-run', + limit: Number(process.env.ENDFORM_TEST_CONCURRENCY || 5), + }], + additionalFiles: [ + '../test-data/70_fields.json', + '../test-data/append.csv', + '../test-data/dashboard1-import.json', + '../test-data/dashboard2-import.json', + '../test-data/dashboardAzure.json', + '../test-data/dashboardV3-import.json', + '../test-data/dashboards-import.json', + '../test-data/enrichment_info.csv', + '../test-data/invalid-alert.json', + '../test-data/line.json', + '../test-data/match_all.json', + '../test-data/pictorial.json', + '../test-data/pipelineRealTime.json', + '../test-data/pipelineScheduled.json', + '../test-data/protocols.csv', + '../test-data/regex_patterns_import.json', + '../test-data/sdr_test_data.json', + 'playwright-tests/utils/auth/user.json', + 'utils/td150.json', + 'fixtures/rum/cdn-sample/*.html', + 'fixtures/rum/cdn-sample/*.js', + 'fixtures/rum/cdn-sample/*.css', + 'fixtures/rum/npm-app/package.json', + 'fixtures/rum/npm-app/package-lock.json', + 'fixtures/rum/npm-app/src/main.js', + 'fixtures/sourcemaps/dist/main.e2efix01.js', + 'fixtures/sourcemaps/dist/main.e2efix01.js.map', + ].map(file => resolve(__dirname, file)), +}); diff --git a/tests/ui-testing/package-lock.json b/tests/ui-testing/package-lock.json index 05115daefe3..b233065f3d7 100644 --- a/tests/ui-testing/package-lock.json +++ b/tests/ui-testing/package-lock.json @@ -21,7 +21,8 @@ "devDependencies": { "@playwright/test": "1.55.1", "@types/node": "^20.12.2", - "dotenv": "^17.2.1" + "dotenv": "^17.2.1", + "endform": "0.81.3" } }, "node_modules/@colors/colors": { @@ -325,6 +326,102 @@ "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==", "license": "MIT" }, + "node_modules/endform": { + "version": "0.81.3", + "resolved": "https://registry.npmjs.org/endform/-/endform-0.81.3.tgz", + "integrity": "sha512-VnlmxakSs1u7N1CjOp1hRtCDcBnmNwUH9UALAoxG2LRU3fWpUJ5lNw0s7cARqZTn5OM9F0iJYBvnlaRrOICApw==", + "dev": true, + "license": "SEE LICENSE IN LICENSE.md", + "bin": { + "endform": "bin/endform" + }, + "optionalDependencies": { + "endform-darwin-arm64": "0.81.3", + "endform-darwin-x86": "0.81.3", + "endform-linux-arm64": "0.81.3", + "endform-linux-x86": "0.81.3", + "endform-win32-arm64": "0.81.3", + "endform-win32-x86": "0.81.3" + } + }, + "node_modules/endform-darwin-arm64": { + "version": "0.81.3", + "resolved": "https://registry.npmjs.org/endform-darwin-arm64/-/endform-darwin-arm64-0.81.3.tgz", + "integrity": "sha512-BV3Ji195mR8GnxTbz+FDBS2wPKymW0o/xzxGz4TbWU0IdD4FLLsx4TZBm/VVEvkrnHnxXeJQhAaN2L+m4YTHVw==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/endform-darwin-x86": { + "version": "0.81.3", + "resolved": "https://registry.npmjs.org/endform-darwin-x86/-/endform-darwin-x86-0.81.3.tgz", + "integrity": "sha512-w9/t/7X09tzGN5CQMISWXPYr5axRbqbAwjSTq0MWwZCF+sq25SjbSppllqzfCGAgieHJdmiMft6/RffRzAlmcg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/endform-linux-arm64": { + "version": "0.81.3", + "resolved": "https://registry.npmjs.org/endform-linux-arm64/-/endform-linux-arm64-0.81.3.tgz", + "integrity": "sha512-YR0FMjgx66Y3IwX/8ZvBtooqTKt05GwHjKKuc14raYK+0c3qxArjP/jU/eyjhAc3ybd8YQ9jcTQjszvc8cf9OA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/endform-linux-x86": { + "version": "0.81.3", + "resolved": "https://registry.npmjs.org/endform-linux-x86/-/endform-linux-x86-0.81.3.tgz", + "integrity": "sha512-TVahPFdCvq2dbyO7WtUjcbmHLNT+dXKB1YImEQ7gDXTLpv9AOigpkN8jYmK1SXgNHiwpibM7dGQYULOLw+oEDg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/endform-win32-arm64": { + "version": "0.81.3", + "resolved": "https://registry.npmjs.org/endform-win32-arm64/-/endform-win32-arm64-0.81.3.tgz", + "integrity": "sha512-etfYsOPqesPE1RIxzHKBPy/aICoFcCd1rCkPUw3xRBMZsCwYDHMFC/awrRN8d7Bav98T6ht4jyI8k07tsbVruQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/endform-win32-x86": { + "version": "0.81.3", + "resolved": "https://registry.npmjs.org/endform-win32-x86/-/endform-win32-x86-0.81.3.tgz", + "integrity": "sha512-2Oz2ys88AS5GbuyUvpUq5KdtPIv6I0DxRiwf+mWd/Onh2diPoTGD/cfZyU3GYF3GYtWFXmc7OKbEOeZiVeDRDw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", diff --git a/tests/ui-testing/package.json b/tests/ui-testing/package.json index aab1186989b..926c97e5015 100644 --- a/tests/ui-testing/package.json +++ b/tests/ui-testing/package.json @@ -15,7 +15,8 @@ "devDependencies": { "@playwright/test": "1.55.1", "@types/node": "^20.12.2", - "dotenv": "^17.2.1" + "dotenv": "^17.2.1", + "endform": "0.81.3" }, "dependencies": { "@google-cloud/local-auth": "^2.1.0", diff --git a/tests/ui-testing/playwright-tests/utils/global-setup.js b/tests/ui-testing/playwright-tests/utils/global-setup.js index ef535c8d3f8..f4d7099bcd3 100644 --- a/tests/ui-testing/playwright-tests/utils/global-setup.js +++ b/tests/ui-testing/playwright-tests/utils/global-setup.js @@ -249,4 +249,5 @@ async function performMetricsIngestion() { } } -module.exports = globalSetup; \ No newline at end of file +module.exports = globalSetup; +module.exports.performGlobalIngestion = performGlobalIngestion; diff --git a/tests/ui-testing/playwright.config.js b/tests/ui-testing/playwright.config.js index 1f29d827d06..6875919bbb7 100644 --- a/tests/ui-testing/playwright.config.js +++ b/tests/ui-testing/playwright.config.js @@ -9,6 +9,14 @@ try { console.warn('⚠️ dotenv not available, using system environment variables'); } +const baseURL = process.env.BASE_URL || process.env.ZO_BASE_URL || 'http://localhost:5080'; +process.env.ZO_BASE_URL = baseURL; + +if (process.env.ENDFORM === 'true') { + // Runtime fixture paths depend on this directory even when a parent config collects the suite. + process.chdir(__dirname); +} + // Check if essential environment variables are set if (!process.env.ZO_BASE_URL || !process.env.ZO_ROOT_USER_EMAIL || !process.env.ZO_ROOT_USER_PASSWORD) { console.warn('⚠️ Essential environment variables not found. Make sure to set ZO_BASE_URL, ZO_ROOT_USER_EMAIL, and ZO_ROOT_USER_PASSWORD'); @@ -25,7 +33,9 @@ module.exports = defineConfig({ /* Exclude archived tests from all test runs */ testIgnore: ['**/test-archives/**', '**/*_old.js'], /* Global setup and teardown */ - globalSetup: './playwright-tests/utils/global-setup.js', + globalSetup: process.env.ENDFORM === 'true' + ? './endform-global-setup.js' + : './playwright-tests/utils/global-setup.js', globalTeardown: './playwright-tests/utils/global-teardown.js', /* Run tests in files in parallel */ fullyParallel: true, @@ -50,10 +60,9 @@ module.exports = defineConfig({ /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ - baseURL: process.env["ZO_BASE_URL"], + baseURL, - /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ - trace: 'on-first-retry', + trace: 'retain-on-failure', /* Navigation and action timeouts for CI stability */ navigationTimeout: process.env.CI ? 90000 : 30000, From 9b595dc3b60ff8642783865981cd1433d90d035c Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 12:07:06 +0200 Subject: [PATCH 02/10] test: preserve worker-hosted RUM fixture networking --- .github/workflows/playwright.yml | 1 + tests/ui-testing/endform.config.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 2496df030a1..1f748268073 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -520,6 +520,7 @@ jobs: exit 1 fi npx endform@latest --version + trap 'echo "TEST_STAGE_FINISHED_AT=$(date -u +%FT%TZ)"' EXIT echo "TEST_STAGE_STARTED_AT=$(date -u +%FT%TZ)" npx endform@latest test --organization-id 2G1ZCj7X --config=playwright.endform.config.js $FILE_PATHS diff --git a/tests/ui-testing/endform.config.ts b/tests/ui-testing/endform.config.ts index 297a1cd56ca..84c82ee0917 100644 --- a/tests/ui-testing/endform.config.ts +++ b/tests/ui-testing/endform.config.ts @@ -2,7 +2,8 @@ import { resolve } from 'node:path'; import { defineEndformConfig } from 'endform'; export default defineEndformConfig({ - proxyNetworkHosts: [''], + // Worker-hosted RUM fixture servers must keep their own loopback ports. + proxyNetworkPorts: [5080, 8025], concurrentTestLimits: [{ scope: 'within-suite-run', limit: Number(process.env.ENDFORM_TEST_CONCURRENCY || 5), From 189de0c6f3ab3baf5ecd1f611c0c17598ea4ab54 Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 12:19:07 +0200 Subject: [PATCH 03/10] test: keep each Endform group report in the aggregate --- .github/workflows/playwright.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 1f748268073..ac4993b932d 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -508,6 +508,7 @@ jobs: - name: Run UI tests on Endform env: ENDFORM_TEST_CONCURRENCY: ${{ matrix.workers || 5 }} + PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.testfolder }}.zip run: | FILE_LIST="${{ join(matrix.run_files, ' ') }}" ACTUAL_FOLDER="${{ matrix.actual_folder }}" From 6b01fc3bb84c5f9e32ce2898f1e3496f7c98ad70 Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 14:12:54 +0200 Subject: [PATCH 04/10] test: preserve shared fixtures and callbacks on Endform --- .github/workflows/playwright.yml | 12 +++++ playwright.endform.config.js | 4 ++ tests/ui-testing/endform-global-setup.js | 7 +++ tests/ui-testing/endform.config.ts | 9 +++- .../utils/variable-helpers.js | 36 +++++++++----- .../playwright-tests/utils/webhook-capture.js | 42 ++++++++++++++++ .../scripts/endform-webhook-relay.cjs | 48 +++++++++++++++++++ 7 files changed, 145 insertions(+), 13 deletions(-) create mode 100644 tests/ui-testing/scripts/endform-webhook-relay.cjs diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index ac4993b932d..2e3fd738da3 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -505,9 +505,21 @@ jobs: - name: Link the suite dependencies run: ln -s tests/ui-testing/node_modules node_modules + - name: Start callback capture relay + if: matrix.testfolder == 'Alerts' + run: | + node tests/ui-testing/scripts/endform-webhook-relay.cjs > webhook-relay.log 2>&1 & + for i in $(seq 1 10); do + if curl -sf http://localhost:9000/health > /dev/null; then exit 0; fi + sleep 1 + done + echo "::error::Callback capture relay failed to start" + exit 1 + - name: Run UI tests on Endform env: ENDFORM_TEST_CONCURRENCY: ${{ matrix.workers || 5 }} + ENDFORM_TEST_GROUP: ${{ matrix.testfolder }} PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.testfolder }}.zip run: | FILE_LIST="${{ join(matrix.run_files, ' ') }}" diff --git a/playwright.endform.config.js b/playwright.endform.config.js index c294636f5f1..f57162e0236 100644 --- a/playwright.endform.config.js +++ b/playwright.endform.config.js @@ -5,6 +5,10 @@ const suiteDir = path.join(__dirname, 'tests/ui-testing'); module.exports = { ...original, + projects: original.projects.map(project => ({ + ...project, + ...(['Reports', 'RUM', 'Alerts'].includes(process.env.ENDFORM_TEST_GROUP) && { fullyParallel: false }), + })), testDir: path.resolve(suiteDir, original.testDir), outputDir: path.resolve(suiteDir, original.outputDir), globalSetup: path.resolve(suiteDir, original.globalSetup), diff --git a/tests/ui-testing/endform-global-setup.js b/tests/ui-testing/endform-global-setup.js index 07c303dcc6e..df79fcda3b2 100644 --- a/tests/ui-testing/endform-global-setup.js +++ b/tests/ui-testing/endform-global-setup.js @@ -1,11 +1,18 @@ const fs = require('fs'); const path = require('path'); +const { execFileSync } = require('child_process'); const { request } = require('@playwright/test'); const { performGlobalIngestion } = require('./playwright-tests/utils/global-setup.js'); const { ingestTraces } = require('./playwright-tests/utils/trace-ingestion.js'); const { ingestRumErrors } = require('./playwright-tests/utils/rum-error-ingestion.js'); async function globalSetup() { + if (process.env.ENDFORM_TEST_GROUP === 'RUM') { + const fixtureDir = path.join(__dirname, 'fixtures/rum/npm-app'); + execFileSync('npm', ['ci'], { cwd: fixtureDir, stdio: 'inherit', timeout: 300000 }); + execFileSync('npm', ['run', 'build'], { cwd: fixtureDir, stdio: 'inherit', timeout: 120000 }); + } + const baseURL = process.env.ZO_BASE_URL; const email = process.env.ZO_ROOT_USER_EMAIL; const password = process.env.ZO_ROOT_USER_PASSWORD; diff --git a/tests/ui-testing/endform.config.ts b/tests/ui-testing/endform.config.ts index 84c82ee0917..351d1e477fd 100644 --- a/tests/ui-testing/endform.config.ts +++ b/tests/ui-testing/endform.config.ts @@ -3,7 +3,7 @@ import { defineEndformConfig } from 'endform'; export default defineEndformConfig({ // Worker-hosted RUM fixture servers must keep their own loopback ports. - proxyNetworkPorts: [5080, 8025], + proxyNetworkPorts: [5080, 8025, ...(process.env.ENDFORM_TEST_GROUP === 'Alerts' ? [9000] : [])], concurrentTestLimits: [{ scope: 'within-suite-run', limit: Number(process.env.ENDFORM_TEST_CONCURRENCY || 5), @@ -28,13 +28,20 @@ export default defineEndformConfig({ '../test-data/sdr_test_data.json', 'playwright-tests/utils/auth/user.json', 'utils/td150.json', + 'utils/webhookDestinationImport.json', + 'utils/webhookTemplateImport.json', 'fixtures/rum/cdn-sample/*.html', 'fixtures/rum/cdn-sample/*.js', 'fixtures/rum/cdn-sample/*.css', 'fixtures/rum/npm-app/package.json', 'fixtures/rum/npm-app/package-lock.json', 'fixtures/rum/npm-app/src/main.js', + ...(process.env.ENDFORM_TEST_GROUP === 'RUM' ? [ + 'fixtures/rum/npm-app/dist/bundle.js', + 'fixtures/rum/npm-app/node_modules/@openobserve/browser-rum/package.json', + ] : []), 'fixtures/sourcemaps/dist/main.e2efix01.js', 'fixtures/sourcemaps/dist/main.e2efix01.js.map', + 'fixtures/sourcemaps/manifest.json', ].map(file => resolve(__dirname, file)), }); diff --git a/tests/ui-testing/playwright-tests/utils/variable-helpers.js b/tests/ui-testing/playwright-tests/utils/variable-helpers.js index 614ce146e4e..44222b975ad 100644 --- a/tests/ui-testing/playwright-tests/utils/variable-helpers.js +++ b/tests/ui-testing/playwright-tests/utils/variable-helpers.js @@ -3,7 +3,7 @@ * Provides common methods for variable API monitoring, assertions, and state validation */ -import testLogger from './test-logger.js'; +const testLogger = require('./test-logger.js'); /** * Monitor variable values API calls and track completion @@ -13,7 +13,7 @@ import testLogger from './test-logger.js'; * @param {number} options.timeout - Timeout in milliseconds (default: 15000) * @returns {Promise} - {success, actualCount, calls, timedOut} */ -export async function monitorVariableAPICalls(page, options = {}) { +async function monitorVariableAPICalls(page, options = {}) { const { expectedCount = 1, timeout = 15000, matchFn = null } = options; const startTime = Date.now(); const apiCalls = []; @@ -180,7 +180,7 @@ export async function monitorVariableAPICalls(page, options = {}) { * @param {Object} options - Configuration options * @returns {Promise} - {success, state, error} */ -export async function waitForVariableToLoad(page, variableName, options = {}) { +async function waitForVariableToLoad(page, variableName, options = {}) { const { timeout = 10000 } = options; try { @@ -221,7 +221,7 @@ export async function waitForVariableToLoad(page, variableName, options = {}) { * @param {Object} options - Configuration options * @returns {Promise} */ -export async function verifyVariableAPITriggered(page, action, options = {}) { +async function verifyVariableAPITriggered(page, action, options = {}) { const { timeout = 10000 } = options; let apiCalled = false; @@ -251,7 +251,7 @@ export async function verifyVariableAPITriggered(page, action, options = {}) { * @param {string} level - 'global' or panel ID * @returns {Promise} */ -export async function hasRefreshIndicator(page, level = 'global') { +async function hasRefreshIndicator(page, level = 'global') { try { if (level === 'global') { const refreshBtn = page.locator('[data-test="dashboard-refresh-btn"]'); @@ -275,7 +275,7 @@ export async function hasRefreshIndicator(page, level = 'global') { * @param {string} panelId - Panel ID * @returns {Promise} */ -export async function panelNeedsRefresh(page, panelId) { +async function panelNeedsRefresh(page, panelId) { try { const warningIcon = page.locator(`[data-test="dashboard-panel-refresh-panel-btn"]`); return await warningIcon.isVisible({ timeout: 3000 }); @@ -292,7 +292,7 @@ export async function panelNeedsRefresh(page, panelId) { * @param {number} timeout - Timeout in milliseconds * @returns {Promise} - {reloaded, queryCount} */ -export async function trackPanelReload(page, panelId, action, timeout = 10000) { +async function trackPanelReload(page, panelId, action, timeout = 10000) { const queryCalls = []; const startTime = Date.now(); @@ -341,7 +341,7 @@ function extractPanelIdFromURL(url) { * @param {string} expectedValue - Expected value * @returns {Promise} */ -export async function verifyVariableValuePersists(page, variableName, tabId, expectedValue) { +async function verifyVariableValuePersists(page, variableName, tabId, expectedValue) { try { // Switch to the tab await page.locator(`[data-test="dashboard-tab-${tabId}"]`).click(); @@ -367,7 +367,7 @@ export async function verifyVariableValuePersists(page, variableName, tabId, exp * @param {number} timeout - Timeout in milliseconds * @returns {Promise} - {success, loadOrder, errors} */ -export async function verifyVariableLoadSequence(page, variableNames, timeout = 20000) { +async function verifyVariableLoadSequence(page, variableNames, timeout = 20000) { const loadOrder = []; const errors = []; const startTime = Date.now(); @@ -429,7 +429,7 @@ function extractVariableNameFromURL(url) { * @param {string} scopeId - Tab or panel ID (if applicable) * @returns {Promise} */ -export async function verifyVariablesInURL(page, expectedVariables, scope = 'global', scopeId = null) { +async function verifyVariablesInURL(page, expectedVariables, scope = 'global', scopeId = null) { const currentURL = page.url(); for (const [name, value] of Object.entries(expectedVariables)) { @@ -460,7 +460,7 @@ export async function verifyVariablesInURL(page, expectedVariables, scope = 'glo * @param {number} timeout - Timeout in milliseconds * @returns {Promise} */ -export async function waitForAllPanelsToLoad(page, panelCount, timeout = 30000) { +async function waitForAllPanelsToLoad(page, panelCount, timeout = 30000) { const startTime = Date.now(); try { @@ -487,7 +487,7 @@ export async function waitForAllPanelsToLoad(page, panelCount, timeout = 30000) * @param {Object} expectations - {success: true, minCalls: 1, maxDuration: 10000} * @throws {Error} if expectations are not met */ -export function assertVariableAPILoading(monitorResult, expectations = {}) { +function assertVariableAPILoading(monitorResult, expectations = {}) { const { success = true, minCalls = 1, @@ -513,3 +513,15 @@ export function assertVariableAPILoading(monitorResult, expectations = {}) { testLogger.info(`Variable API assertion passed: ${monitorResult.actualCount} calls in ${monitorResult.totalDuration}ms`); } + +exports.monitorVariableAPICalls = monitorVariableAPICalls; +exports.waitForVariableToLoad = waitForVariableToLoad; +exports.verifyVariableAPITriggered = verifyVariableAPITriggered; +exports.hasRefreshIndicator = hasRefreshIndicator; +exports.panelNeedsRefresh = panelNeedsRefresh; +exports.trackPanelReload = trackPanelReload; +exports.verifyVariableValuePersists = verifyVariableValuePersists; +exports.verifyVariableLoadSequence = verifyVariableLoadSequence; +exports.verifyVariablesInURL = verifyVariablesInURL; +exports.waitForAllPanelsToLoad = waitForAllPanelsToLoad; +exports.assertVariableAPILoading = assertVariableAPILoading; diff --git a/tests/ui-testing/playwright-tests/utils/webhook-capture.js b/tests/ui-testing/playwright-tests/utils/webhook-capture.js index 8f76f6bc466..5ec530141dd 100644 --- a/tests/ui-testing/playwright-tests/utils/webhook-capture.js +++ b/tests/ui-testing/playwright-tests/utils/webhook-capture.js @@ -22,6 +22,35 @@ class WebhookCapture { * @returns {Promise} The port the server is listening on. */ async start() { + if (process.env.ENDFORM === 'true' && process.env.ENDFORM_TEST_GROUP === 'Alerts') { + const { request } = require('@playwright/test'); + this.remoteContext = await request.newContext({ baseURL: 'http://localhost:9000' }); + const response = await this.remoteContext.post('/captures'); + if (!response.ok()) throw new Error(`Callback receiver setup failed: HTTP ${response.status()}`); + const capture = await response.json(); + this.remoteId = capture.id; + this.port = capture.port; + this.pollGeneration = 0; + this.pollError = null; + const poll = async () => { + const generation = this.pollGeneration; + try { + await this.pendingClear; + const response = await this.remoteContext.get(`/captures/${this.remoteId}`); + if (!response.ok()) throw new Error(`Callback receiver polling failed: HTTP ${response.status()}`); + const payloads = await response.json(); + if (generation === this.pollGeneration) { + this.payloads = payloads; + this.pollError = null; + } + } catch (error) { + if (this.remoteId) this.pollError = error; + } + if (this.remoteId) this.pollTimer = setTimeout(poll, 100); + }; + this.pollTimer = setTimeout(poll, 100); + return this.port; + } return new Promise((resolve, reject) => { this.payloads = []; this.server = http.createServer((req, res) => { @@ -63,6 +92,14 @@ class WebhookCapture { * Stop the capture server. */ async stop() { + if (this.remoteId) { + const id = this.remoteId; + this.remoteId = null; + clearTimeout(this.pollTimer); + await this.remoteContext.delete(`/captures/${id}`); + await this.remoteContext.dispose(); + return; + } return new Promise((resolve) => { if (!this.server) return resolve(); this.server.close(() => resolve()); @@ -83,6 +120,7 @@ class WebhookCapture { * @returns {Object|null} */ getLatestPayload() { + if (this.pollError) throw this.pollError; if (this.payloads.length === 0) return null; return this.payloads[this.payloads.length - 1]; } @@ -92,6 +130,10 @@ class WebhookCapture { */ clear() { this.payloads = []; + if (this.remoteId) { + this.pollGeneration += 1; + this.pendingClear = this.remoteContext.delete(`/captures/${this.remoteId}/payloads`); + } } } diff --git a/tests/ui-testing/scripts/endform-webhook-relay.cjs b/tests/ui-testing/scripts/endform-webhook-relay.cjs new file mode 100644 index 00000000000..6c18ef48caa --- /dev/null +++ b/tests/ui-testing/scripts/endform-webhook-relay.cjs @@ -0,0 +1,48 @@ +const http = require('http'); +const { randomUUID } = require('crypto'); + +const captures = new Map(); +const reply = (res, status, body) => { + res.writeHead(status, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify(body)); +}; + +const control = http.createServer(async (req, res) => { + const parts = req.url.split('/').filter(Boolean); + if (req.method === 'GET' && req.url === '/health') return reply(res, 200, { ready: true }); + if (req.method === 'POST' && req.url === '/captures') { + const id = randomUUID(); + const capture = { payloads: [], server: null }; + capture.server = http.createServer((incoming, response) => { + let rawBody = ''; + incoming.on('data', chunk => { rawBody += chunk; }); + incoming.on('end', () => { + let body; + try { body = JSON.parse(rawBody || '{}'); } catch {} + capture.payloads.push({ method: incoming.method, url: incoming.url, headers: incoming.headers, body, rawBody, timestamp: Date.now() }); + reply(response, 200, { received: true }); + }); + }); + capture.server.on('error', () => reply(res, 500, { error: 'Receiver failed to start' })); + capture.server.listen(0, '127.0.0.1', () => { + captures.set(id, capture); + reply(res, 201, { id, port: capture.server.address().port }); + }); + return; + } + const capture = captures.get(parts[1]); + if (!capture) return reply(res, 404, { error: 'Unknown capture' }); + if (req.method === 'GET') return reply(res, 200, capture.payloads); + if (req.method === 'DELETE' && parts[2] === 'payloads') { + capture.payloads = []; + return reply(res, 200, { cleared: true }); + } + if (req.method === 'DELETE') { + await new Promise(resolve => capture.server.close(resolve)); + captures.delete(parts[1]); + return reply(res, 200, { stopped: true }); + } + reply(res, 405, { error: 'Unsupported method' }); +}); + +control.listen(9000, '127.0.0.1'); From 130cccd2fbbe5e954116c87647daff6b015ee16f Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 14:50:40 +0200 Subject: [PATCH 05/10] test: consolidate Endform into one backend job --- .github/workflows/playwright.yml | 114 +++++++----------- .gitignore | 3 + playwright.endform.config.js | 10 +- scripts/endform-selection.cjs | 28 +++++ scripts/run-endform-suite.cjs | 84 +++++++++++++ .../ci-matrix/endform-serial-files.json | 50 ++++++++ tests/ui-testing/endform-global-setup.js | 2 +- tests/ui-testing/endform.config.ts | 6 +- .../playwright-tests/utils/webhook-capture.js | 2 +- 9 files changed, 218 insertions(+), 81 deletions(-) create mode 100644 scripts/endform-selection.cjs create mode 100644 scripts/run-endform-suite.cjs create mode 100644 tests/ui-testing/ci-matrix/endform-serial-files.json diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 2e3fd738da3..fdaae98ea1d 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -350,8 +350,8 @@ jobs: fi ui_integration_tests: - timeout-minutes: 45 - name: e2e / ${{ matrix.testfolder }} + timeout-minutes: 120 + name: e2e / Endform full selection needs: [resolve_binary, build_binary, generate_matrix] if: >- always() && @@ -364,15 +364,6 @@ jobs: contents: read actions: read id-token: write - # container: - # image: mcr.microsoft.com/playwright:v1.50.0-jammy - # options: --user root - strategy: - fail-fast: false - # Matrix is generated from tests/ui-testing/ci-matrix/ci_matrix.json by the - # generate_matrix job (the single source of truth shared with ENT). To add - # or move a spec, edit that JSON file — do NOT hand-edit a matrix here. - matrix: ${{ fromJSON(needs.generate_matrix.outputs.matrix) }} steps: - name: Kill background apt processes run: | @@ -403,17 +394,7 @@ jobs: run-id: ${{ needs.resolve_binary.outputs.run_id }} - name: Start mail sink (Mailpit) - # A local SMTP server the tests can also READ. Runs as a static binary - # rather than a service container: these shards run on a self-hosted EKS - # runner where Docker-in-Docker is not guaranteed, and the app itself is - # already launched the same way. ~2s, and it makes ZO_SMTP_* real. - # - # Gated to the one shard that needs it — every other shard would - # otherwise pay a GitHub-releases download for nothing. - if: matrix.testfolder == 'Alerts' env: - # Pinned, not "latest": an upstream release must never change what this - # shard does without a commit here. MAILPIT_VERSION: v1.21.8 run: | set +e @@ -424,8 +405,6 @@ jobs: echo "::warning::Mailpit ${MAILPIT_VERSION} could not be fetched; email delivery assertions will skip" exit 0 fi - # Loopback only — this runner is shared, and delivered alert bodies - # should not be readable from the rest of its network. ./mailpit --smtp 127.0.0.1:1025 --listen 127.0.0.1:8025 > mailpit.log 2>&1 & for i in $(seq 1 30); do curl -sf http://localhost:8025/api/v1/messages > /dev/null 2>&1 && { echo "Mailpit ready after ${i}s"; exit 0; } @@ -434,29 +413,16 @@ jobs: echo "::warning::Mailpit did not start; email delivery assertions will skip" - name: Start OpenObserve - # Per-shard override of the workflow-level default. ZO_QUICK_MODE_ENABLED is - # read once at server start and is global to the process, so a shard is the - # smallest unit that can change it. Set via "quick_mode_enabled" in - # tests/ui-testing/ci-matrix/ci_matrix.json; shards that omit it get "false". env: - ZO_QUICK_MODE_ENABLED: ${{ matrix.quick_mode_enabled || 'false' }} - # Same per-shard rule: how far back (hours) ingestion accepts a record. - # The workflow default of 5 DROPS older rows while still answering 200, - # so a suite that seeds history (SLO measures a rolling 7-day window) - # would fail as "No data" with nothing in the artifacts to explain it. - # Set via "ingest_allowed_upto" in ci-matrix/ci_matrix.json; shards that - # omit it keep the workflow default. - ZO_INGEST_ALLOWED_UPTO: ${{ matrix.ingest_allowed_upto || env.ZO_INGEST_ALLOWED_UPTO }} - # Same per-shard rule: how much history one SLO backfill chunk covers. - # The production default (86400 = 1 day) makes a 7-day-window test SLO - # backfill in 7 sequential chunks under ZO_SCHEDULER_SLO_BACKFILL_CONCURRENCY=1 - # (SLO-Measurement's shard note above) — no test here asserts on - # per-chunk timing or partial coverage, only on the final measured - # state, so a shard-only larger chunk is free wall-clock, not a - # behavior change. Set via "slo_backfill_chunk_secs" in - # ci-matrix/ci_matrix.json; shards that omit it keep the server default. - ZO_SLO_BACKFILL_CHUNK_SECS: ${{ matrix.slo_backfill_chunk_secs || env.ZO_SLO_BACKFILL_CHUNK_SECS }} - run: chmod +x ./release-ci-binary/openobserve && ./release-ci-binary/openobserve > o2.log 2>&1 & + ZO_QUICK_MODE_ENABLED: "false" + ZO_INGEST_ALLOWED_UPTO: "240" + ZO_SLO_BACKFILL_CHUNK_SECS: "604800" + run: | + chmod +x ./release-ci-binary/openobserve + ./release-ci-binary/openobserve > o2.log 2>&1 & + echo "$!" > openobserve.pid + echo "BACKEND_CPU_COUNT=$(nproc)" + free -m - name: Wait for start run: | @@ -478,14 +444,6 @@ jobs: node-version: 24 - name: Start distribution-list relay - # Mailpit is a sink: it accepts any address but stores ONE message, so it - # can never show what a real distribution list does. This tiny relay sits - # in front of it and expands the alias into per-member deliveries, the - # same way alerts-content-templates.spec.js stands up an HTTP receiver to - # capture what a webhook destination really sent. - # - # Gated to the one shard that needs it, matching the Mailpit step above. - if: matrix.testfolder == 'Alerts' run: | if ! command -v node > /dev/null 2>&1; then echo "::warning::node not on PATH; DL fan-out case will skip" @@ -506,7 +464,6 @@ jobs: run: ln -s tests/ui-testing/node_modules node_modules - name: Start callback capture relay - if: matrix.testfolder == 'Alerts' run: | node tests/ui-testing/scripts/endform-webhook-relay.cjs > webhook-relay.log 2>&1 & for i in $(seq 1 10); do @@ -518,36 +475,47 @@ jobs: - name: Run UI tests on Endform env: - ENDFORM_TEST_CONCURRENCY: ${{ matrix.workers || 5 }} - ENDFORM_TEST_GROUP: ${{ matrix.testfolder }} - PLAYWRIGHT_BLOB_OUTPUT_NAME: report-${{ matrix.testfolder }}.zip + ENDFORM_TEST_CONCURRENCY: "10" + ENDFORM_SELECTION_JSON: ${{ needs.generate_matrix.outputs.matrix }} run: | - FILE_LIST="${{ join(matrix.run_files, ' ') }}" - ACTUAL_FOLDER="${{ matrix.actual_folder }}" - FILE_PATHS="" - for file in $FILE_LIST; do - FILE_PATHS="$FILE_PATHS ./tests/ui-testing/playwright-tests/$ACTUAL_FOLDER/$file" - done - if [ -z "$FILE_PATHS" ]; then - echo "::error::No files specified for ${{ matrix.testfolder }}" - exit 1 - fi npx endform@latest --version - trap 'echo "TEST_STAGE_FINISHED_AT=$(date -u +%FT%TZ)"' EXIT - echo "TEST_STAGE_STARTED_AT=$(date -u +%FT%TZ)" - npx endform@latest test --organization-id 2G1ZCj7X --config=playwright.endform.config.js $FILE_PATHS + ( + while true; do + date -u +%FT%TZ + ps -p "$(cat openobserve.pid)" -o pid=,pcpu=,rss= || true + sleep 5 + done + ) > backend-resource.log & + monitor_pid=$! + trap 'kill "$monitor_pid" 2>/dev/null || true' EXIT + node scripts/run-endform-suite.cjs - name: Upload blob report to GitHub Actions Artifacts if: ${{ !cancelled() }} uses: actions/upload-artifact@v4 with: - name: blob-report-${{ matrix.testfolder }}-attempt-${{ github.run_attempt }} - path: tests/ui-testing/blob-report + name: blob-report-full-attempt-${{ github.run_attempt }} + path: tests/ui-testing/endform-blob-reports + retention-days: 1 + + - name: Upload backend resource samples + if: always() + uses: actions/upload-artifact@v4 + with: + name: endform-backend-resources-attempt-${{ github.run_attempt }} + path: backend-resource.log retention-days: 1 - name: Check OpenObserve logs if: always() - run: cat o2.log + run: | + cat o2.log + if [ -f o2-quick.log ]; then cat o2-quick.log; fi + + - name: Stop OpenObserve + if: always() + run: | + if [ -f openobserve.pid ]; then kill "$(cat openobserve.pid)" || true; fi merge_reports: timeout-minutes: 20 diff --git a/.gitignore b/.gitignore index 5a57bb9d2d0..aa2fcecfdea 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,6 @@ rustc-ice-*.txt /.claude/settings.local.json /.claude/worktrees/ .screenshots/ + +/openobserve.pid +/tests/ui-testing/endform-blob-reports/ diff --git a/playwright.endform.config.js b/playwright.endform.config.js index f57162e0236..f26aa111f8d 100644 --- a/playwright.endform.config.js +++ b/playwright.endform.config.js @@ -1,14 +1,18 @@ const path = require('path'); const original = require('./tests/ui-testing/playwright.config.js'); +const { selection } = require('./scripts/endform-selection.cjs'); const suiteDir = path.join(__dirname, 'tests/ui-testing'); module.exports = { ...original, - projects: original.projects.map(project => ({ + projects: original.projects.flatMap(project => [false, true].map(fileMode => ({ ...project, - ...(['Reports', 'RUM', 'Alerts'].includes(process.env.ENDFORM_TEST_GROUP) && { fullyParallel: false }), - })), + name: `${project.name}-${fileMode ? 'files' : 'tests'}`, + fullyParallel: !fileMode, + testMatch: selection().filter(file => file.fileMode === fileMode).map(file => + new RegExp(`^${file.filename.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`)), + }))), testDir: path.resolve(suiteDir, original.testDir), outputDir: path.resolve(suiteDir, original.outputDir), globalSetup: path.resolve(suiteDir, original.globalSetup), diff --git a/scripts/endform-selection.cjs b/scripts/endform-selection.cjs new file mode 100644 index 00000000000..a87fef8875c --- /dev/null +++ b/scripts/endform-selection.cjs @@ -0,0 +1,28 @@ +const path = require('node:path'); +const serialFiles = new Set(require('../tests/ui-testing/ci-matrix/endform-serial-files.json')); +const manifest = require('../tests/ui-testing/ci-matrix/ci_matrix.json'); + +const suiteDir = path.resolve(__dirname, '../tests/ui-testing/playwright-tests'); +const sharedGroups = new Set(['RUM-Token', 'SLO', 'SLO-Measurement', 'Dashboards-Isolated']); +const fileGroups = new Set(['Reports', 'RUM', 'Alerts']); + +function selection() { + const groups = process.env.ENDFORM_SELECTION_JSON + ? JSON.parse(process.env.ENDFORM_SELECTION_JSON).include + : manifest; + const phase = process.env.ENDFORM_PHASE || 'main'; + return groups.flatMap(group => { + const groupPhase = group.quick_mode_enabled === 'true' + ? 'quick' + : sharedGroups.has(group.testfolder) ? 'shared' : 'main'; + if (groupPhase !== phase) return []; + return group.run_files.map(file => { + const filename = path.resolve(suiteDir, group.actual_folder, file); + const fileMode = phase === 'shared' || fileGroups.has(group.testfolder) + || serialFiles.has(`${group.actual_folder}/${file}`); + return { filename, fileMode, group: group.testfolder }; + }); + }); +} + +module.exports = { selection }; diff --git a/scripts/run-endform-suite.cjs b/scripts/run-endform-suite.cjs new file mode 100644 index 00000000000..77672d3197d --- /dev/null +++ b/scripts/run-endform-suite.cjs @@ -0,0 +1,84 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const { spawn, spawnSync } = require('node:child_process'); +const { selection } = require('./endform-selection.cjs'); + +const root = path.resolve(__dirname, '..'); +const blobDir = path.join(root, 'tests/ui-testing/blob-report'); +const archiveDir = path.join(root, 'tests/ui-testing/endform-blob-reports'); +const pidFile = path.join(root, 'openobserve.pid'); + +async function restartQuickServer() { + const pid = Number(fs.readFileSync(pidFile, 'utf8')); + process.kill(pid, 'SIGTERM'); + for (let i = 0; i < 120; i++) { + try { process.kill(pid, 0); } + catch { break; } + if (i === 119) throw new Error('OpenObserve did not stop before the quick-mode phase'); + await new Promise(resolve => setTimeout(resolve, 500)); + } + const log = fs.openSync(path.join(root, 'o2-quick.log'), 'a'); + const child = spawn('./release-ci-binary/openobserve', [], { + cwd: root, + env: { + ...process.env, + ZO_QUICK_MODE_ENABLED: 'true', + ZO_DATA_DIR: './data/quick/', + ZO_INGEST_ALLOWED_UPTO: '5', + ZO_SLO_BACKFILL_CHUNK_SECS: '86400', + }, + detached: true, + stdio: ['ignore', log, log], + }); + child.unref(); + fs.closeSync(log); + fs.writeFileSync(pidFile, String(child.pid)); + for (let i = 0; i < 120; i++) { + try { + const response = await fetch(`${process.env.ZO_BASE_URL}/web/login`); + if (response.ok()) return; + } catch {} + await new Promise(resolve => setTimeout(resolve, 1000)); + } + throw new Error('Quick-mode OpenObserve failed to start'); +} + +async function main() { + fs.mkdirSync(archiveDir, { recursive: true }); + let failed = false; + console.log(`TEST_STAGE_STARTED_AT=${new Date().toISOString()}`); + try { + for (const phase of ['main', 'shared', 'quick']) { + process.env.ENDFORM_PHASE = phase; + const files = selection(); + if (files.length === 0) continue; + if (phase === 'quick') await restartQuickServer(); + console.log(`ENDFORM_PHASE_STARTED phase=${phase} files=${files.length} at=${new Date().toISOString()}`); + const result = spawnSync('npx', [ + 'endform@latest', 'test', '--organization-id', '2G1ZCj7X', + '--config=playwright.endform.config.js', + ], { + cwd: root, + env: { + ...process.env, + SKIP_INGESTION: phase === 'shared' ? 'true' : 'false', + PLAYWRIGHT_BLOB_OUTPUT_NAME: `report-${phase}.zip`, + }, + stdio: 'inherit', + }); + for (const file of fs.existsSync(blobDir) ? fs.readdirSync(blobDir) : []) { + if (file.endsWith('.zip')) fs.copyFileSync(path.join(blobDir, file), path.join(archiveDir, file)); + } + console.log(`ENDFORM_PHASE_FINISHED phase=${phase} exit=${result.status} at=${new Date().toISOString()}`); + failed ||= result.status !== 0; + } + } finally { + console.log(`TEST_STAGE_FINISHED_AT=${new Date().toISOString()}`); + } + process.exitCode = failed ? 1 : 0; +} + +main().catch(error => { + console.error(error.message); + process.exitCode = 1; +}); diff --git a/tests/ui-testing/ci-matrix/endform-serial-files.json b/tests/ui-testing/ci-matrix/endform-serial-files.json new file mode 100644 index 00000000000..830f4b2fcd2 --- /dev/null +++ b/tests/ui-testing/ci-matrix/endform-serial-files.json @@ -0,0 +1,50 @@ +[ + "Alerts/alerts-metrics-notification.spec.js", + "Alerts/dl-email-destinations.spec.js", + "Alerts/alerts-history.spec.js", + "Dashboards/maxquery.spec.js", + "Dashboards/dashboard-sql-autocomplete.spec.js", + "Dashboards/dashboard-multi-sql.spec.js", + "Dashboards/crossLinkMultiStream.spec.js", + "Dashboards/dashboard-favorites.spec.js", + "Dashboards/dashboard-legends-copy.spec.js", + "Dashboards/dashboard-raw-query.spec.js", + "Dashboards/dashboard-streaming.spec.js", + "Dashboards/dashboard-table-csv-download.spec.js", + "Functions/functions-form-validation.spec.js", + "GeneralTests/landingPage.spec.js", + "GeneralTests/otableSelectCell.spec.js", + "GeneralTests/shared-components-form-validation.spec.js", + "GeneralTests/ai-toolsets-form-validation.spec.js", + "GeneralTests/login-form-validation.spec.js", + "Management/iam-form-validation.spec.js", + "Management/cipher-keys-form-validation.spec.js", + "Management/settings-form-validation.spec.js", + "Management/theme.spec.js", + "Management/edition-features.spec.js", + "Management/onboarding-form-validation.spec.js", + "Logs/unflattened.spec.js", + "Logs/monaco-query-prefill.spec.js", + "Logs/logs-autocomplete-suggestions.spec.js", + "Logs/logs-sql-autocomplete.spec.js", + "Pipelines/scheduled-pipeline-query-builder.spec.js", + "Reports/reportFolders.spec.js", + "Reports/reports-bulk-operations.spec.js", + "Reports/reports-form-validation.spec.js", + "Reports/scheduledReportsDrawer.spec.js", + "RUM/rum-cdn-dataflow.spec.js", + "RUM/rum-npm-dataflow.spec.js", + "RUM/rum-page-dataflow.spec.js", + "RUM/sourcemap-upload-pretty.spec.js", + "RUM/rum-form-validation.spec.js", + "Streams/streams-form-validation.spec.js", + "Traces/tracesSearch.spec.js", + "Traces/traceQueryEditor.spec.js", + "Traces/traceErrorFilter.spec.js", + "Traces/traceDetails.spec.js", + "Traces/traceAdvancedFiltering.spec.js", + "Traces/traces-autocomplete-suggestions.spec.js", + "SLO/slo-alert-sli.spec.js", + "SLO/slo-timeslice.spec.js", + "SLO/slo-alerts.spec.js" +] diff --git a/tests/ui-testing/endform-global-setup.js b/tests/ui-testing/endform-global-setup.js index df79fcda3b2..23eab8c5abb 100644 --- a/tests/ui-testing/endform-global-setup.js +++ b/tests/ui-testing/endform-global-setup.js @@ -7,7 +7,7 @@ const { ingestTraces } = require('./playwright-tests/utils/trace-ingestion.js'); const { ingestRumErrors } = require('./playwright-tests/utils/rum-error-ingestion.js'); async function globalSetup() { - if (process.env.ENDFORM_TEST_GROUP === 'RUM') { + if (process.env.ENDFORM_PHASE === 'main' && !fs.existsSync(path.join(__dirname, 'fixtures/rum/npm-app/dist/bundle.js'))) { const fixtureDir = path.join(__dirname, 'fixtures/rum/npm-app'); execFileSync('npm', ['ci'], { cwd: fixtureDir, stdio: 'inherit', timeout: 300000 }); execFileSync('npm', ['run', 'build'], { cwd: fixtureDir, stdio: 'inherit', timeout: 120000 }); diff --git a/tests/ui-testing/endform.config.ts b/tests/ui-testing/endform.config.ts index 351d1e477fd..839fefb6b14 100644 --- a/tests/ui-testing/endform.config.ts +++ b/tests/ui-testing/endform.config.ts @@ -3,10 +3,10 @@ import { defineEndformConfig } from 'endform'; export default defineEndformConfig({ // Worker-hosted RUM fixture servers must keep their own loopback ports. - proxyNetworkPorts: [5080, 8025, ...(process.env.ENDFORM_TEST_GROUP === 'Alerts' ? [9000] : [])], + proxyNetworkPorts: [5080, 8025, 9000], concurrentTestLimits: [{ scope: 'within-suite-run', - limit: Number(process.env.ENDFORM_TEST_CONCURRENCY || 5), + limit: process.env.ENDFORM_PHASE === 'shared' ? 1 : Number(process.env.ENDFORM_TEST_CONCURRENCY || 10), }], additionalFiles: [ '../test-data/70_fields.json', @@ -36,7 +36,7 @@ export default defineEndformConfig({ 'fixtures/rum/npm-app/package.json', 'fixtures/rum/npm-app/package-lock.json', 'fixtures/rum/npm-app/src/main.js', - ...(process.env.ENDFORM_TEST_GROUP === 'RUM' ? [ + ...(process.env.ENDFORM_PHASE === 'main' ? [ 'fixtures/rum/npm-app/dist/bundle.js', 'fixtures/rum/npm-app/node_modules/@openobserve/browser-rum/package.json', ] : []), diff --git a/tests/ui-testing/playwright-tests/utils/webhook-capture.js b/tests/ui-testing/playwright-tests/utils/webhook-capture.js index 5ec530141dd..7f19533545b 100644 --- a/tests/ui-testing/playwright-tests/utils/webhook-capture.js +++ b/tests/ui-testing/playwright-tests/utils/webhook-capture.js @@ -22,7 +22,7 @@ class WebhookCapture { * @returns {Promise} The port the server is listening on. */ async start() { - if (process.env.ENDFORM === 'true' && process.env.ENDFORM_TEST_GROUP === 'Alerts') { + if (process.env.ENDFORM === 'true') { const { request } = require('@playwright/test'); this.remoteContext = await request.newContext({ baseURL: 'http://localhost:9000' }); const response = await this.remoteContext.post('/captures'); From 18a33b30b90d8fc37a32ecb3d63597cb4deb69bb Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 14:58:48 +0200 Subject: [PATCH 06/10] test: constrain shared resources with Endform native limits --- playwright.endform.config.js | 15 ++++++++------- scripts/endform-selection.cjs | 10 ++++++---- scripts/run-endform-suite.cjs | 3 +-- tests/ui-testing/endform.config.ts | 13 +++++++++---- 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/playwright.endform.config.js b/playwright.endform.config.js index f26aa111f8d..f22769701a7 100644 --- a/playwright.endform.config.js +++ b/playwright.endform.config.js @@ -6,13 +6,14 @@ const suiteDir = path.join(__dirname, 'tests/ui-testing'); module.exports = { ...original, - projects: original.projects.flatMap(project => [false, true].map(fileMode => ({ - ...project, - name: `${project.name}-${fileMode ? 'files' : 'tests'}`, - fullyParallel: !fileMode, - testMatch: selection().filter(file => file.fileMode === fileMode).map(file => - new RegExp(`^${file.filename.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`)), - }))), + projects: original.projects.flatMap(project => + ['tests', 'files', 'rum', 'slo', 'reports', 'alerts', 'preferences'].map(resource => ({ + ...project, + name: `${project.name}-${resource}`, + fullyParallel: resource === 'tests', + testMatch: selection().filter(file => file.resource === resource).map(file => + new RegExp(`^${file.filename.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`)), + }))), testDir: path.resolve(suiteDir, original.testDir), outputDir: path.resolve(suiteDir, original.outputDir), globalSetup: path.resolve(suiteDir, original.globalSetup), diff --git a/scripts/endform-selection.cjs b/scripts/endform-selection.cjs index a87fef8875c..3d6b7bf20d0 100644 --- a/scripts/endform-selection.cjs +++ b/scripts/endform-selection.cjs @@ -3,7 +3,6 @@ const serialFiles = new Set(require('../tests/ui-testing/ci-matrix/endform-seria const manifest = require('../tests/ui-testing/ci-matrix/ci_matrix.json'); const suiteDir = path.resolve(__dirname, '../tests/ui-testing/playwright-tests'); -const sharedGroups = new Set(['RUM-Token', 'SLO', 'SLO-Measurement', 'Dashboards-Isolated']); const fileGroups = new Set(['Reports', 'RUM', 'Alerts']); function selection() { @@ -14,13 +13,16 @@ function selection() { return groups.flatMap(group => { const groupPhase = group.quick_mode_enabled === 'true' ? 'quick' - : sharedGroups.has(group.testfolder) ? 'shared' : 'main'; + : 'main'; if (groupPhase !== phase) return []; return group.run_files.map(file => { const filename = path.resolve(suiteDir, group.actual_folder, file); - const fileMode = phase === 'shared' || fileGroups.has(group.testfolder) + const fileMode = ['RUM', 'SLO'].includes(group.actual_folder) || fileGroups.has(group.testfolder) || serialFiles.has(`${group.actual_folder}/${file}`); - return { filename, fileMode, group: group.testfolder }; + const resource = group.testfolder === 'Dashboards-Isolated' ? 'preferences' + : ['RUM', 'SLO', 'Reports', 'Alerts'].includes(group.actual_folder) + ? group.actual_folder.toLowerCase() : fileMode ? 'files' : 'tests'; + return { filename, fileMode, resource, group: group.testfolder }; }); }); } diff --git a/scripts/run-endform-suite.cjs b/scripts/run-endform-suite.cjs index 77672d3197d..3c5c2ce2ef3 100644 --- a/scripts/run-endform-suite.cjs +++ b/scripts/run-endform-suite.cjs @@ -48,7 +48,7 @@ async function main() { let failed = false; console.log(`TEST_STAGE_STARTED_AT=${new Date().toISOString()}`); try { - for (const phase of ['main', 'shared', 'quick']) { + for (const phase of ['main', 'quick']) { process.env.ENDFORM_PHASE = phase; const files = selection(); if (files.length === 0) continue; @@ -61,7 +61,6 @@ async function main() { cwd: root, env: { ...process.env, - SKIP_INGESTION: phase === 'shared' ? 'true' : 'false', PLAYWRIGHT_BLOB_OUTPUT_NAME: `report-${phase}.zip`, }, stdio: 'inherit', diff --git a/tests/ui-testing/endform.config.ts b/tests/ui-testing/endform.config.ts index 839fefb6b14..235d5c421d5 100644 --- a/tests/ui-testing/endform.config.ts +++ b/tests/ui-testing/endform.config.ts @@ -4,10 +4,15 @@ import { defineEndformConfig } from 'endform'; export default defineEndformConfig({ // Worker-hosted RUM fixture servers must keep their own loopback ports. proxyNetworkPorts: [5080, 8025, 9000], - concurrentTestLimits: [{ - scope: 'within-suite-run', - limit: process.env.ENDFORM_PHASE === 'shared' ? 1 : Number(process.env.ENDFORM_TEST_CONCURRENCY || 10), - }], + concurrentTestLimits: [ + { scope: 'within-suite-run', limit: Number(process.env.ENDFORM_TEST_CONCURRENCY || 10) }, + { scope: 'within-suite-run', label: 'project:chromium-rum', limit: 1 }, + { scope: 'within-suite-run', label: 'project:chromium-slo', limit: 1 }, + { scope: 'within-suite-run', label: 'project:chromium-preferences', limit: 1 }, + { scope: 'within-suite-run', label: 'project:chromium-reports', limit: 2 }, + { scope: 'within-suite-run', label: 'project:chromium-alerts', limit: 3 }, + { scope: 'within-suite-run', label: 'tag:@pipelines', limit: 3 }, + ], additionalFiles: [ '../test-data/70_fields.json', '../test-data/append.csv', From 47bffd0df80b3e30bf2cc7dddff1caa8cf5c7af6 Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 17:46:54 +0200 Subject: [PATCH 07/10] test: raise Endform distribution limit for the full suite --- .github/workflows/playwright.yml | 4 ++-- tests/ui-testing/endform.config.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index fdaae98ea1d..49cb939e1b6 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -350,7 +350,7 @@ jobs: fi ui_integration_tests: - timeout-minutes: 120 + timeout-minutes: 180 name: e2e / Endform full selection needs: [resolve_binary, build_binary, generate_matrix] if: >- @@ -475,7 +475,7 @@ jobs: - name: Run UI tests on Endform env: - ENDFORM_TEST_CONCURRENCY: "10" + ENDFORM_TEST_CONCURRENCY: "40" ENDFORM_SELECTION_JSON: ${{ needs.generate_matrix.outputs.matrix }} run: | npx endform@latest --version diff --git a/tests/ui-testing/endform.config.ts b/tests/ui-testing/endform.config.ts index 235d5c421d5..186d3622b75 100644 --- a/tests/ui-testing/endform.config.ts +++ b/tests/ui-testing/endform.config.ts @@ -5,7 +5,7 @@ export default defineEndformConfig({ // Worker-hosted RUM fixture servers must keep their own loopback ports. proxyNetworkPorts: [5080, 8025, 9000], concurrentTestLimits: [ - { scope: 'within-suite-run', limit: Number(process.env.ENDFORM_TEST_CONCURRENCY || 10) }, + { scope: 'within-suite-run', limit: Number(process.env.ENDFORM_TEST_CONCURRENCY || 40) }, { scope: 'within-suite-run', label: 'project:chromium-rum', limit: 1 }, { scope: 'within-suite-run', label: 'project:chromium-slo', limit: 1 }, { scope: 'within-suite-run', label: 'project:chromium-preferences', limit: 1 }, From 05938e8dfcc3f9b3bfa5dc746bcafa910b0f3873 Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 18:24:30 +0200 Subject: [PATCH 08/10] fix: use Fetch readiness property for Endform quick-mode restart --- scripts/run-endform-suite.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/run-endform-suite.cjs b/scripts/run-endform-suite.cjs index 3c5c2ce2ef3..df0d36b3233 100644 --- a/scripts/run-endform-suite.cjs +++ b/scripts/run-endform-suite.cjs @@ -36,7 +36,7 @@ async function restartQuickServer() { for (let i = 0; i < 120; i++) { try { const response = await fetch(`${process.env.ZO_BASE_URL}/web/login`); - if (response.ok()) return; + if (response.ok) return; } catch {} await new Promise(resolve => setTimeout(resolve, 1000)); } From 5367295438a841bf5aa76a7354a2f24d26e8087c Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 20:33:26 +0200 Subject: [PATCH 09/10] test: co-locate only native serial groups on Endform --- playwright.endform.config.js | 14 +- scripts/endform-selection.cjs | 17 +- scripts/endform-serial-groups-reporter.cjs | 31 ++ scripts/run-endform-suite.cjs | 9 +- .../ci-matrix/endform-serial-files.json | 50 --- .../ci-matrix/endform-serial-groups.json | 320 ++++++++++++++++++ 6 files changed, 382 insertions(+), 59 deletions(-) create mode 100644 scripts/endform-serial-groups-reporter.cjs delete mode 100644 tests/ui-testing/ci-matrix/endform-serial-files.json create mode 100644 tests/ui-testing/ci-matrix/endform-serial-groups.json diff --git a/playwright.endform.config.js b/playwright.endform.config.js index f22769701a7..ccf50b45d40 100644 --- a/playwright.endform.config.js +++ b/playwright.endform.config.js @@ -3,6 +3,14 @@ const original = require('./tests/ui-testing/playwright.config.js'); const { selection } = require('./scripts/endform-selection.cjs'); const suiteDir = path.join(__dirname, 'tests/ui-testing'); +const selected = selection(); +const escapeRegex = value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +const serialFilters = selected.filter(file => file.resource === 'files').flatMap(file => + file.serialTitles.map(titles => { + const relative = path.relative(path.resolve(suiteDir, original.testDir), file.filename).split(path.sep).join('/'); + const title = [relative, ...titles].map(escapeRegex).join(' '); + return new RegExp(`(?:^| )${title}(?: |$)`); + })); module.exports = { ...original, @@ -11,8 +19,10 @@ module.exports = { ...project, name: `${project.name}-${resource}`, fullyParallel: resource === 'tests', - testMatch: selection().filter(file => file.resource === resource).map(file => - new RegExp(`^${file.filename.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`)), + ...(resource === 'tests' && { grepInvert: serialFilters }), + ...(resource === 'files' && { grep: serialFilters }), + testMatch: selected.filter(file => file.resource === resource).map(file => + new RegExp(`^${escapeRegex(file.filename)}$`)), }))), testDir: path.resolve(suiteDir, original.testDir), outputDir: path.resolve(suiteDir, original.outputDir), diff --git a/scripts/endform-selection.cjs b/scripts/endform-selection.cjs index 3d6b7bf20d0..e299fdda182 100644 --- a/scripts/endform-selection.cjs +++ b/scripts/endform-selection.cjs @@ -1,5 +1,5 @@ const path = require('node:path'); -const serialFiles = new Set(require('../tests/ui-testing/ci-matrix/endform-serial-files.json')); +const serialGroups = require('../tests/ui-testing/ci-matrix/endform-serial-groups.json'); const manifest = require('../tests/ui-testing/ci-matrix/ci_matrix.json'); const suiteDir = path.resolve(__dirname, '../tests/ui-testing/playwright-tests'); @@ -15,14 +15,19 @@ function selection() { ? 'quick' : 'main'; if (groupPhase !== phase) return []; - return group.run_files.map(file => { + return group.run_files.flatMap(file => { const filename = path.resolve(suiteDir, group.actual_folder, file); - const fileMode = ['RUM', 'SLO'].includes(group.actual_folder) || fileGroups.has(group.testfolder) - || serialFiles.has(`${group.actual_folder}/${file}`); const resource = group.testfolder === 'Dashboards-Isolated' ? 'preferences' : ['RUM', 'SLO', 'Reports', 'Alerts'].includes(group.actual_folder) - ? group.actual_folder.toLowerCase() : fileMode ? 'files' : 'tests'; - return { filename, fileMode, resource, group: group.testfolder }; + ? group.actual_folder.toLowerCase() : 'tests'; + if (resource !== 'tests' || fileGroups.has(group.testfolder)) { + return [{ filename, fileMode: true, resource, group: group.testfolder }]; + } + const titles = serialGroups[`${group.actual_folder}/${file}`]; + const parallel = { filename, fileMode: false, resource: 'tests', group: group.testfolder }; + if (!titles) return [parallel]; + const serial = { filename, fileMode: true, resource: 'files', serialTitles: titles, group: group.testfolder }; + return titles.some(title => title.length === 0) ? [serial] : [parallel, serial]; }); }); } diff --git a/scripts/endform-serial-groups-reporter.cjs b/scripts/endform-serial-groups-reporter.cjs new file mode 100644 index 00000000000..d959a7a1956 --- /dev/null +++ b/scripts/endform-serial-groups-reporter.cjs @@ -0,0 +1,31 @@ +const fs = require('node:fs'); +const path = require('node:path'); + +const suiteDir = path.resolve(__dirname, '../tests/ui-testing/playwright-tests'); +const output = path.resolve(__dirname, '../tests/ui-testing/ci-matrix/endform-serial-groups.json'); +const modes = new Set(['none', 'default', 'parallel', 'serial']); + +module.exports = class { + onBegin(config, suite) { + const groups = {}; + function visit(current, parents = []) { + if (!modes.has(current._parallelMode)) { + throw new Error('This Playwright version does not expose the expected serial-group mode'); + } + const titles = current.type === 'describe' ? [...parents, current.title] : parents; + if (current._parallelMode === 'serial') { + const file = path.relative(suiteDir, current.location.file).split(path.sep).join('/'); + groups[file] ||= []; + if (!groups[file].some(group => JSON.stringify(group) === JSON.stringify(titles))) { + groups[file].push(titles); + } + } + for (const child of current.suites) visit(child, titles); + } + visit(suite); + const ordered = Object.fromEntries(Object.keys(groups).sort().map(file => [ + file, groups[file].some(group => group.length === 0) ? [[]] : groups[file], + ])); + fs.writeFileSync(output, `${JSON.stringify(ordered, null, 2)}\n`); + } +}; diff --git a/scripts/run-endform-suite.cjs b/scripts/run-endform-suite.cjs index df0d36b3233..b2f66513bd4 100644 --- a/scripts/run-endform-suite.cjs +++ b/scripts/run-endform-suite.cjs @@ -52,8 +52,15 @@ async function main() { process.env.ENDFORM_PHASE = phase; const files = selection(); if (files.length === 0) continue; + if (phase === 'main') { + const inventory = spawnSync('./tests/ui-testing/node_modules/.bin/playwright', [ + 'test', '--config=playwright.endform.config.js', '--list', + '--reporter=./scripts/endform-serial-groups-reporter.cjs', + ], { cwd: root, env: process.env, stdio: 'inherit' }); + if (inventory.status !== 0) throw new Error('Could not inventory the existing serial groups'); + } if (phase === 'quick') await restartQuickServer(); - console.log(`ENDFORM_PHASE_STARTED phase=${phase} files=${files.length} at=${new Date().toISOString()}`); + console.log(`ENDFORM_PHASE_STARTED phase=${phase} files=${new Set(files.map(file => file.filename)).size} at=${new Date().toISOString()}`); const result = spawnSync('npx', [ 'endform@latest', 'test', '--organization-id', '2G1ZCj7X', '--config=playwright.endform.config.js', diff --git a/tests/ui-testing/ci-matrix/endform-serial-files.json b/tests/ui-testing/ci-matrix/endform-serial-files.json deleted file mode 100644 index 830f4b2fcd2..00000000000 --- a/tests/ui-testing/ci-matrix/endform-serial-files.json +++ /dev/null @@ -1,50 +0,0 @@ -[ - "Alerts/alerts-metrics-notification.spec.js", - "Alerts/dl-email-destinations.spec.js", - "Alerts/alerts-history.spec.js", - "Dashboards/maxquery.spec.js", - "Dashboards/dashboard-sql-autocomplete.spec.js", - "Dashboards/dashboard-multi-sql.spec.js", - "Dashboards/crossLinkMultiStream.spec.js", - "Dashboards/dashboard-favorites.spec.js", - "Dashboards/dashboard-legends-copy.spec.js", - "Dashboards/dashboard-raw-query.spec.js", - "Dashboards/dashboard-streaming.spec.js", - "Dashboards/dashboard-table-csv-download.spec.js", - "Functions/functions-form-validation.spec.js", - "GeneralTests/landingPage.spec.js", - "GeneralTests/otableSelectCell.spec.js", - "GeneralTests/shared-components-form-validation.spec.js", - "GeneralTests/ai-toolsets-form-validation.spec.js", - "GeneralTests/login-form-validation.spec.js", - "Management/iam-form-validation.spec.js", - "Management/cipher-keys-form-validation.spec.js", - "Management/settings-form-validation.spec.js", - "Management/theme.spec.js", - "Management/edition-features.spec.js", - "Management/onboarding-form-validation.spec.js", - "Logs/unflattened.spec.js", - "Logs/monaco-query-prefill.spec.js", - "Logs/logs-autocomplete-suggestions.spec.js", - "Logs/logs-sql-autocomplete.spec.js", - "Pipelines/scheduled-pipeline-query-builder.spec.js", - "Reports/reportFolders.spec.js", - "Reports/reports-bulk-operations.spec.js", - "Reports/reports-form-validation.spec.js", - "Reports/scheduledReportsDrawer.spec.js", - "RUM/rum-cdn-dataflow.spec.js", - "RUM/rum-npm-dataflow.spec.js", - "RUM/rum-page-dataflow.spec.js", - "RUM/sourcemap-upload-pretty.spec.js", - "RUM/rum-form-validation.spec.js", - "Streams/streams-form-validation.spec.js", - "Traces/tracesSearch.spec.js", - "Traces/traceQueryEditor.spec.js", - "Traces/traceErrorFilter.spec.js", - "Traces/traceDetails.spec.js", - "Traces/traceAdvancedFiltering.spec.js", - "Traces/traces-autocomplete-suggestions.spec.js", - "SLO/slo-alert-sli.spec.js", - "SLO/slo-timeslice.spec.js", - "SLO/slo-alerts.spec.js" -] diff --git a/tests/ui-testing/ci-matrix/endform-serial-groups.json b/tests/ui-testing/ci-matrix/endform-serial-groups.json new file mode 100644 index 00000000000..38439658ffd --- /dev/null +++ b/tests/ui-testing/ci-matrix/endform-serial-groups.json @@ -0,0 +1,320 @@ +{ + "Alerts/alerts-history.spec.js": [ + [ + "Alert History Page" + ] + ], + "Alerts/alerts-metrics-notification.spec.js": [ + [ + "Metrics Alert Notification Chain" + ] + ], + "Alerts/dl-email-destinations.spec.js": [ + [ + "Email destinations and distribution lists", + "delivery" + ] + ], + "Dashboards/crossLinkMultiStream.spec.js": [ + [ + "Cross-Linking Multi-Stream testcases" + ] + ], + "Dashboards/dashboard-favorites.spec.js": [ + [] + ], + "Dashboards/dashboard-legends-copy.spec.js": [ + [ + "Dashboard Copy Legends and Table Cells" + ] + ], + "Dashboards/dashboard-multi-sql.spec.js": [ + [ + "Multi-SQL Query Support", + "Reset Query on Chart Type Change" + ] + ], + "Dashboards/dashboard-raw-query.spec.js": [ + [ + "Dashboard Raw Query testcases" + ] + ], + "Dashboards/dashboard-sql-autocomplete.spec.js": [ + [ + "Dashboard SQL Autocomplete" + ] + ], + "Dashboards/dashboard-streaming.spec.js": [ + [] + ], + "Dashboards/dashboard-table-csv-download.spec.js": [ + [ + "Dashboard Table Chart — CSV Download" + ], + [ + "Dashboard PromQL Table Chart — CSV Download" + ] + ], + "Dashboards/maxquery.spec.js": [ + [] + ], + "Functions/functions-form-validation.spec.js": [ + [ + "Functions Form Validation" + ], + [ + "Functions Form Validation", + "AddFunction form validation" + ], + [ + "Functions Form Validation", + "FunctionsToolbar transform type selection" + ], + [ + "Functions Form Validation", + "StreamRouting form validation" + ], + [ + "Functions Form Validation", + "AddEnrichmentTable form validation" + ] + ], + "GeneralTests/ai-toolsets-form-validation.spec.js": [ + [ + "AI Toolsets form validation" + ], + [ + "CrossLink dialog form validation" + ] + ], + "GeneralTests/landingPage.spec.js": [ + [ + "Landing Page Test Cases" + ] + ], + "GeneralTests/login-form-validation.spec.js": [ + [ + "Login form validation" + ] + ], + "GeneralTests/otableSelectCell.spec.js": [ + [ + "OTable Select Cell Click Selection testcases" + ] + ], + "GeneralTests/shared-components-form-validation.spec.js": [ + [ + "AddToDashboard form validation" + ], + [ + "TimeRangeEditor form validation" + ], + [ + "TimeRangeEditor Logs Correlation form validation" + ] + ], + "Logs/logs-autocomplete-suggestions.spec.js": [ + [ + "Autocomplete Value Suggestions" + ], + [ + "Autocomplete Value Suggestions - Edge Cases" + ], + [ + "Autocomplete Value Suggestions - Quoting Behavior" + ], + [ + "Autocomplete Value Suggestions - Cold Start & TTL" + ] + ], + "Logs/logs-sql-autocomplete.spec.js": [ + [ + "SQL Autocomplete — Logs" + ], + [ + "SQL Autocomplete — Dashboard" + ], + [ + "SQL Autocomplete — Traces" + ] + ], + "Logs/monaco-query-prefill.spec.js": [ + [ + "Monaco Editor Query Pre-fill Tests" + ] + ], + "Logs/unflattened.spec.js": [ + [] + ], + "Management/cipher-keys-form-validation.spec.js": [ + [ + "Cipher Keys — AddCipherKey empty form validation" + ], + [ + "Cipher Keys — Akeyless credential errors" + ], + [ + "Cipher Keys — AddAkeylessType URL format validation" + ], + [ + "Cipher Keys — AddEncryptionMechanism form validation" + ], + [ + "Cipher Keys — valid OpenObserve key creation" + ] + ], + "Management/edition-features.spec.js": [ + [] + ], + "Management/iam-form-validation.spec.js": [ + [ + "IAM Group form validation" + ], + [ + "IAM Role form validation" + ], + [ + "IAM Service Account name format validation" + ], + [ + "IAM Add User form validation" + ], + [ + "IAM UpdateRole form validation" + ] + ], + "Management/onboarding-form-validation.spec.js": [ + [ + "Onboarding GetStarted form validation" + ], + [ + "AWS Marketplace Setup form validation" + ], + [ + "Azure Marketplace Setup form validation" + ] + ], + "Management/settings-form-validation.spec.js": [ + [ + "Settings OrganizationSettings form validation" + ], + [ + "Settings OrgStorageEditor form validation" + ], + [ + "Settings DomainManagement form validation" + ], + [ + "Settings ModelPricingEditor form validation" + ], + [ + "Correlation Settings SemanticGroupItem display name validation" + ] + ], + "Management/theme.spec.js": [ + [ + "Theme Management Tests" + ] + ], + "Pipelines/scheduled-pipeline-query-builder.spec.js": [ + [] + ], + "RUM/rum-cdn-dataflow.spec.js": [ + [ + "RUM CDN Data Flow" + ] + ], + "RUM/rum-form-validation.spec.js": [ + [ + "RUM Source Maps upload form validation" + ] + ], + "RUM/rum-npm-dataflow.spec.js": [ + [ + "RUM NPM Data Flow" + ] + ], + "RUM/rum-page-dataflow.spec.js": [ + [ + "RUM Page Data Flow" + ] + ], + "RUM/sourcemap-upload-pretty.spec.js": [ + [ + "Sourcemap Upload & Pretty Stack Trace" + ] + ], + "Reports/reportFolders.spec.js": [ + [ + "Report Folders" + ] + ], + "Reports/reports-bulk-operations.spec.js": [ + [ + "Report Bulk Operations" + ] + ], + "Reports/reports-form-validation.spec.js": [ + [ + "Reports form validation — required field errors" + ], + [ + "Reports form validation — absolute timerange with no range blocks save" + ], + [ + "Reports form validation — valid config creates report" + ] + ], + "Reports/scheduledReportsDrawer.spec.js": [ + [ + "Dashboard Scheduled Reports Drawer" + ] + ], + "SLO/slo-alert-sli.spec.js": [ + [] + ], + "SLO/slo-alerts.spec.js": [ + [] + ], + "SLO/slo-timeslice.spec.js": [ + [] + ], + "Streams/streams-form-validation.spec.js": [ + [ + "Streams Add Stream form validation" + ], + [ + "Streams StreamFieldInputs form validation" + ] + ], + "Traces/traceAdvancedFiltering.spec.js": [ + [ + "Trace Advanced Filtering testcases" + ] + ], + "Traces/traceDetails.spec.js": [ + [ + "Trace Details testcases" + ] + ], + "Traces/traceErrorFilter.spec.js": [ + [ + "Trace Error Filter testcases" + ] + ], + "Traces/traceQueryEditor.spec.js": [ + [ + "Trace Query Editor testcases" + ] + ], + "Traces/traces-autocomplete-suggestions.spec.js": [ + [ + "Traces Autocomplete Value Suggestions" + ] + ], + "Traces/tracesSearch.spec.js": [ + [ + "Traces Search testcases" + ] + ] +} From 0361fc5d98f5c9c9818f46fe8ecfe92eb1861e23 Mon Sep 17 00:00:00 2001 From: Oliver Stenbom Date: Thu, 1 Oct 2026 20:55:21 +0200 Subject: [PATCH 10/10] fix: preserve pipeline header target across Playwright runtimes --- tests/ui-testing/pages/pipelinesPages/pipelinesPage.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ui-testing/pages/pipelinesPages/pipelinesPage.js b/tests/ui-testing/pages/pipelinesPages/pipelinesPage.js index 94f8867edf5..bec48a26ac0 100644 --- a/tests/ui-testing/pages/pipelinesPages/pipelinesPage.js +++ b/tests/ui-testing/pages/pipelinesPages/pipelinesPage.js @@ -258,7 +258,7 @@ export class PipelinesPage { this.searchStreamInput = page.getByPlaceholder('Search Stream'); this.exploreButton = page.getByRole('button', { name: 'Explore' }); this.timestampColumnMenu = page.locator('[data-test="o2-table-expand-1"]'); - this.nameCell = page.getByRole('cell', { name: 'Name' }); + this.nameCell = page.locator('[data-test="o2-table-th-name"]'); this.streamIcon = page.getByRole("img", { name: "Stream", exact: true }); this.outputStreamIcon = page.getByRole("img", { name: "Output Stream" }); this.containsOption = page.getByText("Contains", { exact: true });