diff --git a/AGENTS.md b/AGENTS.md new file mode 120000 index 0000000..681311e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1 @@ +CLAUDE.md \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 7236a64..d4a5b74 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -QAS CLI (`qas-cli`) is a Node.js CLI tool for uploading test automation results (JUnit XML / Playwright JSON) to [QA Sphere](https://qasphere.com/). It matches test case markers (e.g., `PRJ-123`) in report files to QA Sphere test cases, creates or reuses test runs, and uploads results with optional attachments. +QAS CLI (`qas-cli`) is a Node.js CLI tool for uploading test automation results (JUnit XML / Playwright JSON / Allure results directories) to [QA Sphere](https://qasphere.com/). It matches test case references (e.g., `PRJ-123` markers or Allure TMS links) to QA Sphere test cases, creates or reuses test runs, and uploads results with optional attachments. ## Commands @@ -29,7 +29,7 @@ Node.js compatibility tests: `cd mnode-test && ./docker-test.sh` (requires Docke ### Entry Point & CLI Framework - `src/bin/qasphere.ts` — Entry point (`#!/usr/bin/env node`). Validates Node version, delegates to `run()`. -- `src/commands/main.ts` — Yargs setup. Registers two commands (`junit-upload`, `playwright-json-upload`) as instances of the same `ResultUploadCommandModule` class. +- `src/commands/main.ts` — Yargs setup. Registers three commands (`junit-upload`, `playwright-json-upload`, `allure-upload`) as instances of the same `ResultUploadCommandModule` class. - `src/commands/resultUpload.ts` — `ResultUploadCommandModule` defines CLI options shared by both commands. Loads env vars, then delegates to `ResultUploadCommandHandler`. ### Core Upload Pipeline (src/utils/result-upload/) @@ -37,12 +37,13 @@ Node.js compatibility tests: `cd mnode-test && ./docker-test.sh` (requires Docke The upload flow has two stages handled by two classes: 1. **`ResultUploadCommandHandler`** — Orchestrates the overall flow: - - Parses report files using the appropriate parser (JUnit XML or Playwright JSON) - - Detects project code from test case names (or from `--run-url`) - - Creates a new test run (or reuses an existing one if title conflicts) - - Delegates actual result uploading to `ResultUploader` -2. **`ResultUploader`** — Handles the upload-to-run mechanics: +- Parses report inputs using the appropriate parser (JUnit XML file, Playwright JSON file, or Allure results directory) +- Detects project code from test case names (or from `--run-url`) +- Creates a new test run (or reuses an existing one if title conflicts) +- Delegates actual result uploading to `ResultUploader` + +1. **`ResultUploader`** — Handles the upload-to-run mechanics: - Fetches test cases from the run, maps parsed results to them via marker matching - Validates unmatched/missing test cases (respects `--force`, `--ignore-unmatched`) - Uploads file attachments concurrently (max 10 parallel), then creates results in batches (max 50 per request) @@ -51,6 +52,7 @@ The upload flow has two stages handled by two classes: - `junitXmlParser.ts` — Parses JUnit XML via `xml2js` + Zod validation. Extracts attachments from `[[ATTACHMENT|path]]` markers in system-out/failure/error/skipped elements. - `playwrightJsonParser.ts` — Parses Playwright JSON report. Supports two test case linking methods: (1) test annotations with `type: "test case"` and URL description, (2) marker in test name. Handles nested suites recursively. +- `allureParser.ts` — Parses Allure 2 JSON results directories (`*-result.json` files only). Supports test case linking via TMS links (`type: "tms"`) or marker in test name, maps Allure statuses to QA Sphere result statuses, and resolves attachments via `attachments[].source`. - `types.ts` — Shared `TestCaseResult` and `Attachment` interfaces used by both parsers. ### API Layer (src/api/) @@ -72,9 +74,10 @@ Composable fetch wrappers using higher-order functions: Tests use **Vitest** with **MSW** (Mock Service Worker) for API mocking. Test files are in `src/tests/`: -- `result-upload.spec.ts` — Integration tests for the full upload flow (both JUnit and Playwright), with MSW intercepting all API calls +- `result-upload.spec.ts` — Integration tests for the full upload flow (JUnit, Playwright, and Allure), with MSW intercepting all API calls - `junit-xml-parsing.spec.ts` — Unit tests for JUnit XML parser - `playwright-json-parsing.spec.ts` — Unit tests for Playwright JSON parser +- `allure-parsing.spec.ts` — Unit tests for Allure parser - `template-string-processing.spec.ts` — Unit tests for run name template processing Test fixtures live in `src/tests/fixtures/` (XML files, JSON files, and mock test case data). diff --git a/README.md b/README.md index efa2dc0..a73282f 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The QAS CLI is a command-line tool for submitting your test automation results to [QA Sphere](https://qasphere.com/). It provides the most efficient way to collect and report test results from your test automation workflow, CI/CD pipeline, and build servers. -The tool can upload test case results from JUnit XML and Playwright JSON files to QA Sphere test runs by matching test case names (mentions of special markers) to QA Sphere's test cases. +The tool can upload test case results from JUnit XML files, Playwright JSON files, and Allure result directories to QA Sphere test runs by matching test case references to QA Sphere test cases. ## Installation @@ -39,7 +39,7 @@ Verify installation: `qasphere --version` The CLI requires the following variables to be defined: - `QAS_TOKEN` - QA Sphere API token (see [docs](https://docs.qasphere.com/api/authentication) if you need help generating one) -- `QAS_URL` - Base URL of your QA Sphere instance (e.g., https://qas.eu2.qasphere.com) +- `QAS_URL` - Base URL of your QA Sphere instance (e.g., `https://qas.eu2.qasphere.com`) These variables could be defined: @@ -59,9 +59,9 @@ QAS_URL=https://qas.eu1.qasphere.com # QAS_URL=https://qas.eu1.qasphere.com ``` -## Commands: `junit-upload`, `playwright-json-upload` +## Commands: `junit-upload`, `playwright-json-upload`, `allure-upload` -The `junit-upload` and `playwright-json-upload` commands upload test results from JUnit XML and Playwright JSON reports to QA Sphere respectively. +The `junit-upload`, `playwright-json-upload`, and `allure-upload` commands upload test results to QA Sphere. There are two modes for uploading results using the commands: @@ -70,6 +70,7 @@ There are two modes for uploading results using the commands: ### Options +- `` / `` - Input paths. Use report files for `junit-upload` and `playwright-json-upload`, and Allure results directories for `allure-upload` - `-r`/`--run-url` - Upload results to an existing test run - `--project-code`, `--run-name`, `--create-tcases` - Create a new test run and upload results to it - `--project-code` - Project code for creating new test run. It can also be auto detected from test case markers in the results, but this is not fully reliable, so it is recommended to specify the project code explicitly @@ -170,14 +171,28 @@ Ensure the required environment variables are defined before running these comma This will exclude stdout from passed tests while still including it for failed, blocked, or skipped tests. 10. Skip both stdout and stderr for passed tests: + ```bash qasphere junit-upload --skip-report-stdout on-success --skip-report-stderr on-success ./test-results.xml ``` + This is useful when you have verbose logging in tests but only want to see output for failures. +11. Upload Allure results from a directory: + + ```bash + qasphere allure-upload -r https://qas.eu1.qasphere.com/project/P1/run/23 ./allure-results + ``` + +12. Continue Allure upload when some `*-result.json` files are malformed (skip invalid files): + + ```bash + qasphere allure-upload --force -r https://qas.eu1.qasphere.com/project/P1/run/23 ./allure-results + ``` + ## Test Report Requirements -The QAS CLI maps test results from your reports (JUnit XML or Playwright JSON) to corresponding test cases in QA Sphere using test case markers. If a test result lacks a valid marker, the CLI will display an error unless you use `--create-tcases` to automatically create test cases, or `--ignore-unmatched`/`--force` to skip unmatched results. +The QAS CLI maps test results from your reports (JUnit XML, Playwright JSON, or Allure) to corresponding test cases in QA Sphere. If a test result lacks a valid marker/reference, the CLI will display an error unless you use `--create-tcases` to automatically create test cases, or `--ignore-unmatched`/`--force` to skip unmatched results. ### JUnit XML @@ -218,6 +233,18 @@ Playwright JSON reports support two methods for referencing test cases (checked 2. **Test Case Marker in Name** - Include the `PROJECT-SEQUENCE` marker in the test name (same format as JUnit XML) +### Allure + +Allure results use one `*-result.json` file per test in a results directory. `allure-upload` matches test cases using: + +1. **TMS links (Recommended)** - `links[]` entries with: + - `type`: `"tms"` + - `url`: QA Sphere test case URL, e.g. `https://qas.eu1.qasphere.com/project/PRJ/tcase/123` +2. **TMS link name fallback** - If `url` is not a QA Sphere URL, a marker in `links[].name` is used (for example `PRJ-123`) +3. **Test case marker in name** - Marker in `name` field (same `PROJECT-SEQUENCE` format as JUnit XML) + +Only Allure 2 JSON (`*-result.json`) is supported. Legacy Allure 1 XML files are ignored. + ## Development (for those who want to contribute to the tool) 1. Install and build: `npm install && npm run build && npm link` diff --git a/package-lock.json b/package-lock.json index 7674e76..36c9776 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "qas-cli", - "version": "0.4.4", + "version": "0.4.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "qas-cli", - "version": "0.4.4", + "version": "0.4.5", "license": "ISC", "dependencies": { "chalk": "^5.4.1", diff --git a/src/commands/main.ts b/src/commands/main.ts index 98ed8a3..05af261 100644 --- a/src/commands/main.ts +++ b/src/commands/main.ts @@ -13,6 +13,7 @@ Required variables: ${qasEnvs.join(', ')} ) .command(new ResultUploadCommandModule('junit-upload')) .command(new ResultUploadCommandModule('playwright-json-upload')) + .command(new ResultUploadCommandModule('allure-upload')) .demandCommand(1, '') .help('h') .alias('h', 'help') diff --git a/src/commands/resultUpload.ts b/src/commands/resultUpload.ts index 78b6687..0a530e8 100644 --- a/src/commands/resultUpload.ts +++ b/src/commands/resultUpload.ts @@ -10,11 +10,19 @@ import { const commandTypeDisplayStrings: Record = { 'junit-upload': 'JUnit XML', 'playwright-json-upload': 'Playwright JSON', + 'allure-upload': 'Allure', } -const commandTypeFileExtensions: Record = { - 'junit-upload': 'xml', - 'playwright-json-upload': 'json', +const commandTypeInputKinds: Record = { + 'junit-upload': 'files', + 'playwright-json-upload': 'files', + 'allure-upload': 'directories', +} + +const commandTypeExampleInputs: Record = { + 'junit-upload': './test-results.xml', + 'playwright-json-upload': './test-results.json', + 'allure-upload': './allure-results', } export class ResultUploadCommandModule implements CommandModule { @@ -25,10 +33,19 @@ export class ResultUploadCommandModule implements CommandModule { + argv.positional('files', { + describe: + this.type === 'allure-upload' + ? 'One or more Allure results directories' + : 'One or more test report files', + type: 'string', + array: true, + }) + argv.options({ 'run-url': { alias: 'r', @@ -83,20 +100,20 @@ export class ResultUploadCommandModule implements CommandModule = {}) => ({ + name: 'Sample test', + status: 'passed', + uuid: 'result-uuid', + start: 1000, + stop: 1200, + ...overrides, +}) + +const createTempAllureDir = async (files: Record) => { + const dir = await mkdtemp(join(tmpdir(), 'qas-allure-fixture-')) + tempDirsToCleanup.push(dir) + + await Promise.all( + Object.entries(files).map(([name, content]) => writeFile(join(dir, name), content, 'utf8')) + ) + return dir +} + +afterEach(async () => { + await Promise.all( + tempDirsToCleanup.splice(0).map((dir) => rm(dir, { recursive: true, force: true })) + ) +}) + +describe('Allure parsing', () => { + test('Should parse matching directory with marker extraction and status mapping', async () => { + const dir = `${allureBasePath}/matching-tcases` + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + allowPartialParse: false, + }) + + expect(testcases).toHaveLength(5) + expect(testcases[0].name).toBe('TEST-002: Test cart') + expect(testcases[1].name).toBe('TEST-003: Test checkout') + expect(testcases[2].name).toBe('TEST-004: About page content TEST-004') + expect(testcases[3].name).toBe('TEST-006: Navigation bar items') + expect(testcases[4].name).toBe('TEST-007: Welcome page content (updated)') + + expect(testcases[0].folder).toBe('ui.cart.spec.ts') + expect(testcases[1].folder).toBe('ui.cart.spec.ts') + expect(testcases[2].folder).toBe('ui.contents.spec.ts') + expect(testcases[3].folder).toBe('ui.contents.spec.ts') + expect(testcases[4].folder).toBe('') + + expect(testcases[0].timeTaken).toBe(500) + expect(testcases[3].status).toBe('failed') + expect(testcases[4].status).toBe('open') + expect(testcases[3].message).toContain('AssertionError: navbar items mismatch') + expect(testcases[3].message).toContain('Traceback line 2') + + testcases.forEach((tcase) => { + expect(tcase.attachments).toHaveLength(1) + expect(tcase.attachments[0].buffer).not.toBeNull() + expect(tcase.attachments[0].error).toBeNull() + }) + }) + + test('Should map broken and skipped statuses correctly', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ name: 'TEST-100 Broken test', status: 'broken' }) + ), + '002-result.json': JSON.stringify( + makeResult({ name: 'TEST-101 Skipped test', status: 'skipped' }) + ), + }) + + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases).toHaveLength(2) + expect(testcases[0].status).toBe('blocked') + expect(testcases[1].status).toBe('skipped') + }) + + test('Should honor skip-report options for message and trace blocks', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-200 Passed with status details', + statusDetails: { + message: 'stdout-like text', + trace: 'stderr-like text', + }, + }) + ), + }) + + const skippedOnSuccess = await parseAllureResults(dir, dir, { + skipStdout: 'on-success', + skipStderr: 'on-success', + }) + expect(skippedOnSuccess[0].message).toBe('') + + const neverSkip = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(neverSkip[0].message).toContain('stdout-like text') + expect(neverSkip[0].message).toContain('stderr-like text') + }) + + test('Should apply folder priority suite > parentSuite > feature > package', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-301 has suite and parentSuite', + labels: [ + { name: 'parentSuite', value: 'parent-folder' }, + { name: 'suite', value: 'suite-folder' }, + ], + }) + ), + '002-result.json': JSON.stringify( + makeResult({ + name: 'TEST-302 has parentSuite and feature', + labels: [ + { name: 'feature', value: 'feature-folder' }, + { name: 'parentSuite', value: 'parent-folder-2' }, + ], + }) + ), + '003-result.json': JSON.stringify( + makeResult({ + name: 'TEST-303 has feature and package', + labels: [ + { name: 'package', value: 'package-folder' }, + { name: 'feature', value: 'feature-folder-2' }, + ], + }) + ), + '004-result.json': JSON.stringify( + makeResult({ + name: 'TEST-304 has package', + labels: [{ name: 'package', value: 'package-only-folder' }], + }) + ), + '005-result.json': JSON.stringify(makeResult({ name: 'TEST-305 has no labels', labels: [] })), + }) + + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases).toHaveLength(5) + expect(testcases[0].folder).toBe('suite-folder') + expect(testcases[1].folder).toBe('parent-folder-2') + expect(testcases[2].folder).toBe('feature-folder-2') + expect(testcases[3].folder).toBe('package-only-folder') + expect(testcases[4].folder).toBe('') + }) + + test('Should keep attachment errors without crashing parse', async () => { + const dir = `${allureBasePath}/missing-attachments` + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + + expect(testcases).toHaveLength(5) + const erroredAttachments = testcases + .flatMap((t) => t.attachments) + .filter((a) => a.error !== null) + expect(erroredAttachments).toHaveLength(1) + expect(erroredAttachments[0].buffer).toBeNull() + expect(erroredAttachments[0].filename).toBe('missing-attachment.txt') + }) + + test('Should fail by default for malformed or schema-invalid files', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-002 Valid result', + }) + ), + '002-result.json': `{ + "name": "Malformed fixture", + "status": "passed", + "uuid": "malformed-uuid", + "start": 1, + "stop": 2,`, + }) + await expect( + parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + allowPartialParse: false, + }) + ).rejects.toThrowError() + }) + + test('Should skip malformed or schema-invalid files when partial parsing is allowed', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-002 Valid result', + }) + ), + '002-result.json': `{ + "name": "Malformed fixture", + "status": "passed", + "uuid": "malformed-uuid", + "start": 1, + "stop": 2,`, + '003-result.json': JSON.stringify({ + name: 'Schema invalid fixture', + uuid: 'schema-invalid-uuid', + start: 10, + stop: 20, + }), + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + allowPartialParse: true, + }) + expect(testcases).toHaveLength(1) + expect(testcases[0].name).toContain('TEST-002') + }) + + test('Should prioritize marker extraction as TMS URL > TMS link name > test name', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-404 marker in test name', + links: [ + { + type: 'tms', + url: 'https://qas.eu1.qasphere.com/project/TEST/tcase/2', + name: 'TEST-003', + }, + ], + }) + ), + '002-result.json': JSON.stringify( + makeResult({ + name: 'TEST-405 marker in test name', + links: [ + { + type: 'tms', + url: 'https://external.example.com/tms/entry', + name: 'TEST-006', + }, + ], + }) + ), + }) + + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + + expect(testcases).toHaveLength(2) + expect(testcases[0].name).toBe('TEST-002: TEST-404 marker in test name') + expect(testcases[1].name).toBe('TEST-006: TEST-405 marker in test name') + }) +}) diff --git a/src/tests/fixtures/allure/empty-tsuite/001-result.json b/src/tests/fixtures/allure/empty-tsuite/001-result.json new file mode 100644 index 0000000..4ca5ec5 --- /dev/null +++ b/src/tests/fixtures/allure/empty-tsuite/001-result.json @@ -0,0 +1,13 @@ +{ + "name": "TEST-002 Test cart", + "status": "passed", + "uuid": "empty-tsuite-001", + "start": 1700000040000, + "stop": 1700000040100, + "labels": [ + { + "name": "suite", + "value": "ui.cart.spec.ts" + } + ] +} diff --git a/src/tests/fixtures/allure/empty-tsuite/002-container.json b/src/tests/fixtures/allure/empty-tsuite/002-container.json new file mode 100644 index 0000000..0aec70e --- /dev/null +++ b/src/tests/fixtures/allure/empty-tsuite/002-container.json @@ -0,0 +1,8 @@ +{ + "uuid": "empty-tsuite-container-002", + "children": [], + "befores": [], + "afters": [], + "start": 1700000041000, + "stop": 1700000041100 +} diff --git a/src/tests/fixtures/allure/invalid-results/001-result.json b/src/tests/fixtures/allure/invalid-results/001-result.json new file mode 100644 index 0000000..b9133d4 --- /dev/null +++ b/src/tests/fixtures/allure/invalid-results/001-result.json @@ -0,0 +1,7 @@ +{ + "name": "TEST-002 Test cart from partial directory", + "status": "passed", + "uuid": "invalid-results-001", + "start": 1700000050000, + "stop": 1700000050200 +} diff --git a/src/tests/fixtures/allure/invalid-results/003-result.json b/src/tests/fixtures/allure/invalid-results/003-result.json new file mode 100644 index 0000000..0bdb266 --- /dev/null +++ b/src/tests/fixtures/allure/invalid-results/003-result.json @@ -0,0 +1,6 @@ +{ + "name": "Schema invalid fixture", + "uuid": "invalid-results-003", + "start": 1700000052000, + "stop": 1700000052300 +} diff --git a/src/tests/fixtures/allure/matching-tcases/001-result.json b/src/tests/fixtures/allure/matching-tcases/001-result.json new file mode 100644 index 0000000..8513fa7 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/001-result.json @@ -0,0 +1,27 @@ +{ + "name": "Test cart", + "status": "passed", + "uuid": "matching-001", + "start": 1700000000000, + "stop": 1700000000500, + "statusDetails": {}, + "attachments": [ + { + "name": "attachment", + "source": "shared-attachment.txt", + "type": "text/plain" + } + ], + "labels": [ + { + "name": "suite", + "value": "ui.cart.spec.ts" + } + ], + "links": [ + { + "type": "tms", + "url": "https://qas.eu1.qasphere.com/project/TEST/tcase/2" + } + ] +} diff --git a/src/tests/fixtures/allure/matching-tcases/002-result.json b/src/tests/fixtures/allure/matching-tcases/002-result.json new file mode 100644 index 0000000..29628d1 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/002-result.json @@ -0,0 +1,28 @@ +{ + "name": "Test checkout", + "status": "passed", + "uuid": "matching-002", + "start": 1700000001000, + "stop": 1700000001600, + "statusDetails": null, + "attachments": [ + { + "name": "attachment", + "source": "shared-attachment.txt", + "type": "text/plain" + } + ], + "labels": [ + { + "name": "parentSuite", + "value": "ui.cart.spec.ts" + } + ], + "links": [ + { + "type": "tms", + "url": "https://external.example.com/test-case/3", + "name": "TEST-003" + } + ] +} diff --git a/src/tests/fixtures/allure/matching-tcases/003-result.json b/src/tests/fixtures/allure/matching-tcases/003-result.json new file mode 100644 index 0000000..467d963 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/003-result.json @@ -0,0 +1,21 @@ +{ + "name": "About page content TEST-004", + "status": "passed", + "uuid": "matching-003", + "start": 1700000002000, + "stop": 1700000002800, + "attachments": [ + { + "name": "attachment", + "source": "shared-attachment.txt", + "type": "text/plain" + } + ], + "labels": [ + { + "name": "feature", + "value": "ui.contents.spec.ts" + } + ], + "links": [] +} diff --git a/src/tests/fixtures/allure/matching-tcases/004-result.json b/src/tests/fixtures/allure/matching-tcases/004-result.json new file mode 100644 index 0000000..e8dded2 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/004-result.json @@ -0,0 +1,30 @@ +{ + "name": "Navigation bar items", + "status": "failed", + "uuid": "matching-004", + "start": 1700000003000, + "stop": 1700000004200, + "statusDetails": { + "message": "AssertionError: navbar items mismatch", + "trace": "Traceback line 1\nTraceback line 2" + }, + "attachments": [ + { + "name": "attachment", + "source": "shared-attachment.txt", + "type": "text/plain" + } + ], + "labels": [ + { + "name": "package", + "value": "ui.contents.spec.ts" + } + ], + "links": [ + { + "type": "tms", + "url": "https://qas.eu1.qasphere.com/project/TEST/tcase/6" + } + ] +} diff --git a/src/tests/fixtures/allure/matching-tcases/005-result.json b/src/tests/fixtures/allure/matching-tcases/005-result.json new file mode 100644 index 0000000..16f04ab --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/005-result.json @@ -0,0 +1,24 @@ +{ + "name": "Welcome page content (updated)", + "status": "unknown", + "uuid": "matching-005", + "start": 1700000005000, + "stop": 1700000006200, + "statusDetails": { + "message": "No status was produced by adapter" + }, + "attachments": [ + { + "name": "attachment", + "source": "shared-attachment.txt", + "type": "text/plain" + } + ], + "labels": [], + "links": [ + { + "type": "tms", + "url": "https://qas.eu1.qasphere.com/project/TEST/tcase/7" + } + ] +} diff --git a/src/tests/fixtures/allure/matching-tcases/006-container.json b/src/tests/fixtures/allure/matching-tcases/006-container.json new file mode 100644 index 0000000..a2fd6be --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/006-container.json @@ -0,0 +1,8 @@ +{ + "uuid": "matching-container-006", + "children": ["matching-001"], + "befores": [], + "afters": [], + "start": 1700000000000, + "stop": 1700000000100 +} diff --git a/src/tests/fixtures/allure/matching-tcases/legacy-testsuite.xml b/src/tests/fixtures/allure/matching-tcases/legacy-testsuite.xml new file mode 100644 index 0000000..aa0e71f --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/legacy-testsuite.xml @@ -0,0 +1 @@ + diff --git a/src/tests/fixtures/allure/matching-tcases/shared-attachment.txt b/src/tests/fixtures/allure/matching-tcases/shared-attachment.txt new file mode 100644 index 0000000..9ac09f2 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/shared-attachment.txt @@ -0,0 +1 @@ +Allure fixture attachment content. diff --git a/src/tests/fixtures/allure/missing-attachments/001-result.json b/src/tests/fixtures/allure/missing-attachments/001-result.json new file mode 100644 index 0000000..044aae7 --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/001-result.json @@ -0,0 +1,14 @@ +{ + "name": "TEST-002 Test cart", + "status": "passed", + "uuid": "missing-attach-001", + "start": 1700000020000, + "stop": 1700000020200, + "attachments": [ + { + "name": "attachment", + "source": "existing-attachment.txt", + "type": "text/plain" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/002-result.json b/src/tests/fixtures/allure/missing-attachments/002-result.json new file mode 100644 index 0000000..679e995 --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/002-result.json @@ -0,0 +1,14 @@ +{ + "name": "TEST-003 Test checkout", + "status": "passed", + "uuid": "missing-attach-002", + "start": 1700000021000, + "stop": 1700000021350, + "attachments": [ + { + "name": "attachment", + "source": "existing-attachment.txt", + "type": "text/plain" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/003-result.json b/src/tests/fixtures/allure/missing-attachments/003-result.json new file mode 100644 index 0000000..dc2d74f --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/003-result.json @@ -0,0 +1,14 @@ +{ + "name": "TEST-004 About page content", + "status": "passed", + "uuid": "missing-attach-003", + "start": 1700000022000, + "stop": 1700000022500, + "attachments": [ + { + "name": "attachment", + "source": "existing-attachment.txt", + "type": "text/plain" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/004-result.json b/src/tests/fixtures/allure/missing-attachments/004-result.json new file mode 100644 index 0000000..68b92c2 --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/004-result.json @@ -0,0 +1,17 @@ +{ + "name": "TEST-006 Navigation bar items", + "status": "failed", + "uuid": "missing-attach-004", + "start": 1700000023000, + "stop": 1700000023300, + "statusDetails": { + "message": "AssertionError: navigation items mismatch" + }, + "attachments": [ + { + "name": "attachment", + "source": "existing-attachment.txt", + "type": "text/plain" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/005-result.json b/src/tests/fixtures/allure/missing-attachments/005-result.json new file mode 100644 index 0000000..09e5255 --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/005-result.json @@ -0,0 +1,14 @@ +{ + "name": "TEST-007 Welcome page content", + "status": "passed", + "uuid": "missing-attach-005", + "start": 1700000024000, + "stop": 1700000024700, + "attachments": [ + { + "name": "attachment", + "source": "missing-attachment.txt", + "type": "text/plain" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/existing-attachment.txt b/src/tests/fixtures/allure/missing-attachments/existing-attachment.txt new file mode 100644 index 0000000..102ee6a --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/existing-attachment.txt @@ -0,0 +1 @@ +existing attachment content diff --git a/src/tests/fixtures/allure/missing-tcases/001-result.json b/src/tests/fixtures/allure/missing-tcases/001-result.json new file mode 100644 index 0000000..1d149fa --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/001-result.json @@ -0,0 +1,13 @@ +{ + "name": "TEST-002 Test cart", + "status": "passed", + "uuid": "missing-001", + "start": 1700000010000, + "stop": 1700000010200, + "labels": [ + { + "name": "suite", + "value": "ui.cart.spec.ts" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-tcases/002-result.json b/src/tests/fixtures/allure/missing-tcases/002-result.json new file mode 100644 index 0000000..61eb47d --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/002-result.json @@ -0,0 +1,13 @@ +{ + "name": "TEST-003 Test checkout", + "status": "passed", + "uuid": "missing-002", + "start": 1700000011000, + "stop": 1700000011350, + "labels": [ + { + "name": "suite", + "value": "ui.cart.spec.ts" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-tcases/003-result.json b/src/tests/fixtures/allure/missing-tcases/003-result.json new file mode 100644 index 0000000..c18d456 --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/003-result.json @@ -0,0 +1,13 @@ +{ + "name": "TEST-004 About page content", + "status": "passed", + "uuid": "missing-003", + "start": 1700000012000, + "stop": 1700000012500, + "labels": [ + { + "name": "feature", + "value": "ui.contents.spec.ts" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-tcases/004-result.json b/src/tests/fixtures/allure/missing-tcases/004-result.json new file mode 100644 index 0000000..a0eca24 --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/004-result.json @@ -0,0 +1,16 @@ +{ + "name": "TEST-006 Navigation bar items", + "status": "failed", + "uuid": "missing-004", + "start": 1700000013000, + "stop": 1700000013300, + "statusDetails": { + "message": "AssertionError: navigation items mismatch" + }, + "labels": [ + { + "name": "package", + "value": "ui.contents.spec.ts" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-tcases/005-result.json b/src/tests/fixtures/allure/missing-tcases/005-result.json new file mode 100644 index 0000000..e7bf9c8 --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/005-result.json @@ -0,0 +1,13 @@ +{ + "name": "TEST-999 This test does not exist in run", + "status": "passed", + "uuid": "missing-005", + "start": 1700000014000, + "stop": 1700000014700, + "labels": [ + { + "name": "suite", + "value": "ui.unknown.spec.ts" + } + ] +} diff --git a/src/tests/fixtures/allure/without-markers/001-result.json b/src/tests/fixtures/allure/without-markers/001-result.json new file mode 100644 index 0000000..bedabad --- /dev/null +++ b/src/tests/fixtures/allure/without-markers/001-result.json @@ -0,0 +1,8 @@ +{ + "name": "The cart is still filled after refreshing the page", + "status": "passed", + "uuid": "without-markers-001", + "start": 1700000030000, + "stop": 1700000030200, + "labels": [] +} diff --git a/src/tests/fixtures/allure/without-markers/002-result.json b/src/tests/fixtures/allure/without-markers/002-result.json new file mode 100644 index 0000000..9054ee3 --- /dev/null +++ b/src/tests/fixtures/allure/without-markers/002-result.json @@ -0,0 +1,11 @@ +{ + "name": "The checkout should finish successfully", + "status": "failed", + "uuid": "without-markers-002", + "start": 1700000031000, + "stop": 1700000031800, + "statusDetails": { + "message": "AssertionError: expected true to be false" + }, + "labels": [] +} diff --git a/src/tests/fixtures/allure/without-markers/003-result.json b/src/tests/fixtures/allure/without-markers/003-result.json new file mode 100644 index 0000000..fcb4294 --- /dev/null +++ b/src/tests/fixtures/allure/without-markers/003-result.json @@ -0,0 +1,8 @@ +{ + "name": "The cart is still filled after refreshing the page", + "status": "passed", + "uuid": "without-markers-003", + "start": 1700000032000, + "stop": 1700000032400, + "labels": [] +} diff --git a/src/tests/result-upload.spec.ts b/src/tests/result-upload.spec.ts index 3aee549..57ba18f 100644 --- a/src/tests/result-upload.spec.ts +++ b/src/tests/result-upload.spec.ts @@ -159,29 +159,56 @@ const cleanupGeneratedMappingFiles = (existingMappingFiles?: Set) => { }) } +interface TestFileType { + name: string + command: 'junit-upload' | 'playwright-json-upload' | 'allure-upload' + dataBasePath: string + inputType: 'file' | 'directory' + fileExtension?: string +} + +const fixtureInputPath = (fileType: TestFileType, fixtureName: string) => { + if (fileType.inputType === 'directory') { + return `${fileType.dataBasePath}/${fixtureName}` + } + return `${fileType.dataBasePath}/${fixtureName}.${fileType.fileExtension}` +} + const fileTypes = [ { name: 'JUnit XML', command: 'junit-upload', dataBasePath: './src/tests/fixtures/junit-xml', fileExtension: 'xml', + inputType: 'file', }, { name: 'Playwright JSON', command: 'playwright-json-upload', dataBasePath: './src/tests/fixtures/playwright-json', fileExtension: 'json', + inputType: 'file', + }, +] as const satisfies readonly TestFileType[] + +const fileTypesWithAllure: TestFileType[] = [ + ...fileTypes, + { + name: 'Allure', + command: 'allure-upload', + dataBasePath: './src/tests/fixtures/allure', + inputType: 'directory', }, ] -fileTypes.forEach((fileType) => { +fileTypesWithAllure.forEach((fileType) => { describe(`Uploading ${fileType.name} files`, () => { describe('Argument parsing', () => { test('Passing correct Run URL pattern should result in success', async () => { const patterns = [ - `${fileType.command} --run-url ${runURL} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, - `${fileType.command} -r ${runURL}/ ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, - `${fileType.command} -r ${runURL}/tcase/1 ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, + `${fileType.command} --run-url ${runURL} ${fixtureInputPath(fileType, 'matching-tcases')}`, + `${fileType.command} -r ${runURL}/ ${fixtureInputPath(fileType, 'matching-tcases')}`, + `${fileType.command} -r ${runURL}/tcase/1 ${fixtureInputPath(fileType, 'matching-tcases')}`, ] for (const pattern of patterns) { @@ -198,7 +225,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(1) await run( - `${fileType.command} -r ${qasHost}/project/${projectCode}/run/${runId} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${qasHost}/project/${projectCode}/run/${runId} ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(5) // 5 results total @@ -206,8 +233,8 @@ fileTypes.forEach((fileType) => { test('Passing incorrect Run URL pattern should result in failure', async () => { const patterns = [ - `${fileType.command} -r ${qasHost}/projects/${projectCode}/runs/${runId} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, - `${fileType.command} -r ${runURL}abc/tcase/1 ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, + `${fileType.command} -r ${qasHost}/projects/${projectCode}/runs/${runId} ${fixtureInputPath(fileType, 'matching-tcases')}`, + `${fileType.command} -r ${runURL}abc/tcase/1 ${fixtureInputPath(fileType, 'matching-tcases')}`, ] for (const pattern of patterns) { @@ -233,7 +260,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(2) await run( - `${fileType.command} -r ${runURL} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(3) // 5 results total @@ -243,9 +270,7 @@ fileTypes.forEach((fileType) => { const numFileUploadCalls = countFileUploadApiCalls() const numResultUploadCalls = countResultUploadApiCalls() await expect( - run( - `${fileType.command} -r ${runURL} ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` - ) + run(`${fileType.command} -r ${runURL} ${fixtureInputPath(fileType, 'missing-tcases')}`) ).rejects.toThrowError() expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(0) @@ -256,7 +281,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(3) await run( - `${fileType.command} -r ${runURL} --force ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --force ${fixtureInputPath(fileType, 'missing-tcases')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(2) // 4 results total @@ -266,7 +291,7 @@ fileTypes.forEach((fileType) => { const numFileUploadCalls = countFileUploadApiCalls() const numResultUploadCalls = countResultUploadApiCalls() await run( - `${fileType.command} -r ${runURL} --ignore-unmatched ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --ignore-unmatched ${fixtureInputPath(fileType, 'missing-tcases')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(1) // 4 results total @@ -277,7 +302,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(2) await run( - `${fileType.command} -r ${runURL} --force ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension} ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --force ${fixtureInputPath(fileType, 'missing-tcases')} ${fixtureInputPath(fileType, 'missing-tcases')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(4) // 8 results total @@ -287,7 +312,7 @@ fileTypes.forEach((fileType) => { const numFileUploadCalls = countFileUploadApiCalls() const numResultUploadCalls = countResultUploadApiCalls() await run( - `${fileType.command} -r ${runURL} --force ${fileType.dataBasePath}/empty-tsuite.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --force ${fixtureInputPath(fileType, 'empty-tsuite')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(1) // 1 result total @@ -300,7 +325,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(3) await run( - `${fileType.command} -r ${runURL} --attachments ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --attachments ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(numFileUploadCalls()).toBe(5) expect(numResultUploadCalls()).toBe(2) // 5 results total @@ -310,7 +335,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() await expect( run( - `${fileType.command} -r ${runURL} --attachments ${fileType.dataBasePath}/missing-attachments.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --attachments ${fixtureInputPath(fileType, 'missing-attachments')}` ) ).rejects.toThrow() expect(numFileUploadCalls()).toBe(0) @@ -321,7 +346,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(1) await run( - `${fileType.command} -r ${runURL} --attachments --force ${fileType.dataBasePath}/missing-attachments.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --attachments --force ${fixtureInputPath(fileType, 'missing-attachments')}` ) expect(numFileUploadCalls()).toBe(4) expect(numResultUploadCalls()).toBe(5) // 5 results total @@ -342,7 +367,7 @@ fileTypes.forEach((fileType) => { try { // This should create a new run since no --run-url is specified await run( - `${fileType.command} --run-name "CI Build {env:TEST_BUILD_NUMBER}" ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --run-name "CI Build {env:TEST_BUILD_NUMBER}" ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(lastCreatedRunTitle).toBe('CI Build 456') @@ -363,7 +388,7 @@ fileTypes.forEach((fileType) => { const expectedDay = String(now.getDate()).padStart(2, '0') await run( - `${fileType.command} --run-name "Test Run {YYYY}-{MM}-{DD}" ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --run-name "Test Run {YYYY}-{MM}-{DD}" ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(lastCreatedRunTitle).toBe(`Test Run ${expectedYear}-${expectedMonth}-${expectedDay}`) @@ -375,7 +400,7 @@ fileTypes.forEach((fileType) => { try { await run( - `${fileType.command} --run-name "{env:TEST_PROJECT} - {YYYY}/{MM}" ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --run-name "{env:TEST_PROJECT} - {YYYY}/{MM}" ${fixtureInputPath(fileType, 'matching-tcases')}` ) const now = new Date() @@ -398,7 +423,7 @@ fileTypes.forEach((fileType) => { createRunTitleConflict = true await run( - `${fileType.command} --run-name "duplicate run title" ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --run-name "duplicate run title" ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(lastCreatedRunTitle).toBe('duplicate run title') @@ -407,9 +432,7 @@ fileTypes.forEach((fileType) => { }) test('Should use default name template when --run-name is not specified', async () => { - await run( - `${fileType.command} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` - ) + await run(`${fileType.command} ${fixtureInputPath(fileType, 'matching-tcases')}`) // Should use default format: "Automated test run - {MMM} {DD}, {YYYY}, {hh}:{mm}:{ss} {AMPM}" expect(lastCreatedRunTitle).toContain('Automated test run - ') @@ -447,7 +470,7 @@ fileTypes.forEach((fileType) => { } await run( - `${fileType.command} --project-code ${projectCode} --create-tcases ${fileType.dataBasePath}/without-markers.${fileType.fileExtension}` + `${fileType.command} --project-code ${projectCode} --create-tcases ${fixtureInputPath(fileType, 'without-markers')}` ) expect(numCreateTCasesCalls()).toBe(1) expect(numResultUploadCalls()).toBe(3) // 3 results total @@ -484,7 +507,7 @@ fileTypes.forEach((fileType) => { } await run( - `${fileType.command} --project-code ${projectCode} --create-tcases ${fileType.dataBasePath}/without-markers.${fileType.fileExtension}` + `${fileType.command} --project-code ${projectCode} --create-tcases ${fixtureInputPath(fileType, 'without-markers')}` ) expect(numCreateTCasesCalls()).toBe(1) expect(numResultUploadCalls()).toBe(3) // 3 results total @@ -496,7 +519,7 @@ fileTypes.forEach((fileType) => { setMaxResultsInRequest(1) await run( - `${fileType.command} --project-code ${projectCode} --create-tcases ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --project-code ${projectCode} --create-tcases ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(numCreateTCasesCalls()).toBe(0) expect(numResultUploadCalls()).toBe(5) // 5 results total @@ -504,3 +527,19 @@ fileTypes.forEach((fileType) => { }) }) }) + +describe('Allure invalid result file handling', () => { + test('Should fail when directory contains malformed files without --force', async () => { + const numResultUploadCalls = countResultUploadApiCalls() + await expect( + run(`allure-upload -r ${runURL} ./src/tests/fixtures/allure/invalid-results`) + ).rejects.toThrowError() + expect(numResultUploadCalls()).toBe(0) + }) + + test('Should skip invalid files and continue when --force is set', async () => { + const numResultUploadCalls = countResultUploadApiCalls() + await run(`allure-upload -r ${runURL} --force ./src/tests/fixtures/allure/invalid-results`) + expect(numResultUploadCalls()).toBe(1) // 1 valid result total + }) +}) diff --git a/src/utils/result-upload/ResultUploadCommandHandler.ts b/src/utils/result-upload/ResultUploadCommandHandler.ts index f4b90be..7a49372 100644 --- a/src/utils/result-upload/ResultUploadCommandHandler.ts +++ b/src/utils/result-upload/ResultUploadCommandHandler.ts @@ -9,17 +9,21 @@ import { TestCaseResult } from './types' import { ResultUploader } from './ResultUploader' import { parseJUnitXml } from './junitXmlParser' import { parsePlaywrightJson } from './playwrightJsonParser' +import { parseAllureResults } from './allureParser' -export type UploadCommandType = 'junit-upload' | 'playwright-json-upload' +export type UploadCommandType = 'junit-upload' | 'playwright-json-upload' | 'allure-upload' export type SkipOutputOption = 'on-success' | 'never' export interface ParserOptions { skipStdout: SkipOutputOption skipStderr: SkipOutputOption + allowPartialParse?: boolean } export type Parser = ( + // Primary parser input. File-based parsers receive file contents while + // directory-based parsers (like Allure) receive a directory path. data: string, attachmentBaseDirectory: string, options: ParserOptions @@ -62,6 +66,7 @@ const DEFAULT_MAPPING_FILENAME_TEMPLATE = 'qasphere-automapping-{YYYY}{MM}{DD}-{ const commandTypeParsers: Record = { 'junit-upload': parseJUnitXml, 'playwright-json-upload': parsePlaywrightJson, + 'allure-upload': parseAllureResults, } export class ResultUploadCommandHandler { @@ -129,13 +134,14 @@ export class ResultUploadCommandHandler { const parserOptions: ParserOptions = { skipStdout: this.args.skipReportStdout, skipStderr: this.args.skipReportStderr, + allowPartialParse: this.args.force, } for (const file of this.args.files) { - const fileData = readFileSync(file).toString() + const fileData = this.type === 'allure-upload' ? file : readFileSync(file).toString() const fileResults = await commandTypeParsers[this.type]( fileData, - dirname(file), + this.type === 'allure-upload' ? file : dirname(file), parserOptions ) results.push({ file, results: fileResults }) @@ -163,20 +169,20 @@ export class ResultUploadCommandHandler { ) } - private execRegexWithPriority(pattern: string, str: string): RegExpExecArray | null { + private execRegexWithPriority(pattern: string, str: string): RegExpMatchArray | null { // Try matching at start first const startRegex = new RegExp(`^${pattern}`) - let match = startRegex.exec(str) + let match = str.match(startRegex) if (match) return match // Try matching at end const endRegex = new RegExp(`${pattern}$`) - match = endRegex.exec(str) + match = str.match(endRegex) if (match) return match // Fall back to matching anywhere const anywhereRegex = new RegExp(pattern) - return anywhereRegex.exec(str) + return str.match(anywhereRegex) } protected async getTCaseIds(projectCode: string, fileResults: FileResults[]) { diff --git a/src/utils/result-upload/ResultUploader.ts b/src/utils/result-upload/ResultUploader.ts index bb96596..8cfae67 100644 --- a/src/utils/result-upload/ResultUploader.ts +++ b/src/utils/result-upload/ResultUploader.ts @@ -81,6 +81,8 @@ export class ResultUploader { this.printJUnitGuidance() } else if (this.type === 'playwright-json-upload') { this.printPlaywrightGuidance(missing[0]?.name || 'your test name') + } else if (this.type === 'allure-upload') { + this.printAllureGuidance(missing[0]?.name || 'your test name') } console.error( chalk.yellow( @@ -128,6 +130,36 @@ ${chalk.yellow('To fix this issue, choose one of the following options:')} `) } + private printAllureGuidance(exampleTestName: string) { + console.error(` +${chalk.yellow('To fix this issue, choose one of the following options:')} + + ${chalk.bold('Option 1: Add a TMS Link (Recommended)')} + Add a TMS link in the Allure result with: + - ${chalk.green('type')}: ${chalk.green('"tms"')} + - ${chalk.green('url')}: ${chalk.green( + `"https://your-qas-instance.com/project/${this.project}/tcase/123"` + )} + + Example: + ${chalk.green(`{ + "links": [ + { + "type": "tms", + "url": "https://your-qas-instance.com/project/${this.project}/tcase/123" + } + ] +}`)} + + ${chalk.bold('Option 2: Include Test Case Marker in Name')} + Include marker ${chalk.green(`${this.project}-`)} in the test name: + + Format: ${chalk.green(`${this.project}-: Your test name`)} + Example: ${chalk.green(`${this.project}-1024: ${exampleTestName}`)} + ${chalk.dim('Where is the test case number (minimum 3 digits, zero-padded if needed)')} +`) + } + private validateAndPrintMissingAttachments = (results: TCaseWithResult[]) => { if (this.args.attachments) { let hasAttachmentErrors = false diff --git a/src/utils/result-upload/allureParser.ts b/src/utils/result-upload/allureParser.ts new file mode 100644 index 0000000..e891bc2 --- /dev/null +++ b/src/utils/result-upload/allureParser.ts @@ -0,0 +1,233 @@ +import escapeHtml from 'escape-html' +import { readdirSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import stripAnsi from 'strip-ansi' +import z from 'zod' +import { ResultStatus } from '../../api/schemas' +import { getTCaseMarker, parseTCaseUrl } from '../misc' +import { Parser, ParserOptions } from './ResultUploadCommandHandler' +import { Attachment, TestCaseResult } from './types' +import { getAttachments } from './utils' + +const allureStatusSchema = z.enum(['passed', 'failed', 'broken', 'skipped', 'unknown']) +type AllureStatus = z.infer + +const allureStatusDetailsSchema = z + .object({ + message: z.string().optional(), + trace: z.string().optional(), + known: z.boolean().optional(), + muted: z.boolean().optional(), + flaky: z.boolean().optional(), + }) + .nullish() + +const allureAttachmentSchema = z.object({ + name: z.string(), + source: z.string(), + type: z.string(), +}) + +const allureLabelSchema = z.object({ + name: z.string(), + value: z.string(), +}) + +const allureParameterSchema = z.object({ + name: z.string(), + value: z.string(), + excluded: z.boolean().optional(), + mode: z.enum(['default', 'masked', 'hidden']).optional(), +}) + +const allureLinkSchema = z.object({ + name: z.string().optional(), + url: z.string(), + type: z.string().optional(), +}) + +const allureResultSchema = z.object({ + name: z.string(), + status: allureStatusSchema, + uuid: z.string(), + start: z.number(), + stop: z.number(), + fullName: z.string().optional(), + historyId: z.string().optional(), + testCaseId: z.string().optional(), + description: z.string().optional(), + descriptionHtml: z.string().optional(), + stage: z.string().optional(), + statusDetails: allureStatusDetailsSchema, + attachments: z.array(allureAttachmentSchema).nullish(), + labels: z.array(allureLabelSchema).nullish(), + links: z.array(allureLinkSchema).nullish(), + parameters: z.array(allureParameterSchema).nullish(), + steps: z.array(z.unknown()).nullish(), +}) + +type AllureResult = z.infer + +const markerRegex = /\b([A-Za-z0-9]{1,5})-(\d+)\b/ + +export const parseAllureResults: Parser = async ( + resultsDirectory: string, + attachmentBaseDirectory: string, + options: ParserOptions +): Promise => { + const resultFiles = readdirSync(resultsDirectory) + .filter((f) => f.endsWith('-result.json')) + .sort() + + const testcases: TestCaseResult[] = [] + const attachmentsPromises: Array<{ + index: number + promise: Promise + }> = [] + const allowPartialParse = options.allowPartialParse ?? false + + for (const resultFile of resultFiles) { + const resultFilePath = join(resultsDirectory, resultFile) + + let parsedResult: AllureResult + try { + const fileContent = readFileSync(resultFilePath, 'utf8') + parsedResult = allureResultSchema.parse(JSON.parse(fileContent)) + } catch (error) { + if (allowPartialParse) { + console.warn( + `Warning: Skipping invalid Allure result file "${resultFilePath}": ${getErrorMessage(error)}` + ) + continue + } + + throw new Error( + `Failed to parse Allure result file "${resultFilePath}": ${getErrorMessage(error)}` + ) + } + + const status = mapAllureStatus(parsedResult.status) + const marker = extractMarker(parsedResult) + const index = + testcases.push({ + name: marker ? `${marker}: ${parsedResult.name}` : parsedResult.name, + folder: getFolder(parsedResult), + status, + message: buildMessage(parsedResult, status, options), + timeTaken: + parsedResult.stop >= parsedResult.start ? parsedResult.stop - parsedResult.start : null, + attachments: [], + }) - 1 + + const attachmentPaths = (parsedResult.attachments || []).map((attachment) => attachment.source) + attachmentsPromises.push({ + index, + promise: getAttachments(attachmentPaths, attachmentBaseDirectory || resultsDirectory), + }) + } + + const attachments = await Promise.all(attachmentsPromises.map((p) => p.promise)) + attachments.forEach((tcaseAttachment, i) => { + const tcaseIndex = attachmentsPromises[i].index + testcases[tcaseIndex].attachments = tcaseAttachment + }) + + return testcases +} + +const mapAllureStatus = (status: AllureStatus): ResultStatus => { + switch (status) { + case 'passed': + return 'passed' + case 'failed': + return 'failed' + case 'broken': + return 'blocked' + case 'skipped': + return 'skipped' + case 'unknown': + return 'open' + default: + return 'open' + } +} + +const getFolder = (result: AllureResult): string => { + const labels = result.labels || [] + const folderLabelPriority = ['suite', 'parentSuite', 'feature', 'package'] + + for (const labelName of folderLabelPriority) { + const label = labels.find((item) => item.name === labelName) + if (label?.value) { + return label.value + } + } + + return '' +} + +const buildMessage = ( + result: AllureResult, + status: ResultStatus, + options: ParserOptions +): string => { + const statusDetails = result.statusDetails + if (!statusDetails) { + return '' + } + + const includeStdout = !(status === 'passed' && options.skipStdout === 'on-success') + const includeStderr = !(status === 'passed' && options.skipStderr === 'on-success') + + let message = '' + + if (includeStdout && statusDetails.message) { + message += `
${escapeHtml(stripAnsi(statusDetails.message))}
` + } + if (includeStderr && statusDetails.trace) { + message += `
${escapeHtml(stripAnsi(statusDetails.trace))}
` + } + + return message +} + +const extractMarker = (result: AllureResult): string | undefined => { + return getMarkerFromTmsLinks(result.links) || getMarkerFromText(result.name) +} + +const getMarkerFromTmsLinks = (links: AllureResult['links']): string | undefined => { + const tmsLinks = (links || []).filter((link) => link.type?.toLowerCase() === 'tms') + + for (const link of tmsLinks) { + const parsed = parseTCaseUrl(link.url) + if (parsed) { + return getTCaseMarker(parsed.project, parsed.tcaseSeq) + } + } + + for (const link of tmsLinks) { + const markerFromName = getMarkerFromText(link.name) + if (markerFromName) { + return markerFromName + } + } + + return undefined +} + +const getMarkerFromText = (text: string | undefined): string | undefined => { + if (!text) { + return undefined + } + + const match = text.match(markerRegex) + if (!match) { + return undefined + } + + return getTCaseMarker(match[1], Number(match[2])) +} + +const getErrorMessage = (error: unknown) => { + return error instanceof Error ? error.message : String(error) +}