commit d7f86eb62aff1627279eaf62b08e6f7613eccdc5
parent 188c09c6edc28281713f5e890a107ca1d6168012
Author: Florian Dold <dold@taler.net>
Date: Mon, 27 Jul 2026 17:38:59 +0200
harness: support todo tests in run-integrationtests
Diffstat:
3 files changed, 91 insertions(+), 11 deletions(-)
diff --git a/packages/taler-harness/src/harness/harness.ts b/packages/taler-harness/src/harness/harness.ts
@@ -2784,6 +2784,12 @@ export interface TestRunResult {
status: TestStatus;
+ /**
+ * Was the test marked as "todo"? A failure of a todo test is
+ * reported separately and does not make the test run fail.
+ */
+ todo?: boolean;
+
reason?: string;
}
diff --git a/packages/taler-harness/src/index.ts b/packages/taler-harness/src/index.ts
@@ -1761,6 +1761,9 @@ talerHarnessCli
if (t.experimental) {
s += ` [experimental]`;
}
+ if (t.todo) {
+ s += ` [todo]`;
+ }
console.log(s);
}
});
@@ -1782,6 +1785,9 @@ talerHarnessCli
.flag("experimental", ["--experimental"], {
help: "Include tests marked as experimental",
})
+ .flag("strictTodo", ["--strict-todo"], {
+ help: "Count failures of tests marked as todo as regular failures",
+ })
.flag("failFast", ["--fail-fast"], {
help: "Exit after the first error",
})
@@ -1805,6 +1811,7 @@ talerHarnessCli
dryRun: args.runIntegrationtests.dryRun,
verbosity: args.runIntegrationtests.quiet ? 0 : 1,
includeExperimental: args.runIntegrationtests.experimental ?? false,
+ strictTodo: args.runIntegrationtests.strictTodo ?? false,
noTimeout: noTimeout ?? args.runIntegrationtests.noTimeout,
testDir: args.runIntegrationtests.testDir,
});
diff --git a/packages/taler-harness/src/integrationtests/testrunner.ts b/packages/taler-harness/src/integrationtests/testrunner.ts
@@ -252,6 +252,12 @@ interface TestMainFunction {
(t: GlobalTestState): Promise<void>;
timeoutMs?: number;
experimental?: boolean;
+ /**
+ * Mark the test as "todo": it is always run and reported, but a failure
+ * is tolerated and does not make the whole test run fail. Use this for
+ * tests that describe behavior that is not implemented (correctly) yet.
+ */
+ todo?: boolean;
suites?: string[];
}
@@ -472,6 +478,10 @@ export interface TestRunSpec {
failFast?: boolean;
waitOnFail?: boolean;
includeExperimental: boolean;
+ /**
+ * Treat failures of todo tests as real failures.
+ */
+ strictTodo: boolean;
noTimeout: boolean;
verbosity: number;
}
@@ -480,6 +490,7 @@ export interface TestInfo {
name: string;
suites: string[];
experimental: boolean;
+ todo: boolean;
}
function updateCurrentSymlink(testDir: string): void {
@@ -573,7 +584,10 @@ export async function runTests(spec: TestRunSpec) {
if (currentChild) {
currentChild.kill("SIGTERM");
}
- reportAndQuit(testRootDir, testResults, true);
+ reportAndQuit(testRootDir, testResults, {
+ interrupted: true,
+ strictTodo: spec.strictTodo,
+ });
};
process.on("SIGINT", (s) => handleSignal(s));
@@ -623,11 +637,15 @@ export async function runTests(spec: TestRunSpec) {
console.log(`selected ${filteredTests.length} tests`);
let numFailed = 0;
+ let numTodoFailed = 0;
for (const [n, testCase] of filteredTests.entries()) {
const testName = getTestName(testCase);
+ const isTodo = testCase.todo ?? false;
if (spec.dryRun) {
- console.log(`dry run: would run test ${testName}`);
+ console.log(
+ `dry run: would run test ${testName}${isTodo ? " (todo)" : ""}`,
+ );
continue;
}
@@ -677,12 +695,19 @@ export async function runTests(spec: TestRunSpec) {
if (numFailed > 0) {
progressText = progressText + `, failed ${numFailed}`;
}
+ if (numTodoFailed > 0) {
+ progressText = progressText + `, todo failed ${numTodoFailed}`;
+ }
+
+ const todoText = isTodo ? " [todo]" : "";
if (spec.noTimeout) {
- console.log(`running ${testName} (${progressText}), no timeout`);
+ console.log(
+ `running ${testName}${todoText} (${progressText}), no timeout`,
+ );
} else {
console.log(
- `running ${testName} (${progressText}) with timeout ${testTimeoutMs}ms`,
+ `running ${testName}${todoText} (${progressText}) with timeout ${testTimeoutMs}ms`,
);
}
@@ -731,10 +756,6 @@ export async function runTests(spec: TestRunSpec) {
try {
result = await token.racePromise(resultPromise);
- if (result.status === "fail" && spec.failFast) {
- logger.error("test failed and failing fast, exit!");
- throw Error("exit on fail fast");
- }
} catch (e: any) {
if (token.isCancelled) {
result = {
@@ -759,8 +780,19 @@ export async function runTests(spec: TestRunSpec) {
}
}
+ if (isTodo) {
+ result.todo = true;
+ }
+
+ // A failing todo test is expected, unless we're asked to be strict.
+ const toleratedFailure = isTodo && !spec.strictTodo;
+
if (result.status === "fail") {
- numFailed++;
+ if (toleratedFailure) {
+ numTodoFailed++;
+ } else {
+ numFailed++;
+ }
}
harnessLogStream.close();
@@ -778,22 +810,49 @@ export async function runTests(spec: TestRunSpec) {
console.log(`parent: got result ${JSON.stringify(result)}`);
testResults.push(result);
+
+ if (result.status === "fail" && !toleratedFailure && spec.failFast) {
+ logger.error("test failed and failing fast, exit!");
+ break;
+ }
}
- reportAndQuit(testRootDir, testResults);
+ reportAndQuit(testRootDir, testResults, { strictTodo: spec.strictTodo });
+}
+
+export interface ReportOptions {
+ interrupted?: boolean;
+ /**
+ * Count failures of todo tests as real failures.
+ */
+ strictTodo?: boolean;
}
export function reportAndQuit(
testRootDir: string,
testResults: TestRunResult[],
- interrupted: boolean = false,
+ options: ReportOptions = {},
): never {
+ const interrupted = options.interrupted ?? false;
+ const strictTodo = options.strictTodo ?? false;
+
let numTotal = 0;
let numFail = 0;
let numSkip = 0;
let numPass = 0;
+ // Results of todo tests, reported separately. Unless we run in strict
+ // mode, they are not counted in the tallies above and don't influence
+ // the exit status.
+ const todoResults: TestRunResult[] = [];
+
for (const result of testResults) {
+ if (result.todo) {
+ todoResults.push(result);
+ if (!strictTodo) {
+ continue;
+ }
+ }
numTotal++;
if (result.status === "fail") {
numFail++;
@@ -818,6 +877,13 @@ export function reportAndQuit(
console.log(`Failed: ${numFail}/${numTotal}`);
console.log(`Passed: ${numPass}/${numTotal}`);
+ if (todoResults.length > 0) {
+ // Which todo test had which result is in results.json.
+ const numTodoFail = todoResults.filter((x) => x.status === "fail").length;
+ const suffix = strictTodo ? " (counted, --strict-todo)" : " (not counted)";
+ console.log(`Todo failed${suffix}: ${numTodoFail}/${todoResults.length}`);
+ }
+
if (interrupted) {
process.exit(3);
} else if (numPass < numTotal - numSkip) {
@@ -832,6 +898,7 @@ export function getTestInfo(): TestInfo[] {
name: getTestName(x),
suites: x.suites ?? [],
experimental: x.experimental ?? false,
+ todo: x.todo ?? false,
}));
}
