mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-04 07:28:15 +02:00
374 lines
17 KiB
TypeScript
374 lines
17 KiB
TypeScript
import { execFileSync } from "node:child_process";
|
|
import { readFileSync } from "node:fs";
|
|
import { describe, expect, it } from "vitest";
|
|
import { parse } from "yaml";
|
|
import { findLaneByName } from "../../scripts/lib/docker-e2e-plan.mjs";
|
|
import { BUNDLED_PLUGIN_INSTALL_UNINSTALL_SHARDS } from "../../scripts/lib/docker-e2e-scenarios.mjs";
|
|
import {
|
|
PLUGIN_PRERELEASE_REQUIRED_SURFACES,
|
|
assertPluginPrereleaseTestPlanComplete,
|
|
createPluginPrereleaseTestPlan,
|
|
} from "../../scripts/lib/plugin-prerelease-test-plan.mjs";
|
|
|
|
function readCiWorkflow() {
|
|
return parse(readFileSync(".github/workflows/ci.yml", "utf8"));
|
|
}
|
|
|
|
function readFullReleaseValidationWorkflow() {
|
|
return parse(readFileSync(".github/workflows/full-release-validation.yml", "utf8"));
|
|
}
|
|
|
|
function readPluginPrereleaseWorkflow() {
|
|
return parse(readFileSync(".github/workflows/plugin-prerelease.yml", "utf8"));
|
|
}
|
|
|
|
describe("scripts/lib/plugin-prerelease-test-plan.mjs", () => {
|
|
it("covers every pre-release plugin skill surface in the plugin prerelease plan", () => {
|
|
const plan = assertPluginPrereleaseTestPlanComplete();
|
|
|
|
expect(plan.surfaces).toEqual(
|
|
[...PLUGIN_PRERELEASE_REQUIRED_SURFACES].toSorted((a, b) => a.localeCompare(b)),
|
|
);
|
|
});
|
|
|
|
it("runs the package and Docker product lanes through the existing scheduler", () => {
|
|
const plan = createPluginPrereleaseTestPlan();
|
|
|
|
expect(plan.dockerLanes).toEqual([
|
|
"npm-onboard-channel-agent",
|
|
"npm-onboard-discord-channel-agent",
|
|
"doctor-switch",
|
|
"update-channel-switch",
|
|
"plugins-offline",
|
|
"plugins",
|
|
"kitchen-sink-plugin",
|
|
"plugin-update",
|
|
"config-reload",
|
|
"gateway-network",
|
|
"mcp-channels",
|
|
"cron-mcp-cleanup",
|
|
...Array.from(
|
|
{ length: BUNDLED_PLUGIN_INSTALL_UNINSTALL_SHARDS },
|
|
(_, index) => `bundled-plugin-install-uninstall-${index}`,
|
|
),
|
|
]);
|
|
|
|
for (const lane of plan.dockerLanes) {
|
|
expect(findLaneByName(lane), lane).toBeTruthy();
|
|
}
|
|
});
|
|
|
|
it("keeps live-ish coverage outside provider-backed Docker lanes", () => {
|
|
const plan = createPluginPrereleaseTestPlan();
|
|
|
|
expect(plan.dockerLanes).not.toContain("openai-web-search-minimal");
|
|
expect(plan.dockerLanes.some((lane) => lane.startsWith("live-"))).toBe(false);
|
|
expect(plan.staticChecks).toContainEqual({
|
|
check: "live-ish-availability",
|
|
checkName: "checks-plugin-prerelease-live-ish-availability",
|
|
command: "node scripts/plugin-prerelease-liveish-matrix.mjs",
|
|
surfaces: ["live-ish-availability"],
|
|
});
|
|
});
|
|
|
|
it("keeps SDK/package boundary checks inside the plugin prerelease suite", () => {
|
|
const plan = createPluginPrereleaseTestPlan();
|
|
|
|
expect(plan.staticChecks.map((check) => check.checkName)).toEqual([
|
|
"checks-plugin-prerelease-package-boundary-compile",
|
|
"checks-plugin-prerelease-package-boundary-canary",
|
|
"checks-plugin-prerelease-live-ish-availability",
|
|
]);
|
|
});
|
|
|
|
it("uses kitchen-sink npm and ClawHub scenarios as the registry install canary", () => {
|
|
const lane = findLaneByName("kitchen-sink-plugin");
|
|
const script = readFileSync("scripts/e2e/kitchen-sink-plugin-docker.sh", "utf8");
|
|
const sweepScript = readFileSync("scripts/e2e/lib/kitchen-sink-plugin/sweep.sh", "utf8");
|
|
const assertionsScript = readFileSync(
|
|
"scripts/e2e/lib/kitchen-sink-plugin/assertions.mjs",
|
|
"utf8",
|
|
);
|
|
|
|
expect(lane).toEqual(
|
|
expect.objectContaining({
|
|
command: "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:kitchen-sink-plugin",
|
|
e2eImageKind: "functional",
|
|
name: "kitchen-sink-plugin",
|
|
resources: expect.arrayContaining(["npm"]),
|
|
stateScenario: "empty",
|
|
}),
|
|
);
|
|
expect(script).toContain("npm:@openclaw/kitchen-sink@0.1.5");
|
|
expect(script).toContain("npm-pinned-conformance");
|
|
expect(script).toContain("npm-pinned-adversarial");
|
|
expect(script).toContain("npm:@openclaw/kitchen-sink@beta");
|
|
expect(script).toContain("clawhub:@openclaw/kitchen-sink@latest");
|
|
expect(script).toContain("clawhub:@openclaw/kitchen-sink@beta");
|
|
expect(script).toContain(
|
|
"npm-to-clawhub|clawhub:@openclaw/kitchen-sink@latest|openclaw-kitchen-sink-fixture|clawhub|success|basic||${KITCHEN_SINK_NPM_SPEC}",
|
|
);
|
|
expect(script).toContain("scripts/e2e/lib/kitchen-sink-plugin/sweep.sh");
|
|
expect(sweepScript).toContain('plugins install "$KITCHEN_SINK_SPEC"');
|
|
expect(sweepScript).toContain('plugins install "$KITCHEN_SINK_PREINSTALL_SPEC"');
|
|
expect(sweepScript).toContain("assert-cutover-preinstalled");
|
|
expect(sweepScript).toContain('install_args+=("--force")');
|
|
expect(sweepScript).toContain("KITCHEN_SINK_PERSONALITY");
|
|
expect(sweepScript).toContain("OPENCLAW_KITCHEN_SINK_PERSONALITY");
|
|
expect(sweepScript).toContain('plugins uninstall "$KITCHEN_SINK_SPEC" --force');
|
|
const successScenario = sweepScript.slice(
|
|
sweepScript.indexOf("run_success_scenario()"),
|
|
sweepScript.indexOf("run_failure_scenario()"),
|
|
);
|
|
expect(successScenario.indexOf('plugins install "${install_args[@]}"')).toBeLessThan(
|
|
successScenario.indexOf("configure_kitchen_sink_runtime"),
|
|
);
|
|
expect(successScenario.indexOf("configure_kitchen_sink_runtime")).toBeLessThan(
|
|
successScenario.indexOf('plugins enable "$KITCHEN_SINK_ID"'),
|
|
);
|
|
expect(successScenario).toContain('plugins inspect "$KITCHEN_SINK_ID" --runtime --json');
|
|
expect(successScenario).toContain("plugins inspect --all --runtime --json");
|
|
expect(sweepScript).toContain("run_failure_scenario");
|
|
expect(assertionsScript).toContain("assertCutoverPreinstalled");
|
|
expect(assertionsScript).toContain("record.source !== source");
|
|
expect(assertionsScript).toContain("record.clawhubPackage !== packageName");
|
|
expect(assertionsScript).toContain("record.clawpackSha256");
|
|
expect(assertionsScript).toContain("record.artifactKind");
|
|
expect(assertionsScript).toContain("record.npmIntegrity");
|
|
expect(assertionsScript).toContain("assertClawHubExternalInstallContract");
|
|
expect(assertionsScript).toContain("expectedErrorMessages");
|
|
expect(assertionsScript).toContain(
|
|
'const INVALID_PROBE_DIAGNOSTIC_SURFACE_MODES = new Set(["full", "conformance", "adversarial"]);',
|
|
);
|
|
expect(assertionsScript).toContain("!INVALID_PROBE_DIAGNOSTIC_SURFACE_MODES.has(surfaceMode)");
|
|
expect(readFileSync("scripts/e2e/lib/clawhub-fixture-server.cjs", "utf8")).toContain(
|
|
'from "openclaw/plugin-sdk/plugin-entry"',
|
|
);
|
|
expect(readFileSync("scripts/e2e/lib/clawhub-fixture-server.cjs", "utf8")).toContain(
|
|
"X-ClawHub-Artifact-Sha256",
|
|
);
|
|
expect(script).toContain("docker stats --no-stream");
|
|
expect(sweepScript).toContain("scan_logs_for_unexpected_errors");
|
|
});
|
|
|
|
it("keeps the generic plugin Docker lane as an external install contract canary", () => {
|
|
const lane = findLaneByName("plugins");
|
|
const sweepScript = readFileSync("scripts/e2e/lib/plugins/sweep.sh", "utf8");
|
|
const clawhubScript = readFileSync("scripts/e2e/lib/plugins/clawhub.sh", "utf8");
|
|
const assertionsScript = readFileSync("scripts/e2e/lib/plugins/assertions.mjs", "utf8");
|
|
const fixtureServer = readFileSync("scripts/e2e/lib/clawhub-fixture-server.cjs", "utf8");
|
|
const prereleasePlan = createPluginPrereleaseTestPlan();
|
|
|
|
expect(lane).toEqual(
|
|
expect.objectContaining({
|
|
command: "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugins",
|
|
name: "plugins",
|
|
resources: expect.arrayContaining(["npm"]),
|
|
stateScenario: "empty",
|
|
}),
|
|
);
|
|
expect(prereleasePlan.surfaces).toContain("external-install-boundary");
|
|
expect(sweepScript).toContain("run_plugins_clawhub_scenario");
|
|
expect(clawhubScript).toContain('plugins install "$CLAWHUB_PLUGIN_SPEC"');
|
|
expect(assertionsScript).toContain("assertClawHubExternalInstallContract");
|
|
expect(assertionsScript).toContain('node_modules", "openclaw');
|
|
expect(fixtureServer).toContain('"is-number": "7.0.0"');
|
|
expect(fixtureServer).toContain('openclaw: ">=2026.4.11"');
|
|
expect(fixtureServer).toContain("/versions/${fixture.version}/artifact");
|
|
});
|
|
|
|
it("wires the full plugin prerelease plan into its release workflow", () => {
|
|
const workflow = readCiWorkflow();
|
|
const preflight = workflow.jobs.preflight;
|
|
const pluginWorkflow = readPluginPrereleaseWorkflow();
|
|
const pluginPreflight = pluginWorkflow.jobs.preflight;
|
|
const staticShard = pluginWorkflow.jobs["plugin-prerelease-static-shard"];
|
|
const nodeShard = pluginWorkflow.jobs["plugin-prerelease-node-shard"];
|
|
const extensionShard = pluginWorkflow.jobs["plugin-prerelease-extension-shard"];
|
|
const dockerSuite = pluginWorkflow.jobs["plugin-prerelease-docker-suite"];
|
|
const suite = pluginWorkflow.jobs["plugin-prerelease-suite"];
|
|
const releaseWorkflow = readFullReleaseValidationWorkflow();
|
|
const manifestScript = preflight.steps.find((step) => step.name === "Build CI manifest").run;
|
|
const manifestEnv = preflight.steps.find((step) => step.name === "Build CI manifest").env;
|
|
const pluginManifestScript = pluginPreflight.steps.find(
|
|
(step) => step.name === "Build plugin prerelease manifest",
|
|
).run;
|
|
const pluginManifestEnv = pluginPreflight.steps.find(
|
|
(step) => step.name === "Build plugin prerelease manifest",
|
|
).env;
|
|
const normalCiScript = releaseWorkflow.jobs.normal_ci.steps.find(
|
|
(step) => step.name === "Dispatch and monitor CI",
|
|
).run;
|
|
const pluginPrereleaseScript = releaseWorkflow.jobs.plugin_prerelease.steps.find(
|
|
(step) => step.name === "Dispatch and monitor plugin prerelease",
|
|
).run;
|
|
|
|
expect(workflow.jobs["plugin-prerelease-static-shard"]).toBeUndefined();
|
|
expect(workflow.jobs["plugin-prerelease-docker-suite"]).toBeUndefined();
|
|
expect(workflow.jobs["plugin-prerelease-suite"]).toBeUndefined();
|
|
expect(workflow.jobs["checks-node-extensions-shard"]).toBeUndefined();
|
|
expect(preflight.outputs).not.toHaveProperty("run_plugin_prerelease_suite");
|
|
expect(preflight.outputs).not.toHaveProperty("run_checks_node_extensions");
|
|
expect(staticShard).toMatchObject({
|
|
name: "${{ matrix.check_name }}",
|
|
"runs-on": "blacksmith-8vcpu-ubuntu-2404",
|
|
});
|
|
expect(workflow.on.workflow_dispatch.inputs.full_release_validation).toBeUndefined();
|
|
expect(workflow.on.workflow_dispatch.inputs.include_android).toMatchObject({
|
|
default: false,
|
|
type: "boolean",
|
|
});
|
|
expect(manifestEnv).toMatchObject({
|
|
OPENCLAW_CI_RUN_ANDROID:
|
|
"${{ github.event_name == 'workflow_dispatch' && inputs.include_android && 'true' || steps.changed_scope.outputs.run_android || 'false' }}",
|
|
});
|
|
expect(manifestEnv).not.toHaveProperty("OPENCLAW_CI_FULL_RELEASE_VALIDATION");
|
|
expect(manifestScript).toContain("includeReleaseOnlyPluginShards: false");
|
|
expect(manifestScript).not.toContain("plugin-prerelease-test-plan.mjs");
|
|
expect(workflow.jobs["check-shard"].strategy.matrix.include).toContainEqual({
|
|
check_name: "check-dependencies",
|
|
task: "dependencies",
|
|
runner: "ubuntu-24.04",
|
|
});
|
|
expect(
|
|
workflow.jobs["check-shard"].steps.find((step) => step.name === "Run check shard").run,
|
|
).toContain("pnpm deadcode:ci");
|
|
expect(normalCiScript).toContain(
|
|
'dispatch_and_wait ci.yml -f target_ref="$TARGET_SHA" -f include_android=true',
|
|
);
|
|
expect(normalCiScript).not.toContain("full_release_validation=true");
|
|
expect(pluginPrereleaseScript).toContain(
|
|
'dispatch_and_wait plugin-prerelease.yml -f target_ref="$TARGET_SHA" -f expected_sha="$TARGET_SHA" -f full_release_validation=true',
|
|
);
|
|
expect(pluginManifestScript).toContain("await import(");
|
|
expect(pluginManifestScript).toContain('"./scripts/lib/plugin-prerelease-test-plan.mjs"');
|
|
expect(pluginManifestScript).toContain('"./scripts/lib/extension-test-plan.mjs"');
|
|
expect(pluginManifestScript).toContain('"./scripts/lib/ci-node-test-plan.mjs"');
|
|
expect(pluginManifestScript).toContain('shard.shardName === "agentic-plugins"');
|
|
expect(pluginManifestScript).toContain(
|
|
"Plugin prerelease plan unavailable in target ref; skipping static and Docker plugin prerelease lanes.",
|
|
);
|
|
expect(pluginWorkflow.on.workflow_dispatch.inputs.target_ref).toMatchObject({
|
|
default: "main",
|
|
type: "string",
|
|
});
|
|
expect(pluginWorkflow.on.workflow_dispatch.inputs.full_release_validation).toMatchObject({
|
|
default: false,
|
|
type: "boolean",
|
|
});
|
|
expect(pluginManifestEnv).toMatchObject({
|
|
FULL_RELEASE_VALIDATION: "${{ inputs.full_release_validation && 'true' || 'false' }}",
|
|
});
|
|
expect(pluginManifestScript).toContain(
|
|
'const fullReleaseValidation = process.env.FULL_RELEASE_VALIDATION === "true";',
|
|
);
|
|
expect(pluginManifestScript).toContain(
|
|
"const runDocker = fullReleaseValidation && dockerLanes.length > 0;",
|
|
);
|
|
expect(pluginPreflight.outputs).toMatchObject({
|
|
checkout_revision: "${{ steps.manifest.outputs.checkout_revision }}",
|
|
plugin_prerelease_docker_lanes:
|
|
"${{ steps.manifest.outputs.plugin_prerelease_docker_lanes }}",
|
|
plugin_prerelease_extension_matrix:
|
|
"${{ steps.manifest.outputs.plugin_prerelease_extension_matrix }}",
|
|
plugin_prerelease_node_matrix: "${{ steps.manifest.outputs.plugin_prerelease_node_matrix }}",
|
|
plugin_prerelease_static_matrix:
|
|
"${{ steps.manifest.outputs.plugin_prerelease_static_matrix }}",
|
|
run_plugin_prerelease_docker: "${{ steps.manifest.outputs.run_plugin_prerelease_docker }}",
|
|
run_plugin_prerelease_extensions:
|
|
"${{ steps.manifest.outputs.run_plugin_prerelease_extensions }}",
|
|
run_plugin_prerelease_node: "${{ steps.manifest.outputs.run_plugin_prerelease_node }}",
|
|
run_plugin_prerelease_static: "${{ steps.manifest.outputs.run_plugin_prerelease_static }}",
|
|
run_plugin_prerelease_suite: "${{ steps.manifest.outputs.run_plugin_prerelease_suite }}",
|
|
});
|
|
expect(staticShard.strategy.matrix).toBe(
|
|
"${{ fromJson(needs.preflight.outputs.plugin_prerelease_static_matrix) }}",
|
|
);
|
|
expect(nodeShard.strategy.matrix).toBe(
|
|
"${{ fromJson(needs.preflight.outputs.plugin_prerelease_node_matrix) }}",
|
|
);
|
|
expect(extensionShard.if).toBe(
|
|
"needs.preflight.outputs.run_plugin_prerelease_extensions == 'true'",
|
|
);
|
|
expect(extensionShard.strategy.matrix).toBe(
|
|
"${{ fromJson(needs.preflight.outputs.plugin_prerelease_extension_matrix) }}",
|
|
);
|
|
expect(
|
|
staticShard.steps.find((step) => step.name === "Run plugin prerelease static shard").run,
|
|
).toContain('bash -c "$PLUGIN_PRERELEASE_COMMAND"');
|
|
expect(dockerSuite).toMatchObject({
|
|
if: "${{ inputs.full_release_validation && needs.preflight.outputs.run_plugin_prerelease_docker == 'true' }}",
|
|
needs: ["preflight"],
|
|
uses: "./.github/workflows/openclaw-live-and-e2e-checks-reusable.yml",
|
|
with: {
|
|
docker_lanes: "${{ needs.preflight.outputs.plugin_prerelease_docker_lanes }}",
|
|
include_live_suites: false,
|
|
include_openwebui: false,
|
|
include_release_path_suites: false,
|
|
include_repo_e2e: false,
|
|
live_models_only: false,
|
|
ref: "${{ needs.preflight.outputs.checkout_revision }}",
|
|
targeted_docker_lane_group_size: 4,
|
|
},
|
|
});
|
|
expect(dockerSuite.secrets).toBeUndefined();
|
|
expect(suite.needs).toEqual([
|
|
"preflight",
|
|
"plugin-prerelease-static-shard",
|
|
"plugin-prerelease-node-shard",
|
|
"plugin-prerelease-extension-shard",
|
|
"plugin-prerelease-docker-suite",
|
|
]);
|
|
});
|
|
|
|
it("keeps release-check reruns independent while cancelling superseded umbrella runs", () => {
|
|
const releaseChecksWorkflow = parse(
|
|
readFileSync(".github/workflows/openclaw-release-checks.yml", "utf8"),
|
|
);
|
|
const fullReleaseWorkflow = readFullReleaseValidationWorkflow();
|
|
|
|
expect(releaseChecksWorkflow.concurrency).toEqual({
|
|
group:
|
|
"openclaw-release-checks-${{ inputs.expected_sha || inputs.ref }}-${{ inputs.rerun_group }}",
|
|
"cancel-in-progress": false,
|
|
});
|
|
expect(fullReleaseWorkflow.concurrency).toEqual({
|
|
group: "full-release-validation-${{ inputs.ref }}-${{ inputs.rerun_group }}",
|
|
"cancel-in-progress": "${{ inputs.ref == 'main' && inputs.rerun_group == 'all' }}",
|
|
});
|
|
expect(releaseChecksWorkflow.jobs.resolve_target["runs-on"]).toBe("ubuntu-24.04");
|
|
expect(releaseChecksWorkflow.jobs.prepare_release_package["runs-on"]).toBe("ubuntu-24.04");
|
|
expect(releaseChecksWorkflow.jobs.summary["runs-on"]).toBe("ubuntu-24.04");
|
|
for (const jobName of [
|
|
"resolve_target",
|
|
"normal_ci",
|
|
"plugin_prerelease",
|
|
"release_checks",
|
|
"prepare_release_package",
|
|
"npm_telegram",
|
|
"summary",
|
|
]) {
|
|
expect(fullReleaseWorkflow.jobs[jobName]["runs-on"]).toBe("ubuntu-24.04");
|
|
}
|
|
});
|
|
|
|
it("keeps the live-ish availability check redacted", () => {
|
|
const output = execFileSync(
|
|
process.execPath,
|
|
["scripts/plugin-prerelease-liveish-matrix.mjs"],
|
|
{
|
|
encoding: "utf8",
|
|
env: {
|
|
DISCORD_TOKEN: "discord-token-should-not-print",
|
|
OPENAI_API_KEY: "openai-token-should-not-print",
|
|
},
|
|
},
|
|
);
|
|
|
|
expect(output).toContain("provider-openai: present (OPENAI_API_KEY, OPENAI_BASE_URL)");
|
|
expect(output).toContain("channel-discord: present (DISCORD_TOKEN, OPENCLAW_DISCORD_TOKEN)");
|
|
expect(output).not.toContain("openai-token-should-not-print");
|
|
expect(output).not.toContain("discord-token-should-not-print");
|
|
});
|
|
});
|