@mcpjam/sdk weekly downloads statistics

@mcpjam/sdk

Use the MCPJam SDK to write unit tests and evals for your MCP server.

Installation

npm install @mcpjam/sdk

Compatible with your favorite testing framework like Jest and Vitest

Quick Start

Unit Test

Test the individual parts, request response flow of your MCP server. MCP unit tests are deterministic.

import { MCPClientManager, assertCallToolResult } from "@mcpjam/sdk";

describe("Everything MCP example", () => {
  let manager: MCPClientManager;

  beforeAll(async () => {
    manager = new MCPClientManager();
    await manager.connectToServer("everything", {
      command: "npx",
      args: ["-y", "@modelcontextprotocol/server-everything"],
    });
  });

  afterAll(async () => {
    await manager.disconnectServer("everything");
  });

  test("server has expected tools", async () => {
    const tools = await manager.listTools("everything");
    expect(tools.tools.map((t) => t.name)).toContain("get-sum");
  });

  test("get-sum tool returns correct result", async () => {
    // `executeTool` can also resolve to a task envelope, and a content block
    // can be an image or a resource, so narrow both before reading the text.
    const result = assertCallToolResult(
      await manager.executeTool("everything", "get-sum", { a: 2, b: 3 }),
    );
    const [block] = result.content;
    if (block.type !== "text") throw new Error("Expected a text block");
    expect(block.text).toBe("The sum of 2 and 3 is 5.");
  });
});

MCP evals

Test that an LLM correctly understands how to use your MCP server. Evals are non-deterministic and multiple runs are needed.

Heads up: renames in 1.11. TestAgent → HostRunner, EvalAgent → HostExecutor, .prompt() → .run(), Host.addServer() → Host.requireServer(). No deprecation aliases. See the changelog for the codemod.

import { MCPClientManager, HostRunner, EvalTest } from "@mcpjam/sdk";

describe("Asana MCP Evals", () => {
  let manager: MCPClientManager;
  let runner: HostRunner;

  beforeAll(async () => {
    manager = new MCPClientManager();
    await manager.connectToServer("asana", {
      url: "https://mcp.asana.com/sse",
      requestInit: {
        headers: { Authorization: `Bearer ${process.env.ASANA_TOKEN}` },
      },
    });

    runner = new HostRunner({
      tools: await manager.getToolsForAiSdk(["asana"]),
      model: "openai/gpt-4o",
      apiKey: process.env.OPENAI_API_KEY!,
    });
  });

  afterAll(async () => {
    await manager.disconnectServer("asana");
  });

  // Single-turn eval
  test("list workspaces > 80% accuracy", async () => {
    const evalTest = new EvalTest({
      id: "c_list_workspaces",
      name: "list-workspaces",
      test: async (runner) => {
        const result = await runner.run("Show me all my Asana workspaces");
        return result.hasToolCall("asana_list_workspaces");
      },
    });

    await evalTest.run(runner, {
      iterations: 10,
      onFailure: (report) => console.error(report), // Print the report when a test iteration fails.
    });

    expect(evalTest.accuracy()).toBeGreaterThan(0.8); // Pass threshold
  });

  // Multi-turn eval
  test("get user then list projects > 80% accuracy", async () => {
    const evalTest = new EvalTest({
      id: "c_user_then_projects",
      name: "user-then-projects",
      test: async (runner) => {
        const r1 = await runner.run("Who am I in Asana?");
        if (!r1.hasToolCall("asana_get_user")) return false;

        const r2 = await runner.run("Now list my projects", {
          context: [r1],
        }); // Continue the conversation from the previous prompt
        return r2.hasToolCall("asana_get_projects");
      },
    });

    await evalTest.run(runner, {
      iterations: 5,
      onFailure: (report) => console.error(report),
    });

    expect(evalTest.accuracy()).toBeGreaterThan(0.8);
  });

  // Validating tool arguments
  test("search tasks passes correct workspace_gid", async () => {
    const evalTest = new EvalTest({
      id: "c_search_args",
      name: "search-args",
      test: async (runner) => {
        const result = await runner.run(
          "Search for tasks containing 'bug' in my workspace"
        );
        const args = result.getToolArguments("asana_search_tasks");
        return (
          result.hasToolCall("asana_search_tasks") &&
          typeof args?.workspace_gid === "string"
        );
      },
    });

    await evalTest.run(runner, {
      iterations: 5,
      onFailure: (report) => console.error(report),
    });

    expect(evalTest.accuracy()).toBeGreaterThan(0.8);
  });
});

Host + HostRuntime: bring your own runtime

Host is the portable host-configuration spec — the same object the MCPJam Inspector uses to drive its Playground and eval suites. You can build one in your own code and run evals against it, so the inspector and your CI exercise the same host behavior (style, model, MCP profile, sandbox/permission policy, OpenAI-Apps compat, tool-visibility policy, etc.).

import { MCPClientManager, Host, EvalTest } from "@mcpjam/sdk";

const manager = new MCPClientManager();
await manager.connectToServer("everything", {
  command: "npx",
  args: ["-y", "@modelcontextprotocol/server-everything"],
});

// Build the spec — Host is just an editable wrapper around the canonical JSON.
const host = new Host({
  style: "mcpjam",
  model: "openai/gpt-4o",
  systemPrompt: "You are a helpful assistant.",
}).requireServer("everything");

// Bind the spec to a live MCP manager. `apiKey` lives on the runtime, not per-call.
const runtime = host.withManager(manager, {
  apiKey: process.env.OPENAI_API_KEY!,
});

const evalTest = new EvalTest({
  id: "c_add",
  name: "add",
  test: async (runner) => (await runner.run("Add 2 and 3")).hasToolCall("add"),
});
await evalTest.run(runtime, { iterations: 10 });
```…