llmgw/internal/vscodeext/e2e/run.js

'use strict';

// The extension in a real VS Code, against a real gateway: VS Code's own
// language-model API lists the gateway's models, gets an answer from one,
// and has one call a tool and read its result back.
//
//   go run ./internal/vscodeext/e2e/pack -out /tmp/llmgw-vsix   # unpacked extension
//   LLMGW_TEST_GATEWAY=https://gw LLMGW_TEST_KEY=sk-llmgw-… \
//   LLMGW_TEST_CHAT=qwen2.5-0.5b-instruct LLMGW_TEST_TOOLS=qwen3-coder-30b \
//   LLMGW_TEST_OUT=/tmp/e2e.json \
//   code --user-data-dir /tmp/vsc-data --extensions-dir /tmp/vsc-ext \
//     --extensionDevelopmentPath=/tmp/llmgw-vsix/extension \
//     --extensionTestsPath=$PWD/internal/vscodeext/e2e/run.js
//
// It writes what happened to LLMGW_TEST_OUT and exits non-zero on failure.

const vscode = require('vscode');
const fs = require('fs');

const env = (k) => {
  const v = process.env[k];
  if (!v) throw new Error(`${k} is not set`);
  return v;
};

async function collect(response) {
  const out = { text: '', calls: [] };
  for await (const part of response.stream) {
    if (part instanceof vscode.LanguageModelTextPart) out.text += part.value;
    else if (part instanceof vscode.LanguageModelToolCallPart) out.calls.push(part);
  }
  return out;
}

function within(ms, what, p) {
  return Promise.race([p, new Promise((_, rej) => setTimeout(() => rej(new Error(`${what}: no answer in ${ms / 1000}s`)), ms))]);
}

async function run() {
  const report = { steps: [] };
  const out = process.env.LLMGW_TEST_OUT;
  // Written after every step, so a run that hangs still says how far it got.
  const step = (name, detail) => {
    report.steps.push({ name, at: new Date().toISOString(), ...detail });
    if (out) fs.writeFileSync(out, JSON.stringify(report, null, 2));
  };
  const send = (model, msgs, opts) => model.sendRequest(msgs, { justification: 'llmgw end-to-end test', ...opts }, new vscode.CancellationTokenSource().token);
  try {
    const ext = vscode.extensions.getExtension('llmgw.llmgw');
    if (!ext) throw new Error('the llmgw extension is not loaded');
    const api = await ext.activate();
    await api.connect(env('LLMGW_TEST_GATEWAY'), env('LLMGW_TEST_KEY'));
    step('connected', { gateway: env('LLMGW_TEST_GATEWAY') });

    let models = [];
    for (let i = 0; i < 20 && !models.length; i++) {
      models = await vscode.lm.selectChatModels({ vendor: 'llmgw' });
      if (!models.length) await new Promise((r) => setTimeout(r, 500));
    }
    step('models', { count: models.length, ids: models.map((m) => m.id) });
    if (!models.length) throw new Error('VS Code lists no llmgw models');

    const chat = models.find((m) => m.id === env('LLMGW_TEST_CHAT'));
    if (!chat) throw new Error(`${env('LLMGW_TEST_CHAT')} is not among the models`);
    step('asking', { model: chat.id });
    const hello = await within(180000, 'chat', send(chat, [vscode.LanguageModelChatMessage.User('Reply with the single word: hello')], {}).then(collect));
    step('chat', { model: chat.id, text: hello.text, maxInputTokens: chat.maxInputTokens });
    if (!hello.text.trim()) throw new Error('the chat model answered nothing');

    const toolModel = models.find((m) => m.id === env('LLMGW_TEST_TOOLS'));
    if (!toolModel) throw new Error(`${env('LLMGW_TEST_TOOLS')} is not among the models`);
    const tools = [{
      name: 'get_build_number',
      description: 'Returns the current build number of the project.',
      inputSchema: { type: 'object', properties: { project: { type: 'string' } }, required: ['project'] },
    }];
    const ask = vscode.LanguageModelChatMessage.User('What is the current build number of the project "llmgw"? Use the tool.');
    step('asking', { model: toolModel.id });
    const first = await within(300000, 'tool call', send(toolModel, [ask], { tools }).then(collect));
    step('tool call', { model: toolModel.id, calls: first.calls.map((c) => ({ name: c.name, input: c.input })), text: first.text });
    if (!first.calls.length) throw new Error('the tool model made no tool call');

    const c = first.calls[0];
    const second = await within(300000, 'tool result', send(toolModel, [
      ask,
      vscode.LanguageModelChatMessage.Assistant([new vscode.LanguageModelToolCallPart(c.callId, c.name, c.input)]),
      vscode.LanguageModelChatMessage.User([new vscode.LanguageModelToolResultPart(c.callId, [new vscode.LanguageModelTextPart('4711')])]),
    ], { tools }).then(collect));
    step('tool result', { text: second.text });
    if (!second.text.includes('4711')) throw new Error('the answer after the tool result does not use it');
    report.ok = true;
  } catch (err) {
    report.ok = false;
    report.error = String(err && err.stack ? err.stack : err);
  }
  if (out) fs.writeFileSync(out, JSON.stringify(report, null, 2));
  else console.log(JSON.stringify(report, null, 2));
  if (!report.ok) throw new Error(report.error);
}

module.exports = { run };