'use strict'; // The extension in a real VS Code, against a real gateway: VS Code's own // language-model API lists the gateway's models, gets an answer from one, // and has one call a tool and read its result back. // // go run ./internal/vscodeext/e2e/pack -out /tmp/llmgw-vsix # unpacked extension // LLMGW_TEST_GATEWAY=https://gw LLMGW_TEST_KEY=sk-llmgw-… \ // LLMGW_TEST_CHAT=qwen2.5-0.5b-instruct LLMGW_TEST_TOOLS=qwen3-coder-30b \ // LLMGW_TEST_OUT=/tmp/e2e.json \ // code --user-data-dir /tmp/vsc-data --extensions-dir /tmp/vsc-ext \ // --extensionDevelopmentPath=/tmp/llmgw-vsix/extension \ // --extensionTestsPath=$PWD/internal/vscodeext/e2e/run.js // // It writes what happened to LLMGW_TEST_OUT and exits non-zero on failure. const vscode = require('vscode'); const fs = require('fs'); const env = (k) => { const v = process.env[k]; if (!v) throw new Error(`${k} is not set`); return v; }; async function collect(response) { const out = { text: '', calls: [] }; for await (const part of response.stream) { if (part instanceof vscode.LanguageModelTextPart) out.text += part.value; else if (part instanceof vscode.LanguageModelToolCallPart) out.calls.push(part); } return out; } function within(ms, what, p) { return Promise.race([p, new Promise((_, rej) => setTimeout(() => rej(new Error(`${what}: no answer in ${ms / 1000}s`)), ms))]); } async function run() { const report = { steps: [] }; const out = process.env.LLMGW_TEST_OUT; // Written after every step, so a run that hangs still says how far it got. const step = (name, detail) => { report.steps.push({ name, at: new Date().toISOString(), ...detail }); if (out) fs.writeFileSync(out, JSON.stringify(report, null, 2)); }; const send = (model, msgs, opts) => model.sendRequest(msgs, { justification: 'llmgw end-to-end test', ...opts }, new vscode.CancellationTokenSource().token); try { const ext = vscode.extensions.getExtension('llmgw.llmgw'); if (!ext) throw new Error('the llmgw extension is not loaded'); const api = await ext.activate(); await api.connect(env('LLMGW_TEST_GATEWAY'), env('LLMGW_TEST_KEY')); step('connected', { gateway: env('LLMGW_TEST_GATEWAY') }); let models = []; for (let i = 0; i < 20 && !models.length; i++) { models = await vscode.lm.selectChatModels({ vendor: 'llmgw' }); if (!models.length) await new Promise((r) => setTimeout(r, 500)); } step('models', { count: models.length, ids: models.map((m) => m.id) }); if (!models.length) throw new Error('VS Code lists no llmgw models'); const chat = models.find((m) => m.id === env('LLMGW_TEST_CHAT')); if (!chat) throw new Error(`${env('LLMGW_TEST_CHAT')} is not among the models`); step('asking', { model: chat.id }); const hello = await within(180000, 'chat', send(chat, [vscode.LanguageModelChatMessage.User('Reply with the single word: hello')], {}).then(collect)); step('chat', { model: chat.id, text: hello.text, maxInputTokens: chat.maxInputTokens }); if (!hello.text.trim()) throw new Error('the chat model answered nothing'); const toolModel = models.find((m) => m.id === env('LLMGW_TEST_TOOLS')); if (!toolModel) throw new Error(`${env('LLMGW_TEST_TOOLS')} is not among the models`); const tools = [{ name: 'get_build_number', description: 'Returns the current build number of the project.', inputSchema: { type: 'object', properties: { project: { type: 'string' } }, required: ['project'] }, }]; const ask = vscode.LanguageModelChatMessage.User('What is the current build number of the project "llmgw"? Use the tool.'); step('asking', { model: toolModel.id }); const first = await within(300000, 'tool call', send(toolModel, [ask], { tools }).then(collect)); step('tool call', { model: toolModel.id, calls: first.calls.map((c) => ({ name: c.name, input: c.input })), text: first.text }); if (!first.calls.length) throw new Error('the tool model made no tool call'); const c = first.calls[0]; const second = await within(300000, 'tool result', send(toolModel, [ ask, vscode.LanguageModelChatMessage.Assistant([new vscode.LanguageModelToolCallPart(c.callId, c.name, c.input)]), vscode.LanguageModelChatMessage.User([new vscode.LanguageModelToolResultPart(c.callId, [new vscode.LanguageModelTextPart('4711')])]), ], { tools }).then(collect)); step('tool result', { text: second.text }); if (!second.text.includes('4711')) throw new Error('the answer after the tool result does not use it'); report.ok = true; } catch (err) { report.ok = false; report.error = String(err && err.stack ? err.stack : err); } if (out) fs.writeFileSync(out, JSON.stringify(report, null, 2)); else console.log(JSON.stringify(report, null, 2)); if (!report.ok) throw new Error(report.error); } module.exports = { run };