VS Code gets the gateway's models from an extension the gateway serves
CI / test (push) Successful in 8m43s
CI / test (push) Successful in 8m43s
llmgw's own extension for VS Code, plain JavaScript with no dependencies, packed into a .vsix by the binary at /setup/vscode/llmgw.vsix with the serving gateway's address in it, so it needs no build, no Node and no marketplace. It registers llmgw as a language-model provider: the chat's model picker, agent mode and any extension that asks VS Code for a model get every chat model the key may use, sized and with tool and vision capabilities as the catalog says, streamed, tool calls both ways. VS Code is a tool on the account page's Connect your editor: a key, the install command, and an Open in VS Code link that hands the key to the extension's secret storage. The extension's Sign In opens the page with a state of its own; a link without it is confirmed, the gateway named, since it decides where code is sent. Tested in VS Code 1.138 against the PoC: 27 models listed, an answer from qwen2.5-0.5b-instruct, and qwen3-coder-30b calling a tool and using its result (internal/vscodeext/e2e). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fea5774fe5
commit
38dfe268c9
17 files changed
+1412
-9
No files matched your search
@@ -0,0 +1,101 @@
|
||||
'use strict';
|
||||
|
||||
// The extension in a real VS Code, against a real gateway: VS Code's own
|
||||
// language-model API lists the gateway's models, gets an answer from one,
|
||||
// and has one call a tool and read its result back.
|
||||
//
|
||||
// go run ./internal/vscodeext/e2e/pack -out /tmp/llmgw-vsix # unpacked extension
|
||||
// LLMGW_TEST_GATEWAY=https://gw LLMGW_TEST_KEY=sk-llmgw-… \
|
||||
// LLMGW_TEST_CHAT=qwen2.5-0.5b-instruct LLMGW_TEST_TOOLS=qwen3-coder-30b \
|
||||
// LLMGW_TEST_OUT=/tmp/e2e.json \
|
||||
// code --user-data-dir /tmp/vsc-data --extensions-dir /tmp/vsc-ext \
|
||||
// --extensionDevelopmentPath=/tmp/llmgw-vsix/extension \
|
||||
// --extensionTestsPath=$PWD/internal/vscodeext/e2e/run.js
|
||||
//
|
||||
// It writes what happened to LLMGW_TEST_OUT and exits non-zero on failure.
|
||||
|
||||
const vscode = require('vscode');
|
||||
const fs = require('fs');
|
||||
|
||||
const env = (k) => {
|
||||
const v = process.env[k];
|
||||
if (!v) throw new Error(`${k} is not set`);
|
||||
return v;
|
||||
};
|
||||
|
||||
async function collect(response) {
|
||||
const out = { text: '', calls: [] };
|
||||
for await (const part of response.stream) {
|
||||
if (part instanceof vscode.LanguageModelTextPart) out.text += part.value;
|
||||
else if (part instanceof vscode.LanguageModelToolCallPart) out.calls.push(part);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function within(ms, what, p) {
|
||||
return Promise.race([p, new Promise((_, rej) => setTimeout(() => rej(new Error(`${what}: no answer in ${ms / 1000}s`)), ms))]);
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const report = { steps: [] };
|
||||
const out = process.env.LLMGW_TEST_OUT;
|
||||
// Written after every step, so a run that hangs still says how far it got.
|
||||
const step = (name, detail) => {
|
||||
report.steps.push({ name, at: new Date().toISOString(), ...detail });
|
||||
if (out) fs.writeFileSync(out, JSON.stringify(report, null, 2));
|
||||
};
|
||||
const send = (model, msgs, opts) => model.sendRequest(msgs, { justification: 'llmgw end-to-end test', ...opts }, new vscode.CancellationTokenSource().token);
|
||||
try {
|
||||
const ext = vscode.extensions.getExtension('llmgw.llmgw');
|
||||
if (!ext) throw new Error('the llmgw extension is not loaded');
|
||||
const api = await ext.activate();
|
||||
await api.connect(env('LLMGW_TEST_GATEWAY'), env('LLMGW_TEST_KEY'));
|
||||
step('connected', { gateway: env('LLMGW_TEST_GATEWAY') });
|
||||
|
||||
let models = [];
|
||||
for (let i = 0; i < 20 && !models.length; i++) {
|
||||
models = await vscode.lm.selectChatModels({ vendor: 'llmgw' });
|
||||
if (!models.length) await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
step('models', { count: models.length, ids: models.map((m) => m.id) });
|
||||
if (!models.length) throw new Error('VS Code lists no llmgw models');
|
||||
|
||||
const chat = models.find((m) => m.id === env('LLMGW_TEST_CHAT'));
|
||||
if (!chat) throw new Error(`${env('LLMGW_TEST_CHAT')} is not among the models`);
|
||||
step('asking', { model: chat.id });
|
||||
const hello = await within(180000, 'chat', send(chat, [vscode.LanguageModelChatMessage.User('Reply with the single word: hello')], {}).then(collect));
|
||||
step('chat', { model: chat.id, text: hello.text, maxInputTokens: chat.maxInputTokens });
|
||||
if (!hello.text.trim()) throw new Error('the chat model answered nothing');
|
||||
|
||||
const toolModel = models.find((m) => m.id === env('LLMGW_TEST_TOOLS'));
|
||||
if (!toolModel) throw new Error(`${env('LLMGW_TEST_TOOLS')} is not among the models`);
|
||||
const tools = [{
|
||||
name: 'get_build_number',
|
||||
description: 'Returns the current build number of the project.',
|
||||
inputSchema: { type: 'object', properties: { project: { type: 'string' } }, required: ['project'] },
|
||||
}];
|
||||
const ask = vscode.LanguageModelChatMessage.User('What is the current build number of the project "llmgw"? Use the tool.');
|
||||
step('asking', { model: toolModel.id });
|
||||
const first = await within(300000, 'tool call', send(toolModel, [ask], { tools }).then(collect));
|
||||
step('tool call', { model: toolModel.id, calls: first.calls.map((c) => ({ name: c.name, input: c.input })), text: first.text });
|
||||
if (!first.calls.length) throw new Error('the tool model made no tool call');
|
||||
|
||||
const c = first.calls[0];
|
||||
const second = await within(300000, 'tool result', send(toolModel, [
|
||||
ask,
|
||||
vscode.LanguageModelChatMessage.Assistant([new vscode.LanguageModelToolCallPart(c.callId, c.name, c.input)]),
|
||||
vscode.LanguageModelChatMessage.User([new vscode.LanguageModelToolResultPart(c.callId, [new vscode.LanguageModelTextPart('4711')])]),
|
||||
], { tools }).then(collect));
|
||||
step('tool result', { text: second.text });
|
||||
if (!second.text.includes('4711')) throw new Error('the answer after the tool result does not use it');
|
||||
report.ok = true;
|
||||
} catch (err) {
|
||||
report.ok = false;
|
||||
report.error = String(err && err.stack ? err.stack : err);
|
||||
}
|
||||
if (out) fs.writeFileSync(out, JSON.stringify(report, null, 2));
|
||||
else console.log(JSON.stringify(report, null, 2));
|
||||
if (!report.ok) throw new Error(report.error);
|
||||
}
|
||||
|
||||
module.exports = { run };
|
||||
Reference in new issue
Block a user