LoRA adapters trained on the nodes: llmgw ctl train
A job names the catalog model the adapter is for, the safetensors weights to train on, the node and the MiB of a card it needs, and question/answer pairs. The control plane sends it when the node's agent is there and nothing else trains on it; the agent trains with its installed vLLM's torch and a pinned overlay of peft and accelerate (no second environment), as the engines' user, on the card with that much free — beside an engine, if it leaves that much — converts the adapter to GGUF with the installed llama.cpp's own conversion code when the model is served by llama.cpp, and sends what it made back in pieces, kept in the store to download. Progress, loss and peak card memory are reported as it goes; a node that goes fails its job. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ae95ac8261
commit
79b3e9f887
24 files changed
+3185
-72
No files matched your search
+502
-71
@@ -106,6 +106,8 @@ type AgentMessage struct {
|
||||
// *AgentMessage_FabricTest
|
||||
// *AgentMessage_Upgrade
|
||||
// *AgentMessage_KvEvents
|
||||
// *AgentMessage_TrainStatus
|
||||
// *AgentMessage_TrainArtifact
|
||||
Msg isAgentMessage_Msg `protobuf_oneof:"msg"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -220,6 +222,24 @@ func (x *AgentMessage) GetKvEvents() *KVEvents {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AgentMessage) GetTrainStatus() *TrainStatus {
|
||||
if x != nil {
|
||||
if x, ok := x.Msg.(*AgentMessage_TrainStatus); ok {
|
||||
return x.TrainStatus
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *AgentMessage) GetTrainArtifact() *TrainArtifact {
|
||||
if x != nil {
|
||||
if x, ok := x.Msg.(*AgentMessage_TrainArtifact); ok {
|
||||
return x.TrainArtifact
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type isAgentMessage_Msg interface {
|
||||
isAgentMessage_Msg()
|
||||
}
|
||||
@@ -256,6 +276,14 @@ type AgentMessage_KvEvents struct {
|
||||
KvEvents *KVEvents `protobuf:"bytes,8,opt,name=kv_events,json=kvEvents,proto3,oneof"`
|
||||
}
|
||||
|
||||
type AgentMessage_TrainStatus struct {
|
||||
TrainStatus *TrainStatus `protobuf:"bytes,9,opt,name=train_status,json=trainStatus,proto3,oneof"`
|
||||
}
|
||||
|
||||
type AgentMessage_TrainArtifact struct {
|
||||
TrainArtifact *TrainArtifact `protobuf:"bytes,10,opt,name=train_artifact,json=trainArtifact,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (*AgentMessage_Register) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_Heartbeat) isAgentMessage_Msg() {}
|
||||
@@ -272,6 +300,10 @@ func (*AgentMessage_Upgrade) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_KvEvents) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_TrainStatus) isAgentMessage_Msg() {}
|
||||
|
||||
func (*AgentMessage_TrainArtifact) isAgentMessage_Msg() {}
|
||||
|
||||
// KVEvents is a change to what one vLLM replica's prefix cache holds, read
|
||||
// by the agent from the engine's KV cache events and sent on for the router's
|
||||
// precise index (plan §7.5 tier 2).
|
||||
@@ -392,6 +424,8 @@ type ControlMessage struct {
|
||||
// *ControlMessage_Firewall
|
||||
// *ControlMessage_RestartAgent
|
||||
// *ControlMessage_AgentSettings
|
||||
// *ControlMessage_TrainStart
|
||||
// *ControlMessage_TrainCancel
|
||||
Msg isControlMessage_Msg `protobuf_oneof:"msg"`
|
||||
// trace is the W3C trace context (traceparent, and tracestate when there
|
||||
// is one) of the control plane's span that sent this command, so what the
|
||||
@@ -566,6 +600,24 @@ func (x *ControlMessage) GetAgentSettings() *AgentSettings {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *ControlMessage) GetTrainStart() *TrainStart {
|
||||
if x != nil {
|
||||
if x, ok := x.Msg.(*ControlMessage_TrainStart); ok {
|
||||
return x.TrainStart
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *ControlMessage) GetTrainCancel() *TrainCancel {
|
||||
if x != nil {
|
||||
if x, ok := x.Msg.(*ControlMessage_TrainCancel); ok {
|
||||
return x.TrainCancel
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *ControlMessage) GetTrace() map[string]string {
|
||||
if x != nil {
|
||||
return x.Trace
|
||||
@@ -633,6 +685,14 @@ type ControlMessage_AgentSettings struct {
|
||||
AgentSettings *AgentSettings `protobuf:"bytes,15,opt,name=agent_settings,json=agentSettings,proto3,oneof"`
|
||||
}
|
||||
|
||||
type ControlMessage_TrainStart struct {
|
||||
TrainStart *TrainStart `protobuf:"bytes,16,opt,name=train_start,json=trainStart,proto3,oneof"`
|
||||
}
|
||||
|
||||
type ControlMessage_TrainCancel struct {
|
||||
TrainCancel *TrainCancel `protobuf:"bytes,17,opt,name=train_cancel,json=trainCancel,proto3,oneof"`
|
||||
}
|
||||
|
||||
func (*ControlMessage_Start) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_Stop) isControlMessage_Msg() {}
|
||||
@@ -661,6 +721,10 @@ func (*ControlMessage_RestartAgent) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_AgentSettings) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_TrainStart) isControlMessage_Msg() {}
|
||||
|
||||
func (*ControlMessage_TrainCancel) isControlMessage_Msg() {}
|
||||
|
||||
// RestartAgent has the agent start again, as llmgw restart asks of every
|
||||
// node: its engines keep serving and are taken over again.
|
||||
type RestartAgent struct {
|
||||
@@ -4699,11 +4763,335 @@ func (x *LoraAdapter) GetSource() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// TrainStart has a node train a LoRA adapter (BACKLOG F4): on a base model's
|
||||
// weights from its registry, on question/answer pairs, with a Python trainer
|
||||
// the agent carries, in an environment it makes over an installed engine's.
|
||||
// One at a time per node.
|
||||
type TrainStart struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
|
||||
// base_repo and base_revision name the base model's weights on its
|
||||
// registry (Hugging Face): safetensors, not the GGUF a catalog may serve.
|
||||
BaseRepo string `protobuf:"bytes,2,opt,name=base_repo,json=baseRepo,proto3" json:"base_repo,omitempty"`
|
||||
BaseRevision string `protobuf:"bytes,3,opt,name=base_revision,json=baseRevision,proto3" json:"base_revision,omitempty"`
|
||||
// data is the training pairs, NDJSON of {"q", "a"}.
|
||||
Data []byte `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"`
|
||||
// params is the trainer's settings as JSON: rank, alpha, epochs, lr,
|
||||
// batch, max_len.
|
||||
Params string `protobuf:"bytes,5,opt,name=params,proto3" json:"params,omitempty"`
|
||||
// gpu_memory_mib is what the job needs of a card; the agent starts it
|
||||
// only when one has that much free, which lets it run beside an engine.
|
||||
GpuMemoryMib int32 `protobuf:"varint,6,opt,name=gpu_memory_mib,json=gpuMemoryMib,proto3" json:"gpu_memory_mib,omitempty"`
|
||||
// gguf asks for the adapter in llama.cpp's format too.
|
||||
Gguf bool `protobuf:"varint,7,opt,name=gguf,proto3" json:"gguf,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *TrainStart) Reset() {
|
||||
*x = TrainStart{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[44]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *TrainStart) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*TrainStart) ProtoMessage() {}
|
||||
|
||||
func (x *TrainStart) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[44]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use TrainStart.ProtoReflect.Descriptor instead.
|
||||
func (*TrainStart) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{44}
|
||||
}
|
||||
|
||||
func (x *TrainStart) GetJobId() string {
|
||||
if x != nil {
|
||||
return x.JobId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *TrainStart) GetBaseRepo() string {
|
||||
if x != nil {
|
||||
return x.BaseRepo
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *TrainStart) GetBaseRevision() string {
|
||||
if x != nil {
|
||||
return x.BaseRevision
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *TrainStart) GetData() []byte {
|
||||
if x != nil {
|
||||
return x.Data
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *TrainStart) GetParams() string {
|
||||
if x != nil {
|
||||
return x.Params
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *TrainStart) GetGpuMemoryMib() int32 {
|
||||
if x != nil {
|
||||
return x.GpuMemoryMib
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *TrainStart) GetGguf() bool {
|
||||
if x != nil {
|
||||
return x.Gguf
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// TrainCancel stops a training job.
|
||||
type TrainCancel struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *TrainCancel) Reset() {
|
||||
*x = TrainCancel{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[45]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *TrainCancel) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*TrainCancel) ProtoMessage() {}
|
||||
|
||||
func (x *TrainCancel) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[45]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use TrainCancel.ProtoReflect.Descriptor instead.
|
||||
func (*TrainCancel) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{45}
|
||||
}
|
||||
|
||||
func (x *TrainCancel) GetJobId() string {
|
||||
if x != nil {
|
||||
return x.JobId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// TrainStatus is how a training job is going, sent as it changes.
|
||||
type TrainStatus struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
|
||||
// state is preparing, running, done or failed.
|
||||
State string `protobuf:"bytes,2,opt,name=state,proto3" json:"state,omitempty"`
|
||||
Step int32 `protobuf:"varint,3,opt,name=step,proto3" json:"step,omitempty"`
|
||||
Steps int32 `protobuf:"varint,4,opt,name=steps,proto3" json:"steps,omitempty"`
|
||||
Loss float64 `protobuf:"fixed64,5,opt,name=loss,proto3" json:"loss,omitempty"`
|
||||
// message is what is being done, or why it failed.
|
||||
Message string `protobuf:"bytes,6,opt,name=message,proto3" json:"message,omitempty"`
|
||||
// peak_mib is the most the job held of the card.
|
||||
PeakMib int32 `protobuf:"varint,7,opt,name=peak_mib,json=peakMib,proto3" json:"peak_mib,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *TrainStatus) Reset() {
|
||||
*x = TrainStatus{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[46]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *TrainStatus) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*TrainStatus) ProtoMessage() {}
|
||||
|
||||
func (x *TrainStatus) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[46]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use TrainStatus.ProtoReflect.Descriptor instead.
|
||||
func (*TrainStatus) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{46}
|
||||
}
|
||||
|
||||
func (x *TrainStatus) GetJobId() string {
|
||||
if x != nil {
|
||||
return x.JobId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *TrainStatus) GetState() string {
|
||||
if x != nil {
|
||||
return x.State
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *TrainStatus) GetStep() int32 {
|
||||
if x != nil {
|
||||
return x.Step
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *TrainStatus) GetSteps() int32 {
|
||||
if x != nil {
|
||||
return x.Steps
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *TrainStatus) GetLoss() float64 {
|
||||
if x != nil {
|
||||
return x.Loss
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *TrainStatus) GetMessage() string {
|
||||
if x != nil {
|
||||
return x.Message
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *TrainStatus) GetPeakMib() int32 {
|
||||
if x != nil {
|
||||
return x.PeakMib
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// TrainArtifact is a piece of what a training job made, sent before its
|
||||
// final status: a gzipped tar of the PEFT adapter (kind peft), or the GGUF
|
||||
// adapter (kind gguf).
|
||||
type TrainArtifact struct {
|
||||
state protoimpl.MessageState `protogen:"open.v1"`
|
||||
JobId string `protobuf:"bytes,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"`
|
||||
Kind string `protobuf:"bytes,2,opt,name=kind,proto3" json:"kind,omitempty"`
|
||||
Seq int32 `protobuf:"varint,3,opt,name=seq,proto3" json:"seq,omitempty"`
|
||||
Data []byte `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"`
|
||||
Last bool `protobuf:"varint,5,opt,name=last,proto3" json:"last,omitempty"`
|
||||
unknownFields protoimpl.UnknownFields
|
||||
sizeCache protoimpl.SizeCache
|
||||
}
|
||||
|
||||
func (x *TrainArtifact) Reset() {
|
||||
*x = TrainArtifact{}
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[47]
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
|
||||
func (x *TrainArtifact) String() string {
|
||||
return protoimpl.X.MessageStringOf(x)
|
||||
}
|
||||
|
||||
func (*TrainArtifact) ProtoMessage() {}
|
||||
|
||||
func (x *TrainArtifact) ProtoReflect() protoreflect.Message {
|
||||
mi := &file_llmgw_v1_agent_proto_msgTypes[47]
|
||||
if x != nil {
|
||||
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
|
||||
if ms.LoadMessageInfo() == nil {
|
||||
ms.StoreMessageInfo(mi)
|
||||
}
|
||||
return ms
|
||||
}
|
||||
return mi.MessageOf(x)
|
||||
}
|
||||
|
||||
// Deprecated: Use TrainArtifact.ProtoReflect.Descriptor instead.
|
||||
func (*TrainArtifact) Descriptor() ([]byte, []int) {
|
||||
return file_llmgw_v1_agent_proto_rawDescGZIP(), []int{47}
|
||||
}
|
||||
|
||||
func (x *TrainArtifact) GetJobId() string {
|
||||
if x != nil {
|
||||
return x.JobId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *TrainArtifact) GetKind() string {
|
||||
if x != nil {
|
||||
return x.Kind
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *TrainArtifact) GetSeq() int32 {
|
||||
if x != nil {
|
||||
return x.Seq
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func (x *TrainArtifact) GetData() []byte {
|
||||
if x != nil {
|
||||
return x.Data
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *TrainArtifact) GetLast() bool {
|
||||
if x != nil {
|
||||
return x.Last
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var File_llmgw_v1_agent_proto protoreflect.FileDescriptor
|
||||
|
||||
const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\n" +
|
||||
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\xb9\x03\n" +
|
||||
"\x14llmgw/v1/agent.proto\x12\bllmgw.v1\"\xb7\x04\n" +
|
||||
"\fAgentMessage\x120\n" +
|
||||
"\bregister\x18\x01 \x01(\v2\x12.llmgw.v1.RegisterH\x00R\bregister\x123\n" +
|
||||
"\theartbeat\x18\x02 \x01(\v2\x13.llmgw.v1.HeartbeatH\x00R\theartbeat\x12-\n" +
|
||||
@@ -4714,7 +5102,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\vfabric_test\x18\x06 \x01(\v2\x19.llmgw.v1.FabricTestReplyH\x00R\n" +
|
||||
"fabricTest\x122\n" +
|
||||
"\aupgrade\x18\a \x01(\v2\x16.llmgw.v1.UpgradeReplyH\x00R\aupgrade\x121\n" +
|
||||
"\tkv_events\x18\b \x01(\v2\x12.llmgw.v1.KVEventsH\x00R\bkvEventsB\x05\n" +
|
||||
"\tkv_events\x18\b \x01(\v2\x12.llmgw.v1.KVEventsH\x00R\bkvEvents\x12:\n" +
|
||||
"\ftrain_status\x18\t \x01(\v2\x15.llmgw.v1.TrainStatusH\x00R\vtrainStatus\x12@\n" +
|
||||
"\x0etrain_artifact\x18\n" +
|
||||
" \x01(\v2\x17.llmgw.v1.TrainArtifactH\x00R\rtrainArtifactB\x05\n" +
|
||||
"\x03msg\"\xb5\x01\n" +
|
||||
"\bKVEvents\x12\x1d\n" +
|
||||
"\n" +
|
||||
@@ -4725,7 +5116,7 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"clearFirst\x12\x16\n" +
|
||||
"\x06stored\x18\x04 \x03(\x06R\x06stored\x12\x18\n" +
|
||||
"\aremoved\x18\x05 \x03(\x06R\aremoved\x12\x18\n" +
|
||||
"\aproblem\x18\x06 \x01(\tR\aproblem\"\x83\a\n" +
|
||||
"\aproblem\x18\x06 \x01(\tR\aproblem\"\xf8\a\n" +
|
||||
"\x0eControlMessage\x12,\n" +
|
||||
"\x05start\x18\x01 \x01(\v2\x14.llmgw.v1.StartShardH\x00R\x05start\x12)\n" +
|
||||
"\x04stop\x18\x02 \x01(\v2\x13.llmgw.v1.StopShardH\x00R\x04stop\x12,\n" +
|
||||
@@ -4743,7 +5134,10 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x0econtrol_planes\x18\v \x01(\v2\x17.llmgw.v1.ControlPlanesH\x00R\rcontrolPlanes\x120\n" +
|
||||
"\bfirewall\x18\r \x01(\v2\x12.llmgw.v1.FirewallH\x00R\bfirewall\x12=\n" +
|
||||
"\rrestart_agent\x18\x0e \x01(\v2\x16.llmgw.v1.RestartAgentH\x00R\frestartAgent\x12@\n" +
|
||||
"\x0eagent_settings\x18\x0f \x01(\v2\x17.llmgw.v1.AgentSettingsH\x00R\ragentSettings\x129\n" +
|
||||
"\x0eagent_settings\x18\x0f \x01(\v2\x17.llmgw.v1.AgentSettingsH\x00R\ragentSettings\x127\n" +
|
||||
"\vtrain_start\x18\x10 \x01(\v2\x14.llmgw.v1.TrainStartH\x00R\n" +
|
||||
"trainStart\x12:\n" +
|
||||
"\ftrain_cancel\x18\x11 \x01(\v2\x15.llmgw.v1.TrainCancelH\x00R\vtrainCancel\x129\n" +
|
||||
"\x05trace\x18\f \x03(\v2#.llmgw.v1.ControlMessage.TraceEntryR\x05trace\x1a8\n" +
|
||||
"\n" +
|
||||
"TraceEntry\x12\x10\n" +
|
||||
@@ -5150,7 +5544,32 @@ const file_llmgw_v1_agent_proto_rawDesc = "" +
|
||||
"\x05error\x18\x03 \x01(\tR\x05error\"9\n" +
|
||||
"\vLoraAdapter\x12\x12\n" +
|
||||
"\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" +
|
||||
"\x06source\x18\x02 \x01(\tR\x06source2O\n" +
|
||||
"\x06source\x18\x02 \x01(\tR\x06source\"\xcb\x01\n" +
|
||||
"\n" +
|
||||
"TrainStart\x12\x15\n" +
|
||||
"\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x1b\n" +
|
||||
"\tbase_repo\x18\x02 \x01(\tR\bbaseRepo\x12#\n" +
|
||||
"\rbase_revision\x18\x03 \x01(\tR\fbaseRevision\x12\x12\n" +
|
||||
"\x04data\x18\x04 \x01(\fR\x04data\x12\x16\n" +
|
||||
"\x06params\x18\x05 \x01(\tR\x06params\x12$\n" +
|
||||
"\x0egpu_memory_mib\x18\x06 \x01(\x05R\fgpuMemoryMib\x12\x12\n" +
|
||||
"\x04gguf\x18\a \x01(\bR\x04gguf\"$\n" +
|
||||
"\vTrainCancel\x12\x15\n" +
|
||||
"\x06job_id\x18\x01 \x01(\tR\x05jobId\"\xad\x01\n" +
|
||||
"\vTrainStatus\x12\x15\n" +
|
||||
"\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x14\n" +
|
||||
"\x05state\x18\x02 \x01(\tR\x05state\x12\x12\n" +
|
||||
"\x04step\x18\x03 \x01(\x05R\x04step\x12\x14\n" +
|
||||
"\x05steps\x18\x04 \x01(\x05R\x05steps\x12\x12\n" +
|
||||
"\x04loss\x18\x05 \x01(\x01R\x04loss\x12\x18\n" +
|
||||
"\amessage\x18\x06 \x01(\tR\amessage\x12\x19\n" +
|
||||
"\bpeak_mib\x18\a \x01(\x05R\apeakMib\"t\n" +
|
||||
"\rTrainArtifact\x12\x15\n" +
|
||||
"\x06job_id\x18\x01 \x01(\tR\x05jobId\x12\x12\n" +
|
||||
"\x04kind\x18\x02 \x01(\tR\x04kind\x12\x10\n" +
|
||||
"\x03seq\x18\x03 \x01(\x05R\x03seq\x12\x12\n" +
|
||||
"\x04data\x18\x04 \x01(\fR\x04data\x12\x12\n" +
|
||||
"\x04last\x18\x05 \x01(\bR\x04last2O\n" +
|
||||
"\fAgentService\x12?\n" +
|
||||
"\aConnect\x12\x16.llmgw.v1.AgentMessage\x1a\x18.llmgw.v1.ControlMessage(\x010\x01B\x93\x01\n" +
|
||||
"\fcom.llmgw.v1B\n" +
|
||||
@@ -5169,7 +5588,7 @@ func file_llmgw_v1_agent_proto_rawDescGZIP() []byte {
|
||||
}
|
||||
|
||||
var file_llmgw_v1_agent_proto_enumTypes = make([]protoimpl.EnumInfo, 1)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 52)
|
||||
var file_llmgw_v1_agent_proto_msgTypes = make([]protoimpl.MessageInfo, 56)
|
||||
var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(ShardStatus_State)(0), // 0: llmgw.v1.ShardStatus.State
|
||||
(*AgentMessage)(nil), // 1: llmgw.v1.AgentMessage
|
||||
@@ -5216,14 +5635,18 @@ var file_llmgw_v1_agent_proto_goTypes = []any{
|
||||
(*UpgradeAgent)(nil), // 42: llmgw.v1.UpgradeAgent
|
||||
(*UpgradeReply)(nil), // 43: llmgw.v1.UpgradeReply
|
||||
(*LoraAdapter)(nil), // 44: llmgw.v1.LoraAdapter
|
||||
nil, // 45: llmgw.v1.ControlMessage.TraceEntry
|
||||
nil, // 46: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 47: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 48: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 49: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 50: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 51: llmgw.v1.EngineWant.OfflineSha256Entry
|
||||
nil, // 52: llmgw.v1.EngineProfile.EnvEntry
|
||||
(*TrainStart)(nil), // 45: llmgw.v1.TrainStart
|
||||
(*TrainCancel)(nil), // 46: llmgw.v1.TrainCancel
|
||||
(*TrainStatus)(nil), // 47: llmgw.v1.TrainStatus
|
||||
(*TrainArtifact)(nil), // 48: llmgw.v1.TrainArtifact
|
||||
nil, // 49: llmgw.v1.ControlMessage.TraceEntry
|
||||
nil, // 50: llmgw.v1.Register.LabelsEntry
|
||||
nil, // 51: llmgw.v1.Register.EngineVersionsEntry
|
||||
nil, // 52: llmgw.v1.GPU.NicPathsEntry
|
||||
nil, // 53: llmgw.v1.IBPort.CountersEntry
|
||||
nil, // 54: llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
nil, // 55: llmgw.v1.EngineWant.OfflineSha256Entry
|
||||
nil, // 56: llmgw.v1.EngineProfile.EnvEntry
|
||||
}
|
||||
var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
8, // 0: llmgw.v1.AgentMessage.register:type_name -> llmgw.v1.Register
|
||||
@@ -5234,62 +5657,66 @@ var file_llmgw_v1_agent_proto_depIdxs = []int32{
|
||||
41, // 5: llmgw.v1.AgentMessage.fabric_test:type_name -> llmgw.v1.FabricTestReply
|
||||
43, // 6: llmgw.v1.AgentMessage.upgrade:type_name -> llmgw.v1.UpgradeReply
|
||||
2, // 7: llmgw.v1.AgentMessage.kv_events:type_name -> llmgw.v1.KVEvents
|
||||
30, // 8: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
33, // 9: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
34, // 10: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
35, // 11: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
32, // 12: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
24, // 13: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
36, // 14: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
38, // 15: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
40, // 16: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
|
||||
42, // 17: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent
|
||||
7, // 18: llmgw.v1.ControlMessage.control_planes:type_name -> llmgw.v1.ControlPlanes
|
||||
5, // 19: llmgw.v1.ControlMessage.firewall:type_name -> llmgw.v1.Firewall
|
||||
4, // 20: llmgw.v1.ControlMessage.restart_agent:type_name -> llmgw.v1.RestartAgent
|
||||
6, // 21: llmgw.v1.ControlMessage.agent_settings:type_name -> llmgw.v1.AgentSettings
|
||||
45, // 22: llmgw.v1.ControlMessage.trace:type_name -> llmgw.v1.ControlMessage.TraceEntry
|
||||
46, // 23: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
12, // 24: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
14, // 25: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
47, // 26: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
11, // 27: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
15, // 28: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
10, // 29: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 30: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
21, // 31: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 32: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
27, // 33: llmgw.v1.RunningShard.compute:type_name -> llmgw.v1.Compute
|
||||
48, // 34: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
13, // 35: llmgw.v1.GPU.mig:type_name -> llmgw.v1.MIGDevice
|
||||
18, // 36: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
17, // 37: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor
|
||||
49, // 38: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
20, // 39: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
10, // 40: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
15, // 41: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
50, // 42: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
26, // 43: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
16, // 44: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
19, // 45: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
14, // 46: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
22, // 47: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
|
||||
21, // 48: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
|
||||
25, // 49: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
51, // 50: llmgw.v1.EngineWant.offline_sha256:type_name -> llmgw.v1.EngineWant.OfflineSha256Entry
|
||||
0, // 51: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
29, // 52: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
27, // 53: llmgw.v1.ShardStatus.compute:type_name -> llmgw.v1.Compute
|
||||
31, // 54: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
52, // 55: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
44, // 56: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter
|
||||
1, // 57: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
3, // 58: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
58, // [58:59] is the sub-list for method output_type
|
||||
57, // [57:58] is the sub-list for method input_type
|
||||
57, // [57:57] is the sub-list for extension type_name
|
||||
57, // [57:57] is the sub-list for extension extendee
|
||||
0, // [0:57] is the sub-list for field type_name
|
||||
47, // 8: llmgw.v1.AgentMessage.train_status:type_name -> llmgw.v1.TrainStatus
|
||||
48, // 9: llmgw.v1.AgentMessage.train_artifact:type_name -> llmgw.v1.TrainArtifact
|
||||
30, // 10: llmgw.v1.ControlMessage.start:type_name -> llmgw.v1.StartShard
|
||||
33, // 11: llmgw.v1.ControlMessage.stop:type_name -> llmgw.v1.StopShard
|
||||
34, // 12: llmgw.v1.ControlMessage.drain:type_name -> llmgw.v1.DrainShard
|
||||
35, // 13: llmgw.v1.ControlMessage.ack:type_name -> llmgw.v1.Ack
|
||||
32, // 14: llmgw.v1.ControlMessage.remove_weights:type_name -> llmgw.v1.RemoveWeights
|
||||
24, // 15: llmgw.v1.ControlMessage.provision:type_name -> llmgw.v1.Provision
|
||||
36, // 16: llmgw.v1.ControlMessage.logs:type_name -> llmgw.v1.LogsRequest
|
||||
38, // 17: llmgw.v1.ControlMessage.fabric_scan:type_name -> llmgw.v1.FabricScanRequest
|
||||
40, // 18: llmgw.v1.ControlMessage.fabric_test:type_name -> llmgw.v1.FabricTestRequest
|
||||
42, // 19: llmgw.v1.ControlMessage.upgrade:type_name -> llmgw.v1.UpgradeAgent
|
||||
7, // 20: llmgw.v1.ControlMessage.control_planes:type_name -> llmgw.v1.ControlPlanes
|
||||
5, // 21: llmgw.v1.ControlMessage.firewall:type_name -> llmgw.v1.Firewall
|
||||
4, // 22: llmgw.v1.ControlMessage.restart_agent:type_name -> llmgw.v1.RestartAgent
|
||||
6, // 23: llmgw.v1.ControlMessage.agent_settings:type_name -> llmgw.v1.AgentSettings
|
||||
45, // 24: llmgw.v1.ControlMessage.train_start:type_name -> llmgw.v1.TrainStart
|
||||
46, // 25: llmgw.v1.ControlMessage.train_cancel:type_name -> llmgw.v1.TrainCancel
|
||||
49, // 26: llmgw.v1.ControlMessage.trace:type_name -> llmgw.v1.ControlMessage.TraceEntry
|
||||
50, // 27: llmgw.v1.Register.labels:type_name -> llmgw.v1.Register.LabelsEntry
|
||||
12, // 28: llmgw.v1.Register.gpus:type_name -> llmgw.v1.GPU
|
||||
14, // 29: llmgw.v1.Register.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
51, // 30: llmgw.v1.Register.engine_versions:type_name -> llmgw.v1.Register.EngineVersionsEntry
|
||||
11, // 31: llmgw.v1.Register.running_shards:type_name -> llmgw.v1.RunningShard
|
||||
15, // 32: llmgw.v1.Register.weights:type_name -> llmgw.v1.LocalWeights
|
||||
10, // 33: llmgw.v1.Register.storage:type_name -> llmgw.v1.Storage
|
||||
9, // 34: llmgw.v1.Register.host:type_name -> llmgw.v1.Host
|
||||
21, // 35: llmgw.v1.Register.imex:type_name -> llmgw.v1.IMEX
|
||||
0, // 36: llmgw.v1.RunningShard.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
27, // 37: llmgw.v1.RunningShard.compute:type_name -> llmgw.v1.Compute
|
||||
52, // 38: llmgw.v1.GPU.nic_paths:type_name -> llmgw.v1.GPU.NicPathsEntry
|
||||
13, // 39: llmgw.v1.GPU.mig:type_name -> llmgw.v1.MIGDevice
|
||||
18, // 40: llmgw.v1.NodeLoad.shards:type_name -> llmgw.v1.ShardLoad
|
||||
17, // 41: llmgw.v1.NodeLoad.gpus:type_name -> llmgw.v1.GPUSensor
|
||||
53, // 42: llmgw.v1.IBPort.counters:type_name -> llmgw.v1.IBPort.CountersEntry
|
||||
20, // 43: llmgw.v1.IBPort.subnet_managers:type_name -> llmgw.v1.SubnetManager
|
||||
10, // 44: llmgw.v1.Heartbeat.storage:type_name -> llmgw.v1.Storage
|
||||
15, // 45: llmgw.v1.Heartbeat.weights:type_name -> llmgw.v1.LocalWeights
|
||||
54, // 46: llmgw.v1.Heartbeat.engine_versions:type_name -> llmgw.v1.Heartbeat.EngineVersionsEntry
|
||||
26, // 47: llmgw.v1.Heartbeat.provision:type_name -> llmgw.v1.ProvisionStatus
|
||||
16, // 48: llmgw.v1.Heartbeat.load:type_name -> llmgw.v1.NodeLoad
|
||||
19, // 49: llmgw.v1.Heartbeat.ib_ports:type_name -> llmgw.v1.IBPort
|
||||
14, // 50: llmgw.v1.Heartbeat.fabrics:type_name -> llmgw.v1.FabricAttachment
|
||||
22, // 51: llmgw.v1.Heartbeat.gpu_fabric:type_name -> llmgw.v1.GPUFabricState
|
||||
21, // 52: llmgw.v1.Heartbeat.imex:type_name -> llmgw.v1.IMEX
|
||||
25, // 53: llmgw.v1.Provision.engines:type_name -> llmgw.v1.EngineWant
|
||||
55, // 54: llmgw.v1.EngineWant.offline_sha256:type_name -> llmgw.v1.EngineWant.OfflineSha256Entry
|
||||
0, // 55: llmgw.v1.ShardStatus.state:type_name -> llmgw.v1.ShardStatus.State
|
||||
29, // 56: llmgw.v1.ShardStatus.fetch:type_name -> llmgw.v1.FetchProgress
|
||||
27, // 57: llmgw.v1.ShardStatus.compute:type_name -> llmgw.v1.Compute
|
||||
31, // 58: llmgw.v1.StartShard.profile:type_name -> llmgw.v1.EngineProfile
|
||||
56, // 59: llmgw.v1.EngineProfile.env:type_name -> llmgw.v1.EngineProfile.EnvEntry
|
||||
44, // 60: llmgw.v1.EngineProfile.adapters:type_name -> llmgw.v1.LoraAdapter
|
||||
1, // 61: llmgw.v1.AgentService.Connect:input_type -> llmgw.v1.AgentMessage
|
||||
3, // 62: llmgw.v1.AgentService.Connect:output_type -> llmgw.v1.ControlMessage
|
||||
62, // [62:63] is the sub-list for method output_type
|
||||
61, // [61:62] is the sub-list for method input_type
|
||||
61, // [61:61] is the sub-list for extension type_name
|
||||
61, // [61:61] is the sub-list for extension extendee
|
||||
0, // [0:61] is the sub-list for field type_name
|
||||
}
|
||||
|
||||
func init() { file_llmgw_v1_agent_proto_init() }
|
||||
@@ -5306,6 +5733,8 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
(*AgentMessage_FabricTest)(nil),
|
||||
(*AgentMessage_Upgrade)(nil),
|
||||
(*AgentMessage_KvEvents)(nil),
|
||||
(*AgentMessage_TrainStatus)(nil),
|
||||
(*AgentMessage_TrainArtifact)(nil),
|
||||
}
|
||||
file_llmgw_v1_agent_proto_msgTypes[2].OneofWrappers = []any{
|
||||
(*ControlMessage_Start)(nil),
|
||||
@@ -5322,6 +5751,8 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
(*ControlMessage_Firewall)(nil),
|
||||
(*ControlMessage_RestartAgent)(nil),
|
||||
(*ControlMessage_AgentSettings)(nil),
|
||||
(*ControlMessage_TrainStart)(nil),
|
||||
(*ControlMessage_TrainCancel)(nil),
|
||||
}
|
||||
type x struct{}
|
||||
out := protoimpl.TypeBuilder{
|
||||
@@ -5329,7 +5760,7 @@ func file_llmgw_v1_agent_proto_init() {
|
||||
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
|
||||
RawDescriptor: unsafe.Slice(unsafe.StringData(file_llmgw_v1_agent_proto_rawDesc), len(file_llmgw_v1_agent_proto_rawDesc)),
|
||||
NumEnums: 1,
|
||||
NumMessages: 52,
|
||||
NumMessages: 56,
|
||||
NumExtensions: 0,
|
||||
NumServices: 1,
|
||||
},
|
||||
|
||||
@@ -28,6 +28,8 @@ message AgentMessage {
|
||||
FabricTestReply fabric_test = 6;
|
||||
UpgradeReply upgrade = 7;
|
||||
KVEvents kv_events = 8;
|
||||
TrainStatus train_status = 9;
|
||||
TrainArtifact train_artifact = 10;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,6 +76,8 @@ message ControlMessage {
|
||||
Firewall firewall = 13;
|
||||
RestartAgent restart_agent = 14;
|
||||
AgentSettings agent_settings = 15;
|
||||
TrainStart train_start = 16;
|
||||
TrainCancel train_cancel = 17;
|
||||
}
|
||||
// trace is the W3C trace context (traceparent, and tracestate when there
|
||||
// is one) of the control plane's span that sent this command, so what the
|
||||
@@ -884,3 +888,55 @@ message LoraAdapter {
|
||||
string name = 1;
|
||||
string source = 2;
|
||||
}
|
||||
|
||||
// TrainStart has a node train a LoRA adapter (BACKLOG F4): on a base model's
|
||||
// weights from its registry, on question/answer pairs, with a Python trainer
|
||||
// the agent carries, in an environment it makes over an installed engine's.
|
||||
// One at a time per node.
|
||||
message TrainStart {
|
||||
string job_id = 1;
|
||||
// base_repo and base_revision name the base model's weights on its
|
||||
// registry (Hugging Face): safetensors, not the GGUF a catalog may serve.
|
||||
string base_repo = 2;
|
||||
string base_revision = 3;
|
||||
// data is the training pairs, NDJSON of {"q", "a"}.
|
||||
bytes data = 4;
|
||||
// params is the trainer's settings as JSON: rank, alpha, epochs, lr,
|
||||
// batch, max_len.
|
||||
string params = 5;
|
||||
// gpu_memory_mib is what the job needs of a card; the agent starts it
|
||||
// only when one has that much free, which lets it run beside an engine.
|
||||
int32 gpu_memory_mib = 6;
|
||||
// gguf asks for the adapter in llama.cpp's format too.
|
||||
bool gguf = 7;
|
||||
}
|
||||
|
||||
// TrainCancel stops a training job.
|
||||
message TrainCancel {
|
||||
string job_id = 1;
|
||||
}
|
||||
|
||||
// TrainStatus is how a training job is going, sent as it changes.
|
||||
message TrainStatus {
|
||||
string job_id = 1;
|
||||
// state is preparing, running, done or failed.
|
||||
string state = 2;
|
||||
int32 step = 3;
|
||||
int32 steps = 4;
|
||||
double loss = 5;
|
||||
// message is what is being done, or why it failed.
|
||||
string message = 6;
|
||||
// peak_mib is the most the job held of the card.
|
||||
int32 peak_mib = 7;
|
||||
}
|
||||
|
||||
// TrainArtifact is a piece of what a training job made, sent before its
|
||||
// final status: a gzipped tar of the PEFT adapter (kind peft), or the GGUF
|
||||
// adapter (kind gguf).
|
||||
message TrainArtifact {
|
||||
string job_id = 1;
|
||||
string kind = 2;
|
||||
int32 seq = 3;
|
||||
bytes data = 4;
|
||||
bool last = 5;
|
||||
}
|
||||
Reference in new issue
Block a user