mirror of
https://github.com/danielmiessler/fabric.git
synced 2026-09-10 07:36:44 -04:00
Dynamically fetch and list available foundation models and inference profiles
This commit is contained in:
Connor Kirkpatrick
parent
3911fd9f5d
commit
76b889733d
7
go.mod
7
go.mod
|
|
@ -41,13 +41,14 @@ require (
|
|||
github.com/ProtonMail/go-crypto v1.3.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.17.27 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/bedrock v1.34.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.17 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.22.4 // indirect
|
||||
|
|
|
|||
8
go.sum
8
go.sum
|
|
@ -33,6 +33,8 @@ github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z
|
|||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.4 h1:GySzjhVvx0ERP6eyfAbAuAXLtAda5TEy19E5q5W8I9E=
|
||||
github.com/aws/aws-sdk-go-v2 v1.36.4/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs=
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14=
|
||||
github.com/aws/aws-sdk-go-v2/config v1.27.27 h1:HdqgGt1OAP0HkEDDShEl0oSYa9ZZBSOmKpdpsDMdO90=
|
||||
|
|
@ -43,10 +45,16 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11 h1:KreluoV8FZDEtI6Co2xuNk
|
|||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.11/go.mod h1:SeSUYBLsMYFoRvHE0Tjvn7kbxaUhl75CJi1sbfhMxkU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 h1:o1v1VFfPcDVlK3ll1L5xHsaQAFdNtZ5GXnNR7SwueC4=
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35/go.mod h1:rZUQNYMNG+8uZxz9FOerQJ+FceCiodXvixpeRtdESrU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 h1:R5b82ubO2NntENm3SAm0ADME+H630HomNJdgv+yZ3xw=
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35/go.mod h1:FuA+nmgMRfkzVKYDNEqQadvEMxtxl9+RLT9ribCwEMs=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
|
||||
github.com/aws/aws-sdk-go-v2/service/bedrock v1.34.1 h1:sD4KqDKG8aOaMWaWTMB8l8VnLa/Di7XHb0Uf4plrndA=
|
||||
github.com/aws/aws-sdk-go-v2/service/bedrock v1.34.1/go.mod h1:lrn8DOVFYFeaUZKxJ95T5eGDBjnhffgGz68Wq2sfBbA=
|
||||
github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.30.0 h1:eMOwQ8ZZK+76+08RfxeaGUtRFN6wxmD1rvqovc2kq2w=
|
||||
github.com/aws/aws-sdk-go-v2/service/bedrockruntime v1.30.0/go.mod h1:0b5Rq7rUvSQFYHI1UO0zFTV/S6j6DUyuykXA80C+YOI=
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8=
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import (
|
|||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/middleware"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/service/bedrock"
|
||||
"github.com/aws/aws-sdk-go-v2/service/bedrockruntime"
|
||||
"github.com/aws/aws-sdk-go-v2/service/bedrockruntime/types"
|
||||
|
||||
|
|
@ -24,7 +25,8 @@ import (
|
|||
// BedrockClient is a plugin to add support for Amazon Bedrock
|
||||
type BedrockClient struct {
|
||||
*plugins.PluginBase
|
||||
client *bedrockruntime.Client
|
||||
runtimeClient *bedrockruntime.Client
|
||||
controlPlaneClient *bedrock.Client
|
||||
}
|
||||
|
||||
// NewClient returns a new Bedrock plugin client
|
||||
|
|
@ -39,14 +41,16 @@ func NewClient() (ret *BedrockClient) {
|
|||
fmt.Printf("Unable to load AWS Config: %s\n", err)
|
||||
}
|
||||
|
||||
client := bedrockruntime.NewFromConfig(cfg)
|
||||
runtimeClient := bedrockruntime.NewFromConfig(cfg)
|
||||
controlPlaneClient := bedrock.NewFromConfig(cfg)
|
||||
|
||||
ret = &BedrockClient{
|
||||
PluginBase: &plugins.PluginBase{
|
||||
Name: vendorName,
|
||||
EnvNamePrefix: plugins.BuildEnvVariablePrefix(vendorName),
|
||||
},
|
||||
client: client,
|
||||
runtimeClient: runtimeClient,
|
||||
controlPlaneClient: controlPlaneClient,
|
||||
}
|
||||
|
||||
return
|
||||
|
|
@ -54,7 +58,32 @@ func NewClient() (ret *BedrockClient) {
|
|||
|
||||
// ListModels lists the models available for use with the Bedrock plugin
|
||||
func (c *BedrockClient) ListModels() ([]string, error) {
|
||||
return MODELS, nil
|
||||
models := []string{}
|
||||
ctx := context.TODO()
|
||||
|
||||
foundationModels, err := c.controlPlaneClient.ListFoundationModels(ctx, &bedrock.ListFoundationModelsInput{})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, model := range foundationModels.ModelSummaries {
|
||||
models = append(models, *model.ModelId)
|
||||
}
|
||||
|
||||
inferenceProfilesPaginator := bedrock.NewListInferenceProfilesPaginator(c.controlPlaneClient, &bedrock.ListInferenceProfilesInput{})
|
||||
|
||||
for inferenceProfilesPaginator.HasMorePages() {
|
||||
inferenceProfiles, err := inferenceProfilesPaginator.NextPage(context.TODO())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, profile := range inferenceProfiles.InferenceProfileSummaries {
|
||||
models = append(models, *profile.InferenceProfileId)
|
||||
}
|
||||
}
|
||||
|
||||
return models, nil
|
||||
}
|
||||
|
||||
// SendStream sends the messages to the the Bedrock ConverseStream API
|
||||
|
|
@ -70,7 +99,7 @@ func (c *BedrockClient) SendStream(msgs []*goopenai.ChatCompletionMessage, opts
|
|||
TopP: aws.Float32(float32(opts.TopP))},
|
||||
}
|
||||
|
||||
response, err := c.client.ConverseStream(context.TODO(), &converseInput)
|
||||
response, err := c.runtimeClient.ConverseStream(context.TODO(), &converseInput)
|
||||
if err != nil {
|
||||
fmt.Printf("Error conversing with Bedrock: %s\n", err)
|
||||
return
|
||||
|
|
@ -114,7 +143,7 @@ func (c *BedrockClient) Send(ctx context.Context, msgs []*goopenai.ChatCompletio
|
|||
ModelId: aws.String(opts.Model),
|
||||
Messages: messages,
|
||||
}
|
||||
response, err := c.client.Converse(ctx, &converseInput)
|
||||
response, err := c.runtimeClient.Converse(ctx, &converseInput)
|
||||
if err != nil {
|
||||
fmt.Printf("Error conversing with Bedrock: %s\n", err)
|
||||
return "", err
|
||||
|
|
@ -157,152 +186,3 @@ func (c *BedrockClient) toMessages(inputMessages []*goopenai.ChatCompletionMessa
|
|||
|
||||
return
|
||||
}
|
||||
|
||||
var MODELS = []string{
|
||||
"amazon.nova-micro-v1:0",
|
||||
"amazon.nova-lite-v1:0",
|
||||
"amazon.nova-pro-v1:0",
|
||||
"amazon.nova-premier-v1:0",
|
||||
|
||||
"amazon.titan-tg1-large",
|
||||
"amazon.titan-text-premier-v1:0",
|
||||
|
||||
"amazon.titan-text-lite-v1",
|
||||
"amazon.titan-text-express-v1",
|
||||
|
||||
"ai21.jamba-instruct-v1:0",
|
||||
"ai21.jamba-1-5-large-v1:0",
|
||||
"ai21.jamba-1-5-mini-v1:0",
|
||||
|
||||
"anthropic.claude-instant-v1",
|
||||
"anthropic.claude-v2",
|
||||
"anthropic.claude-v2:1",
|
||||
"anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"anthropic.claude-3-opus-20240229-v1:0",
|
||||
"anthropic.claude-3-5-haiku-20241022-v1:0",
|
||||
"anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"anthropic.claude-opus-4-20250514-v1:0",
|
||||
|
||||
"meta.llama3-8b-instruct-v1:0",
|
||||
"meta.llama3-70b-instruct-v1:0",
|
||||
"meta.llama3-1-8b-instruct-v1:0",
|
||||
"meta.llama3-1-70b-instruct-v1:0",
|
||||
"meta.llama3-2-11b-instruct-v1:0",
|
||||
"meta.llama3-2-90b-instruct-v1:0",
|
||||
"meta.llama3-2-1b-instruct-v1:0",
|
||||
"meta.llama3-2-3b-instruct-v1:0",
|
||||
"meta.llama3-3-70b-instruct-v1:0",
|
||||
"meta.llama4-scout-17b-instruct-v1:0",
|
||||
"meta.llama4-maverick-17b-instruct-v1:0",
|
||||
|
||||
"mistral.mistral-7b-instruct-v0:2",
|
||||
"mistral.mixtral-8x7b-instruct-v0:1",
|
||||
"mistral.mistral-small-2402-v1:0",
|
||||
"mistral.mistral-large-2402-v1:0",
|
||||
"mistral.pixtral-large-2502-v1:0",
|
||||
|
||||
// Cross Region Inferences Profiles
|
||||
// https://docs.aws.amazon.com/bedrock/latest/userguide/inference-profiles-support.html#inference-profiles-support-system
|
||||
"us.amazon.nova-lite-v1:0",
|
||||
"us.amazon.nova-lite-v1:0",
|
||||
"us.amazon.nova-micro-v1:0",
|
||||
"us.amazon.nova-micro-v1:0",
|
||||
"us.amazon.nova-premier-v1:0",
|
||||
"us.amazon.nova-premier-v1:0",
|
||||
"us.amazon.nova-pro-v1:0",
|
||||
"us.amazon.nova-pro-v1:0",
|
||||
"us.anthropic.claude-3-5-haiku-20241022-v1:0",
|
||||
"us.anthropic.claude-3-5-haiku-20241022-v1:0",
|
||||
"us.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"us.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"us.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"us.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"us.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"us.anthropic.claude-3-opus-20240229-v1:0",
|
||||
"us.anthropic.claude-3-opus-20240229-v1:0",
|
||||
"us.anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"us.anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"us.anthropic.claude-opus-4-20250514-v1:0",
|
||||
"us.anthropic.claude-opus-4-20250514-v1:0",
|
||||
"us.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"us.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"us.deepseek.r1-v1:0",
|
||||
"us.deepseek.r1-v1:0",
|
||||
"us.meta.llama3-1-405b-instruct-v1:0",
|
||||
"us.meta.llama3-1-405b-instruct-v1:0",
|
||||
"us.meta.llama3-1-70b-instruct-v1:0",
|
||||
"us.meta.llama3-1-70b-instruct-v1:0",
|
||||
"us.meta.llama3-1-8b-instruct-v1:0",
|
||||
"us.meta.llama3-1-8b-instruct-v1:0",
|
||||
"us.meta.llama3-2-11b-instruct-v1:0",
|
||||
"us.meta.llama3-2-11b-instruct-v1:0",
|
||||
"us.meta.llama3-2-1b-instruct-v1:0",
|
||||
"us.meta.llama3-2-1b-instruct-v1:0",
|
||||
"us.meta.llama3-2-3b-instruct-v1:0",
|
||||
"us.meta.llama3-2-3b-instruct-v1:0",
|
||||
"us.meta.llama3-2-90b-instruct-v1:0",
|
||||
"us.meta.llama3-2-90b-instruct-v1:0",
|
||||
"us.meta.llama3-3-70b-instruct-v1:0",
|
||||
"us.meta.llama3-3-70b-instruct-v1:0",
|
||||
"us.meta.llama4-maverick-17b-instruct-v1:0",
|
||||
"us.meta.llama4-maverick-17b-instruct-v1:0",
|
||||
"us.meta.llama4-scout-17b-instruct-v1:0",
|
||||
"us.meta.llama4-scout-17b-instruct-v1:0",
|
||||
"us.mistral.pixtral-large-2502-v1:0",
|
||||
"us.mistral.pixtral-large-2502-v1:0",
|
||||
"us.writer.palmyra-x4-v1:0",
|
||||
"us.writer.palmyra-x4-v1:0",
|
||||
"us.writer.palmyra-x5-v1:0",
|
||||
"us.writer.palmyra-x5-v1:0",
|
||||
"us-gov.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"us-gov.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"us-gov.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"us-gov.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"eu.amazon.nova-lite-v1:0",
|
||||
"eu.amazon.nova-lite-v1:0",
|
||||
"eu.amazon.nova-micro-v1:0",
|
||||
"eu.amazon.nova-micro-v1:0",
|
||||
"eu.amazon.nova-pro-v1:0",
|
||||
"eu.amazon.nova-pro-v1:0",
|
||||
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"eu.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"eu.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"eu.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"eu.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"eu.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"eu.anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"eu.anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"eu.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"eu.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"eu.meta.llama3-2-1b-instruct-v1:0",
|
||||
"eu.meta.llama3-2-1b-instruct-v1:0",
|
||||
"eu.meta.llama3-2-3b-instruct-v1:0",
|
||||
"eu.meta.llama3-2-3b-instruct-v1:0",
|
||||
"eu.mistral.pixtral-large-2502-v1:0",
|
||||
"eu.mistral.pixtral-large-2502-v1:0",
|
||||
"apac.amazon.nova-lite-v1:0",
|
||||
"apac.amazon.nova-lite-v1:0",
|
||||
"apac.amazon.nova-micro-v1:0",
|
||||
"apac.amazon.nova-micro-v1:0",
|
||||
"apac.amazon.nova-pro-v1:0",
|
||||
"apac.amazon.nova-pro-v1:0",
|
||||
"apac.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"apac.anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"apac.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"apac.anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"apac.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"apac.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
"apac.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"apac.anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"apac.anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"apac.anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"apac.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
"apac.anthropic.claude-sonnet-4-20250514-v1:0",
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue