Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 60 additions & 0 deletions components/model/claude/claude.go
Original file line number Diff line number Diff line change
Expand Up @@ -1118,6 +1118,15 @@ func convSchemaMessage(message *schema.Message) (mp anthropic.MessageParam, err
} else {
return mp, fmt.Errorf("image part must have either a URL or Base64Data")
}
case schema.ChatMessagePartTypeFileURL:
if message.UserInputMultiContent[i].File == nil {
return mp, fmt.Errorf("file field must not be nil when Type is ChatMessagePartTypeFileURL in user message")
}
block, cErr := newPDFDocumentBlock(message.UserInputMultiContent[i].File)
if cErr != nil {
return mp, cErr
}
messageParams = append(messageParams, block)
default:
return mp, fmt.Errorf("anthropic message type not supported: %s", message.UserInputMultiContent[i].Type)
}
Expand Down Expand Up @@ -1326,13 +1335,64 @@ func convToolMultiContent(callID string, parts []schema.MessageInputPart) (anthr
} else {
return result, fmt.Errorf("image part must have either a URL or Base64Data")
}
case schema.ChatMessagePartTypeFileURL:
if part.File == nil {
return result, fmt.Errorf("file field must not be nil when Type is ChatMessagePartTypeFileURL in tool result")
}
docSource, cErr := pdfDocumentSource(part.File)
if cErr != nil {
return result, cErr
}
result.OfToolResult.Content = append(result.OfToolResult.Content, anthropic.ToolResultBlockParamContentUnion{
OfDocument: &anthropic.DocumentBlockParam{
Source: docSource,
},
})
default:
return result, fmt.Errorf("anthropic message type not supported: %s", part.Type)
}
}
return result, nil
}

// pdfDocumentSource builds an Anthropic PDF document source from a file content
// part. It mirrors the image handling: a URL is used directly, otherwise raw
// base64 with an application/pdf MIME type. Only PDF documents are supported by
// the base64/URL document source, matching the Anthropic Messages `document` block.
func pdfDocumentSource(file *schema.MessageInputFile) (anthropic.DocumentBlockParamSourceUnion, error) {
var source anthropic.DocumentBlockParamSourceUnion
if file.URL != nil && *file.URL != "" {
source.OfURL = &anthropic.URLPDFSourceParam{URL: *file.URL}
return source, nil
}
if file.Base64Data != nil && *file.Base64Data != "" {
if file.MIMEType == "" {
return source, fmt.Errorf("file part must have MIMEType when use Base64Data")
}
if file.MIMEType != "application/pdf" {
return source, fmt.Errorf("file part only supports application/pdf as a document, got %s", file.MIMEType)
}
if strings.HasPrefix(*file.Base64Data, "data:") {
return source, fmt.Errorf("Base64Data should be a raw base64 string, but it has a 'data:' prefix")
}
source.OfBase64 = &anthropic.Base64PDFSourceParam{Data: *file.Base64Data}
return source, nil
}
return source, fmt.Errorf("file part must have either a URL or Base64Data")
}

// newPDFDocumentBlock builds a message-level PDF document content block from a
// file content part (used for user-input multi content).
func newPDFDocumentBlock(file *schema.MessageInputFile) (anthropic.ContentBlockParamUnion, error) {
source, err := pdfDocumentSource(file)
if err != nil {
return anthropic.ContentBlockParamUnion{}, err
}
return anthropic.ContentBlockParamUnion{
OfDocument: &anthropic.DocumentBlockParam{Source: source},
}, nil
}

func populateContentBlockBreakPoint(block anthropic.ContentBlockParamUnion, cacheCtrl *CacheControl) {
ctrl := newCacheControlParam(cacheCtrl)
if block.OfText != nil {
Expand Down
142 changes: 142 additions & 0 deletions components/model/claude/claude_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1269,3 +1269,145 @@ func TestVertexServiceAccountJSON(t *testing.T) {
})
})
}

func Test_convSchemaMessage_Document(t *testing.T) {
// A tiny but valid base64 blob; the conversion does not parse the PDF, it only
// carries the bytes, so any raw base64 exercises the path.
rawPDF := "JVBERi0xLjQK"
invalidDataURL := "data:application/pdf;base64," + rawPDF
httpURL := "https://example.com/doc.pdf"

t.Run("UserInputMultiContent", func(t *testing.T) {
t.Run("success with base64 pdf", func(t *testing.T) {
msg := &schema.Message{
Role: schema.User,
UserInputMultiContent: []schema.MessageInputPart{
{Type: schema.ChatMessagePartTypeText, Text: "read this"},
{Type: schema.ChatMessagePartTypeFileURL, File: &schema.MessageInputFile{
MessagePartCommon: schema.MessagePartCommon{Base64Data: &rawPDF, MIMEType: "application/pdf"},
}},
},
}
result, err := convSchemaMessage(msg)
assert.NoError(t, err)
assert.Len(t, result.Content, 2)
assert.Equal(t, "read this", result.Content[0].OfText.Text)
assert.NotNil(t, result.Content[1].OfDocument)
assert.NotNil(t, result.Content[1].OfDocument.Source.OfBase64)
assert.Equal(t, rawPDF, result.Content[1].OfDocument.Source.OfBase64.Data)
})

t.Run("success with url pdf", func(t *testing.T) {
msg := &schema.Message{
Role: schema.User,
UserInputMultiContent: []schema.MessageInputPart{
{Type: schema.ChatMessagePartTypeFileURL, File: &schema.MessageInputFile{
MessagePartCommon: schema.MessagePartCommon{URL: &httpURL},
}},
},
}
result, err := convSchemaMessage(msg)
assert.NoError(t, err)
assert.Len(t, result.Content, 1)
assert.NotNil(t, result.Content[0].OfDocument.Source.OfURL)
assert.Equal(t, httpURL, result.Content[0].OfDocument.Source.OfURL.URL)
})

t.Run("error with data url prefix", func(t *testing.T) {
msg := &schema.Message{
Role: schema.User,
UserInputMultiContent: []schema.MessageInputPart{
{Type: schema.ChatMessagePartTypeFileURL, File: &schema.MessageInputFile{
MessagePartCommon: schema.MessagePartCommon{Base64Data: &invalidDataURL, MIMEType: "application/pdf"},
}},
},
}
_, err := convSchemaMessage(msg)
assert.ErrorContains(t, err, "Base64Data should be a raw base64 string")
})

t.Run("error with no mime type for base64", func(t *testing.T) {
msg := &schema.Message{
Role: schema.User,
UserInputMultiContent: []schema.MessageInputPart{
{Type: schema.ChatMessagePartTypeFileURL, File: &schema.MessageInputFile{
MessagePartCommon: schema.MessagePartCommon{Base64Data: &rawPDF},
}},
},
}
_, err := convSchemaMessage(msg)
assert.ErrorContains(t, err, "file part must have MIMEType when use Base64Data")
})

t.Run("error with non-pdf mime type", func(t *testing.T) {
docx := "application/msword"
msg := &schema.Message{
Role: schema.User,
UserInputMultiContent: []schema.MessageInputPart{
{Type: schema.ChatMessagePartTypeFileURL, File: &schema.MessageInputFile{
MessagePartCommon: schema.MessagePartCommon{Base64Data: &rawPDF, MIMEType: docx},
}},
},
}
_, err := convSchemaMessage(msg)
assert.ErrorContains(t, err, "only supports application/pdf")
})

t.Run("error with no url or base64", func(t *testing.T) {
msg := &schema.Message{
Role: schema.User,
UserInputMultiContent: []schema.MessageInputPart{
{Type: schema.ChatMessagePartTypeFileURL, File: &schema.MessageInputFile{}},
},
}
_, err := convSchemaMessage(msg)
assert.ErrorContains(t, err, "file part must have either a URL or Base64Data")
})

t.Run("error with nil file", func(t *testing.T) {
msg := &schema.Message{
Role: schema.User,
UserInputMultiContent: []schema.MessageInputPart{
{Type: schema.ChatMessagePartTypeFileURL, File: nil},
},
}
_, err := convSchemaMessage(msg)
assert.ErrorContains(t, err, "file field must not be nil")
})
})

t.Run("ToolResult", func(t *testing.T) {
// A tool message whose result carries a PDF document part — the path the
// filesystem middleware's multimodal read_file exercises.
t.Run("success with base64 pdf in tool result", func(t *testing.T) {
msg := &schema.Message{
Role: schema.Tool,
ToolCallID: "call_1",
UserInputMultiContent: []schema.MessageInputPart{
{Type: schema.ChatMessagePartTypeFileURL, File: &schema.MessageInputFile{
MessagePartCommon: schema.MessagePartCommon{Base64Data: &rawPDF, MIMEType: "application/pdf"},
}},
},
}
result, err := convSchemaMessage(msg)
assert.NoError(t, err)
assert.Len(t, result.Content, 1)
assert.NotNil(t, result.Content[0].OfToolResult)
assert.Len(t, result.Content[0].OfToolResult.Content, 1)
assert.NotNil(t, result.Content[0].OfToolResult.Content[0].OfDocument)
assert.Equal(t, rawPDF, result.Content[0].OfToolResult.Content[0].OfDocument.Source.OfBase64.Data)
})

t.Run("error with nil file in tool result", func(t *testing.T) {
msg := &schema.Message{
Role: schema.Tool,
ToolCallID: "call_1",
UserInputMultiContent: []schema.MessageInputPart{
{Type: schema.ChatMessagePartTypeFileURL, File: nil},
},
}
_, err := convSchemaMessage(msg)
assert.ErrorContains(t, err, "file field must not be nil")
})
})
}
Loading