diff --git a/docreader/parser/registry.py b/docreader/parser/registry.py index 330c0211ade..ad58e0a1503 100644 --- a/docreader/parser/registry.py +++ b/docreader/parser/registry.py @@ -24,11 +24,12 @@ # File types that builtin does not implement natively. When the requested # engine (including empty / builtin) cannot parse them, route to this engine -# instead of raising ValueError — PPT/PPTX/CSV are markitdown-only. +# instead of raising ValueError — PPT/PPTX/CSV/SQL are markitdown-only. _DEFAULT_ENGINE_BY_TYPE = { "ppt": "markitdown", "pptx": "markitdown", "csv": "markitdown", + "sql": "markitdown", } @@ -191,6 +192,7 @@ def _build_default_registry() -> ParserEngineRegistry: "xlsx": MarkitdownParser, "xls": MarkitdownParser, "csv": MarkitdownParser, + "sql": MarkitdownParser, }, description="MarkItDown 解析引擎（微软 MarkItDown 库）", ) diff --git a/docreader/tests/test_parser_routing.py b/docreader/tests/test_parser_routing.py index 505f2a59315..65cdb652f22 100644 --- a/docreader/tests/test_parser_routing.py +++ b/docreader/tests/test_parser_routing.py @@ -67,6 +67,19 @@ def test_empty_engine_routes_pptx_to_markitdown(self): self.assertIs(registry.get_parser_class("", "ppt"), MarkitdownParser) self.assertIs(registry.get_parser_class("", ".PPTX"), MarkitdownParser) + def test_empty_engine_routes_sql_to_markitdown(self): + self.assertIs(registry.get_parser_class("", "sql"), MarkitdownParser) + self.assertIs(registry.get_parser_class(BUILTIN_ENGINE, ".SQL"), MarkitdownParser) + + def test_sql_file_is_parsed_as_text(self): + result = Parser().parse_file( + "schema.sql", + "sql", + b"-- marker\nSELECT 1 AS marker;\n", + ) + + self.assertIn("SELECT 1 AS marker;", result.content) + def test_builtin_engine_still_parses_pptx(self): self.assertIs(registry.get_parser_class(BUILTIN_ENGINE, "pptx"), MarkitdownParser) @@ -75,13 +88,20 @@ def test_type_default_when_builtin_lacks_pptx(self): isolated.register(BUILTIN_ENGINE, {"pdf": PDFParser}) isolated.register( "markitdown", - {"pptx": MarkitdownParser, "ppt": MarkitdownParser, "csv": MarkitdownParser}, + { + "pptx": MarkitdownParser, + "ppt": MarkitdownParser, + "csv": MarkitdownParser, + "sql": MarkitdownParser, + }, ) self.assertIs(isolated.get_parser_class("", "pptx"), MarkitdownParser) self.assertIs(isolated.get_parser_class(BUILTIN_ENGINE, "pptx"), MarkitdownParser) self.assertIs(isolated.get_parser_class("", "csv"), MarkitdownParser) self.assertIs(isolated.get_parser_class(BUILTIN_ENGINE, "csv"), MarkitdownParser) + self.assertIs(isolated.get_parser_class("", "sql"), MarkitdownParser) + self.assertIs(isolated.get_parser_class(BUILTIN_ENGINE, "sql"), MarkitdownParser) self.assertIs(isolated.get_parser_class(BUILTIN_ENGINE, "pdf"), PDFParser) self.assertIs(isolated.get_parser_class("", "pdf"), PDFParser) diff --git a/frontend/src/utils/fileTypeVerification.test.ts b/frontend/src/utils/fileTypeVerification.test.ts index 8b1cc37681b..188c9b034ee 100644 --- a/frontend/src/utils/fileTypeVerification.test.ts +++ b/frontend/src/utils/fileTypeVerification.test.ts @@ -15,6 +15,13 @@ test('shouldRejectKnowledgeFileType preserves dynamic whitelist behavior', () => assert.equal(shouldRejectKnowledgeFileType('page.html', []), false) }) +test('shouldRejectKnowledgeFileType accepts SQL before parser metadata loads', () => { + assert.equal(shouldRejectKnowledgeFileType('schema.sql'), false) + assert.equal(shouldRejectKnowledgeFileType('SCHEMA.SQL', []), false) + assert.equal(shouldRejectKnowledgeFileType('schema.sql', new Set()), false) + assert.equal(shouldRejectKnowledgeFileType('schema.sql', ['pdf']), true) +}) + test('XMind uploads work before engine discovery and respect discovered capabilities', () => { for (const filename of ['architecture.xmind', 'ARCHITECTURE.XMIND']) { assert.equal(shouldRejectKnowledgeFileType(filename), false) diff --git a/frontend/src/utils/fileTypeVerification.ts b/frontend/src/utils/fileTypeVerification.ts index 586678e22ce..db8e6bcff0b 100644 --- a/frontend/src/utils/fileTypeVerification.ts +++ b/frontend/src/utils/fileTypeVerification.ts @@ -1,6 +1,7 @@ const DEFAULT_VALID_TYPES = new Set([ "pdf", "txt", + "sql", "md", "docx", "doc", diff --git a/internal/application/service/knowledge_create_test.go b/internal/application/service/knowledge_create_test.go index 937a63754ce..aea767ebccc 100644 --- a/internal/application/service/knowledge_create_test.go +++ b/internal/application/service/knowledge_create_test.go @@ -3,6 +3,7 @@ package service import ( "bytes" "context" + "encoding/json" "errors" "io" "mime/multipart" @@ -116,7 +117,8 @@ func (s *createKnowledgeFileServiceStub) CopyFile(ctx context.Context, srcPath s } type createKnowledgeTaskEnqueuerStub struct { - calls int + calls int + lastTask *asynq.Task } func (s *createKnowledgeTaskEnqueuerStub) Enqueue( @@ -124,9 +126,48 @@ func (s *createKnowledgeTaskEnqueuerStub) Enqueue( opts ...asynq.Option, ) (*asynq.TaskInfo, error) { s.calls++ + s.lastTask = task return &asynq.TaskInfo{ID: "task-1", Queue: "default"}, nil } +func TestCreateKnowledgeFromSQLFileEnqueuesDocumentProcessing(t *testing.T) { + t.Parallel() + + for _, fileType := range []string{"sql", "SQL"} { + t.Run(fileType, func(t *testing.T) { + repo := &createKnowledgeFileRepoStub{} + fileSvc := &createKnowledgeFileServiceStub{} + task := &createKnowledgeTaskEnqueuerStub{} + svc := &knowledgeService{ + repo: repo, + kbService: &createKnowledgeFileKBServiceStub{kb: &types.KnowledgeBase{ID: "kb-1"}}, + fileSvc: fileSvc, + task: task, + } + filename := "schema." + fileType + + knowledge, err := svc.CreateKnowledgeFromFile( + newCreateKnowledgeFileContext(), "kb-1", + newMultipartFileHeader(t, filename, "-- marker\nSELECT 1 AS marker;\n"), + nil, nil, "", nil, "", nil, + ) + + require.NoError(t, err) + require.NotNil(t, knowledge) + require.Equal(t, 1, fileSvc.saveCalls) + require.Equal(t, 1, repo.createCalls) + require.Equal(t, fileType, repo.createdKnowledge.FileType) + require.Equal(t, 1, task.calls) + require.Equal(t, types.TypeDocumentProcess, task.lastTask.Type()) + var payload types.DocumentProcessPayload + require.NoError(t, json.Unmarshal(task.lastTask.Payload(), &payload)) + require.Equal(t, filename, payload.FileName) + require.Equal(t, fileType, payload.FileType) + require.Equal(t, knowledge.ID, payload.KnowledgeID) + }) + } +} + func TestCreateKnowledgeFromFileDoesNotPersistWhenStorageSaveFails(t *testing.T) { t.Parallel() diff --git a/internal/application/service/knowledge_util.go b/internal/application/service/knowledge_util.go index e520edbc69f..d5f555fdae9 100644 --- a/internal/application/service/knowledge_util.go +++ b/internal/application/service/knowledge_util.go @@ -30,7 +30,7 @@ const unknownFileType = "unknown" // and the worker's post-download re-check. Keeping one set avoids the drift // that let direct upload accept xlsx while URL import rejected it (#2447). var supportedImportFileExtensions = map[string]struct{}{ - "pdf": {}, "txt": {}, "docx": {}, "doc": {}, "epub": {}, + "pdf": {}, "txt": {}, "sql": {}, "docx": {}, "doc": {}, "epub": {}, "html": {}, "htm": {}, "mhtml": {}, "md": {}, "markdown": {}, "xmind": {}, "png": {}, "jpg": {}, "jpeg": {}, "gif": {}, diff --git a/internal/application/service/knowledge_util_filetype_test.go b/internal/application/service/knowledge_util_filetype_test.go index 5ab482a308a..7487978e8f1 100644 --- a/internal/application/service/knowledge_util_filetype_test.go +++ b/internal/application/service/knowledge_util_filetype_test.go @@ -13,6 +13,8 @@ func TestIsValidFileTypeHTML(t *testing.T) { {name: "htm", filename: "legacy.htm", want: true}, {name: "xmind", filename: "architecture.xmind", want: true}, {name: "uppercase xmind", filename: "ARCHITECTURE.XMIND", want: true}, + {name: "sql", filename: "schema.sql", want: true}, + {name: "uppercase sql", filename: "SCHEMA.SQL", want: true}, {name: "unsupported", filename: "payload.exe", want: false}, } @@ -34,6 +36,8 @@ func TestIsSupportedImportExtension(t *testing.T) { {name: "xlsx", ext: "xlsx", want: true}, {name: "xls", ext: "xls", want: true}, {name: "csv", ext: "csv", want: true}, + {name: "sql", ext: "sql", want: true}, + {name: "uppercase sql with dot", ext: ".SQL", want: true}, {name: "dot prefix", ext: ".xlsx", want: true}, {name: "uppercase", ext: "XLSX", want: true}, {name: "surrounding space", ext: " xlsx ", want: true}, @@ -74,7 +78,7 @@ func TestIsDataTableFileType(t *testing.T) { t.Errorf("isDataTableFileType(%q) = false, want true", ext) } } - for _, ext := range []string{"pdf", "png", "", unknownFileType} { + for _, ext := range []string{"pdf", "png", "sql", "", unknownFileType} { if isDataTableFileType(ext) { t.Errorf("isDataTableFileType(%q) = true, want false", ext) }