From 8772a33f71683c0f12a29db7c7d207b85f5acfbb Mon Sep 17 00:00:00 2001 From: dvd233 <111864431+dvd233@users.noreply.github.com> Date: Sun, 20 Sep 2026 19:26:09 -0700 Subject: [PATCH 1/2] fix(docreader): route SQL files through MarkItDown --- docreader/parser/registry.py | 4 +++- docreader/tests/test_parser_routing.py | 22 +++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/docreader/parser/registry.py b/docreader/parser/registry.py index 330c0211ade..ad58e0a1503 100644 --- a/docreader/parser/registry.py +++ b/docreader/parser/registry.py @@ -24,11 +24,12 @@ # File types that builtin does not implement natively. When the requested # engine (including empty / builtin) cannot parse them, route to this engine -# instead of raising ValueError — PPT/PPTX/CSV are markitdown-only. +# instead of raising ValueError — PPT/PPTX/CSV/SQL are markitdown-only. _DEFAULT_ENGINE_BY_TYPE = { "ppt": "markitdown", "pptx": "markitdown", "csv": "markitdown", + "sql": "markitdown", } @@ -191,6 +192,7 @@ def _build_default_registry() -> ParserEngineRegistry: "xlsx": MarkitdownParser, "xls": MarkitdownParser, "csv": MarkitdownParser, + "sql": MarkitdownParser, }, description="MarkItDown 解析引擎（微软 MarkItDown 库）", ) diff --git a/docreader/tests/test_parser_routing.py b/docreader/tests/test_parser_routing.py index 505f2a59315..65cdb652f22 100644 --- a/docreader/tests/test_parser_routing.py +++ b/docreader/tests/test_parser_routing.py @@ -67,6 +67,19 @@ def test_empty_engine_routes_pptx_to_markitdown(self): self.assertIs(registry.get_parser_class("", "ppt"), MarkitdownParser) self.assertIs(registry.get_parser_class("", ".PPTX"), MarkitdownParser) + def test_empty_engine_routes_sql_to_markitdown(self): + self.assertIs(registry.get_parser_class("", "sql"), MarkitdownParser) + self.assertIs(registry.get_parser_class(BUILTIN_ENGINE, ".SQL"), MarkitdownParser) + + def test_sql_file_is_parsed_as_text(self): + result = Parser().parse_file( + "schema.sql", + "sql", + b"-- marker\nSELECT 1 AS marker;\n", + ) + + self.assertIn("SELECT 1 AS marker;", result.content) + def test_builtin_engine_still_parses_pptx(self): self.assertIs(registry.get_parser_class(BUILTIN_ENGINE, "pptx"), MarkitdownParser) @@ -75,13 +88,20 @@ def test_type_default_when_builtin_lacks_pptx(self): isolated.register(BUILTIN_ENGINE, {"pdf": PDFParser}) isolated.register( "markitdown", - {"pptx": MarkitdownParser, "ppt": MarkitdownParser, "csv": MarkitdownParser}, + { + "pptx": MarkitdownParser, + "ppt": MarkitdownParser, + "csv": MarkitdownParser, + "sql": MarkitdownParser, + }, ) self.assertIs(isolated.get_parser_class("", "pptx"), MarkitdownParser) self.assertIs(isolated.get_parser_class(BUILTIN_ENGINE, "pptx"), MarkitdownParser) self.assertIs(isolated.get_parser_class("", "csv"), MarkitdownParser) self.assertIs(isolated.get_parser_class(BUILTIN_ENGINE, "csv"), MarkitdownParser) + self.assertIs(isolated.get_parser_class("", "sql"), MarkitdownParser) + self.assertIs(isolated.get_parser_class(BUILTIN_ENGINE, "sql"), MarkitdownParser) self.assertIs(isolated.get_parser_class(BUILTIN_ENGINE, "pdf"), PDFParser) self.assertIs(isolated.get_parser_class("", "pdf"), PDFParser) From 8fd9535b736aab40af73343fc4796e7691127438 Mon Sep 17 00:00:00 2001 From: dvd233 <111864431+dvd233@users.noreply.github.com> Date: Tue, 29 Sep 2026 04:55:38 -0700 Subject: [PATCH 2/2] fix(knowledge): admit SQL files across upload gates --- .../src/utils/fileTypeVerification.test.ts | 7 +++ frontend/src/utils/fileTypeVerification.ts | 1 + .../service/knowledge_create_test.go | 43 ++++++++++++++++++- .../application/service/knowledge_util.go | 2 +- .../service/knowledge_util_filetype_test.go | 6 ++- 5 files changed, 56 insertions(+), 3 deletions(-) diff --git a/frontend/src/utils/fileTypeVerification.test.ts b/frontend/src/utils/fileTypeVerification.test.ts index 338f9ef60d4..b48fb59fda6 100644 --- a/frontend/src/utils/fileTypeVerification.test.ts +++ b/frontend/src/utils/fileTypeVerification.test.ts @@ -14,3 +14,10 @@ test('shouldRejectKnowledgeFileType preserves dynamic whitelist behavior', () => assert.equal(shouldRejectKnowledgeFileType('page.html', ['pdf']), true) assert.equal(shouldRejectKnowledgeFileType('page.html', []), false) }) + +test('shouldRejectKnowledgeFileType accepts SQL before parser metadata loads', () => { + assert.equal(shouldRejectKnowledgeFileType('schema.sql'), false) + assert.equal(shouldRejectKnowledgeFileType('SCHEMA.SQL', []), false) + assert.equal(shouldRejectKnowledgeFileType('schema.sql', new Set()), false) + assert.equal(shouldRejectKnowledgeFileType('schema.sql', ['pdf']), true) +}) diff --git a/frontend/src/utils/fileTypeVerification.ts b/frontend/src/utils/fileTypeVerification.ts index 24d46ddde0a..300700ef52e 100644 --- a/frontend/src/utils/fileTypeVerification.ts +++ b/frontend/src/utils/fileTypeVerification.ts @@ -1,6 +1,7 @@ const DEFAULT_VALID_TYPES = new Set([ "pdf", "txt", + "sql", "md", "docx", "doc", diff --git a/internal/application/service/knowledge_create_test.go b/internal/application/service/knowledge_create_test.go index 8dda1b42c2e..71571e3ec41 100644 --- a/internal/application/service/knowledge_create_test.go +++ b/internal/application/service/knowledge_create_test.go @@ -3,6 +3,7 @@ package service import ( "bytes" "context" + "encoding/json" "errors" "io" "mime/multipart" @@ -116,7 +117,8 @@ func (s *createKnowledgeFileServiceStub) CopyFile(ctx context.Context, srcPath s } type createKnowledgeTaskEnqueuerStub struct { - calls int + calls int + lastTask *asynq.Task } func (s *createKnowledgeTaskEnqueuerStub) Enqueue( @@ -124,9 +126,48 @@ func (s *createKnowledgeTaskEnqueuerStub) Enqueue( opts ...asynq.Option, ) (*asynq.TaskInfo, error) { s.calls++ + s.lastTask = task return &asynq.TaskInfo{ID: "task-1", Queue: "default"}, nil } +func TestCreateKnowledgeFromSQLFileEnqueuesDocumentProcessing(t *testing.T) { + t.Parallel() + + for _, fileType := range []string{"sql", "SQL"} { + t.Run(fileType, func(t *testing.T) { + repo := &createKnowledgeFileRepoStub{} + fileSvc := &createKnowledgeFileServiceStub{} + task := &createKnowledgeTaskEnqueuerStub{} + svc := &knowledgeService{ + repo: repo, + kbService: &createKnowledgeFileKBServiceStub{kb: &types.KnowledgeBase{ID: "kb-1"}}, + fileSvc: fileSvc, + task: task, + } + filename := "schema." + fileType + + knowledge, err := svc.CreateKnowledgeFromFile( + newCreateKnowledgeFileContext(), "kb-1", + newMultipartFileHeader(t, filename, "-- marker\nSELECT 1 AS marker;\n"), + nil, nil, "", nil, "", nil, + ) + + require.NoError(t, err) + require.NotNil(t, knowledge) + require.Equal(t, 1, fileSvc.saveCalls) + require.Equal(t, 1, repo.createCalls) + require.Equal(t, fileType, repo.createdKnowledge.FileType) + require.Equal(t, 1, task.calls) + require.Equal(t, types.TypeDocumentProcess, task.lastTask.Type()) + var payload types.DocumentProcessPayload + require.NoError(t, json.Unmarshal(task.lastTask.Payload(), &payload)) + require.Equal(t, filename, payload.FileName) + require.Equal(t, fileType, payload.FileType) + require.Equal(t, knowledge.ID, payload.KnowledgeID) + }) + } +} + func TestCreateKnowledgeFromFileDoesNotPersistWhenStorageSaveFails(t *testing.T) { t.Parallel() diff --git a/internal/application/service/knowledge_util.go b/internal/application/service/knowledge_util.go index c3da26e70f0..aec77f6e685 100644 --- a/internal/application/service/knowledge_util.go +++ b/internal/application/service/knowledge_util.go @@ -28,7 +28,7 @@ const unknownFileType = "unknown" // and the worker's post-download re-check. Keeping one set avoids the drift // that let direct upload accept xlsx while URL import rejected it (#2447). var supportedImportFileExtensions = map[string]struct{}{ - "pdf": {}, "txt": {}, "docx": {}, "doc": {}, "epub": {}, + "pdf": {}, "txt": {}, "sql": {}, "docx": {}, "doc": {}, "epub": {}, "html": {}, "htm": {}, "mhtml": {}, "md": {}, "markdown": {}, "xmind": {}, "png": {}, "jpg": {}, "jpeg": {}, "gif": {}, diff --git a/internal/application/service/knowledge_util_filetype_test.go b/internal/application/service/knowledge_util_filetype_test.go index 5ab482a308a..7487978e8f1 100644 --- a/internal/application/service/knowledge_util_filetype_test.go +++ b/internal/application/service/knowledge_util_filetype_test.go @@ -13,6 +13,8 @@ func TestIsValidFileTypeHTML(t *testing.T) { {name: "htm", filename: "legacy.htm", want: true}, {name: "xmind", filename: "architecture.xmind", want: true}, {name: "uppercase xmind", filename: "ARCHITECTURE.XMIND", want: true}, + {name: "sql", filename: "schema.sql", want: true}, + {name: "uppercase sql", filename: "SCHEMA.SQL", want: true}, {name: "unsupported", filename: "payload.exe", want: false}, } @@ -34,6 +36,8 @@ func TestIsSupportedImportExtension(t *testing.T) { {name: "xlsx", ext: "xlsx", want: true}, {name: "xls", ext: "xls", want: true}, {name: "csv", ext: "csv", want: true}, + {name: "sql", ext: "sql", want: true}, + {name: "uppercase sql with dot", ext: ".SQL", want: true}, {name: "dot prefix", ext: ".xlsx", want: true}, {name: "uppercase", ext: "XLSX", want: true}, {name: "surrounding space", ext: " xlsx ", want: true}, @@ -74,7 +78,7 @@ func TestIsDataTableFileType(t *testing.T) { t.Errorf("isDataTableFileType(%q) = false, want true", ext) } } - for _, ext := range []string{"pdf", "png", "", unknownFileType} { + for _, ext := range []string{"pdf", "png", "sql", "", unknownFileType} { if isDataTableFileType(ext) { t.Errorf("isDataTableFileType(%q) = true, want false", ext) }