From 35b58a315408a1df76d0809a8e6d25855d95d345 Mon Sep 17 00:00:00 2001 From: Detective-X Date: Mon, 1 Jun 2026 09:54:46 +0100 Subject: [PATCH] feat: add CJK predefined CMap decoders (Shift-JIS and UCS-2 BE) getEncoder() now handles two families of predefined CMaps that previously fell through to the default case and returned U+FFFD replacement characters: Shift-JIS (90ms-RKSJ-H, 90ms-RKSJ-V, 90pv-RKSJ-H): multibyteCMapEncoder wraps golang.org/x/text/encoding/japanese.ShiftJIS. Raw PDF content-stream bytes are Shift-JIS encoded; x/text does the charset conversion to UTF-8. UCS-2 big-endian (UniGB-UCS2-H/V, UniCNS-UCS2-H/V, UniJIS-UCS2-H/V, UniKS-UCS2-H/V): ucs2BEEncoder reads successive 2-byte big-endian code points directly as Unicode runes. No new dependency -- BMP-only uint16 arithmetic is correct for all Uni*-UCS2-* CMaps. golang.org/x/text was already an indirect dep; japanese.ShiftJIS adds no new transitive dependencies. Tests in page_cjk_test.go cover both encoders with Simplified Chinese, Traditional Chinese, Japanese, and Korean inputs plus edge cases. --- go.mod | 2 + go.sum | 2 + page.go | 39 ++++++++++++++++ page_cjk_test.go | 113 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+) create mode 100644 go.sum create mode 100644 page_cjk_test.go diff --git a/go.mod b/go.mod index 7dd1b3c5..65ea9fd3 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/ledongthuc/pdf go 1.24.1 + +require golang.org/x/text v0.25.0 diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..3470e4e3 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= diff --git a/page.go b/page.go index 31db8ab9..95119cac 100644 --- a/page.go +++ b/page.go @@ -11,6 +11,9 @@ import ( "io" "sort" "strings" + + "golang.org/x/text/encoding" + "golang.org/x/text/encoding/japanese" ) // A Page represent a single page in a PDF file. @@ -210,6 +213,13 @@ func (f Font) getEncoder() TextEncoding { return &byteEncoder{&macRomanEncoding} case "Identity-H": return f.charmapEncoding() + case "90ms-RKSJ-H", "90ms-RKSJ-V", "90pv-RKSJ-H": + return &multibyteCMapEncoder{japanese.ShiftJIS} + case "UniGB-UCS2-H", "UniGB-UCS2-V", + "UniCNS-UCS2-H", "UniCNS-UCS2-V", + "UniJIS-UCS2-H", "UniJIS-UCS2-V", + "UniKS-UCS2-H", "UniKS-UCS2-V": + return &ucs2BEEncoder{} default: if DebugOn { println("unknown encoding", enc.Name()) @@ -287,6 +297,35 @@ func (e *nopEncoder) Decode(raw string) (text string) { return raw } +// multibyteCMapEncoder decodes PDF content-stream bytes using an x/text Encoding. +// Used for predefined CMaps whose raw bytes are a well-known legacy encoding +// (e.g. Shift-JIS for 90ms-RKSJ-H/V). Falls back to raw bytes on error. +type multibyteCMapEncoder struct { + enc encoding.Encoding +} + +func (e *multibyteCMapEncoder) Decode(raw string) (text string) { + decoded, err := e.enc.NewDecoder().Bytes([]byte(raw)) + if err != nil { + return raw + } + return string(decoded) +} + +// ucs2BEEncoder decodes PDF content-stream bytes encoded as UCS-2 big-endian. +// Used for predefined CMaps such as UniGB-UCS2-H/V, UniCNS-UCS2-H/V, +// UniJIS-UCS2-H/V, and UniKS-UCS2-H/V. Each glyph selector is a 2-byte +// big-endian code point in the BMP. No external dependency required. +type ucs2BEEncoder struct{} + +func (e *ucs2BEEncoder) Decode(raw string) (text string) { + r := make([]rune, 0, len(raw)/2) + for i := 0; i+1 < len(raw); i += 2 { + r = append(r, rune(uint16(raw[i])<<8|uint16(raw[i+1]))) + } + return string(r) +} + type byteEncoder struct { table *[256]rune } diff --git a/page_cjk_test.go b/page_cjk_test.go new file mode 100644 index 00000000..e0c82451 --- /dev/null +++ b/page_cjk_test.go @@ -0,0 +1,113 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pdf + +import ( + "testing" + + "golang.org/x/text/encoding/japanese" +) + +// TestUCS2BEEncoder verifies that ucs2BEEncoder correctly decodes UCS-2 +// big-endian byte sequences produced by Uni*-UCS2-H/V predefined CMaps. +func TestUCS2BEEncoder(t *testing.T) { + tests := []struct { + name string + // raw holds two-byte UCS-2 BE pairs for each rune. + raw []byte + want string + }{ + { + name: "simplified Chinese characters", + // 中(0x4E2D) 文(0x6587) 测(0x6D4B) 试(0x8BD5) + raw: []byte{0x4E, 0x2D, 0x65, 0x87, 0x6D, 0x4B, 0x8B, 0xD5}, + want: "中文测试", + }, + { + name: "traditional Chinese characters", + // 繁(0x7E41) 體(0x9AD4) 中(0x4E2D) 文(0x6587) + raw: []byte{0x7E, 0x41, 0x9A, 0xD4, 0x4E, 0x2D, 0x65, 0x87}, + want: "繁體中文", + }, + { + name: "Japanese hiragana", + // あ(0x3042) い(0x3044) う(0x3046) + raw: []byte{0x30, 0x42, 0x30, 0x44, 0x30, 0x46}, + want: "あいう", + }, + { + name: "Korean hangul", + // 한(0xD55C) 국(0xAD6D) 어(0xC5B4) + raw: []byte{0xD5, 0x5C, 0xAD, 0x6D, 0xC5, 0xB4}, + want: "한국어", + }, + { + name: "ASCII via UCS-2", + // A(0x0041) B(0x0042) + raw: []byte{0x00, 0x41, 0x00, 0x42}, + want: "AB", + }, + { + name: "empty input", + raw: []byte{}, + want: "", + }, + { + name: "trailing odd byte is ignored", + // 中(0x4E2D) + one trailing byte + raw: []byte{0x4E, 0x2D, 0xFF}, + want: "中", + }, + } + + enc := &ucs2BEEncoder{} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := enc.Decode(string(tt.raw)) + if got != tt.want { + t.Errorf("Decode: got %q, want %q", got, tt.want) + } + }) + } +} + +// TestMultibyteCMapEncoder_ShiftJIS verifies that multibyteCMapEncoder +// correctly decodes Shift-JIS bytes produced by 90ms-RKSJ-H/V predefined CMaps. +func TestMultibyteCMapEncoder_ShiftJIS(t *testing.T) { + enc := &multibyteCMapEncoder{japanese.ShiftJIS} + + tests := []struct { + name string + raw []byte + want string + }{ + { + name: "Japanese katakana word", + // テスト (te-su-to) in Shift-JIS + raw: []byte{0x83, 0x65, 0x83, 0x58, 0x83, 0x67}, + want: "テスト", + }, + { + name: "mixed kanji and hiragana", + // 日本語 in Shift-JIS + raw: []byte{0x93, 0xFA, 0x96, 0x7B, 0x8C, 0xEA}, + want: "日本語", + }, + { + name: "empty input", + raw: []byte{}, + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := enc.Decode(string(tt.raw)) + if got != tt.want { + t.Errorf("Decode: got %q, want %q", got, tt.want) + } + }) + } +}