Fix truncation of strings containing non-terminal NUL characters

GRDB/Core/DatabaseFunction.swift

@@ -106,7 +106,18 @@ public final class DatabaseFunction: Identifiable, Sendable {
             case .double(let double):
                 sqlite3_result_double(context, double)
             case .string(let string):
-                sqlite3_result_text(context, string, -1, SQLITE_TRANSIENT)
+                var string = string
+                string.withUTF8 { buffer in
+                    guard let baseAddress = buffer.baseAddress else {
+                        // baseAddress may be nil for an empty string
+                        return sqlite3_result_text(context, "", 0, SQLITE_TRANSIENT)
+                    }
+                    sqlite3_result_text(
+                        context,
+                        UnsafeRawPointer(baseAddress).assumingMemoryBound(to: CChar.self),
+                        CInt(buffer.count),
+                        SQLITE_TRANSIENT)
+                }
             case .blob(let data):
                 data.withUnsafeBytes {
                     sqlite3_result_blob(context, $0.baseAddress, CInt($0.count), SQLITE_TRANSIENT)
@@ -448,7 +459,18 @@ public final class DatabaseFunction: Identifiable, Sendable {
         case .double(let double):
             sqlite3_result_double(sqliteContext, double)
         case .string(let string):
-            sqlite3_result_text(sqliteContext, string, -1, SQLITE_TRANSIENT)
+            var string = string
+            string.withUTF8 { buffer in
+                guard let baseAddress = buffer.baseAddress else {
+                    // baseAddress may be nil for an empty string
+                    return sqlite3_result_text(sqliteContext, "", 0, SQLITE_TRANSIENT)
+                }
+                sqlite3_result_text(
+                    sqliteContext,
+                    UnsafeRawPointer(baseAddress).assumingMemoryBound(to: CChar.self),
+                    CInt(buffer.count),
+                    SQLITE_TRANSIENT)
+            }
         case .blob(let data):
             data.withUnsafeBytes {
                 sqlite3_result_blob(sqliteContext, $0.baseAddress, CInt($0.count), SQLITE_TRANSIENT)

GRDB/Core/DatabaseValue.swift

@@ -147,7 +147,9 @@ public struct DatabaseValue: Hashable {
         case SQLITE_FLOAT:
             storage = .double(sqlite3_value_double(sqliteValue))
         case SQLITE_TEXT:
-            storage = .string(String(cString: sqlite3_value_text(sqliteValue)!))
+            storage = .string(String(
+                utf8Bytes: sqlite3_value_text(sqliteValue)!,
+                count: sqlite3_value_bytes(sqliteValue)))
         case SQLITE_BLOB:
             if let bytes = sqlite3_value_blob(sqliteValue) {
                 let count = Int(sqlite3_value_bytes(sqliteValue))
@@ -171,7 +173,9 @@ public struct DatabaseValue: Hashable {
         case SQLITE_FLOAT:
             storage = .double(sqlite3_column_double(sqliteStatement, index))
         case SQLITE_TEXT:
-            storage = .string(String(cString: sqlite3_column_text(sqliteStatement, index)))
+            storage = .string(String(
+                utf8Bytes: sqlite3_column_text(sqliteStatement, index)!,
+                count: sqlite3_column_bytes(sqliteStatement, index)))
         case SQLITE_BLOB:
             if let bytes = sqlite3_column_blob(sqliteStatement, index) {
                 let count = Int(sqlite3_column_bytes(sqliteStatement, index))

GRDB/Core/Support/StandardLibrary/StandardLibrary.swift

@@ -592,7 +592,9 @@ extension String: DatabaseValueConvertible, StatementColumnConvertible {
     ///     - sqliteStatement: A pointer to an SQLite statement.
     ///     - index: The column index.
     public init(sqliteStatement: SQLiteStatement, index: CInt) {
-        self = String(cString: sqlite3_column_text(sqliteStatement, index)!)
+        self = String(
+            utf8Bytes: sqlite3_column_text(sqliteStatement, index)!,
+            count: sqlite3_column_bytes(sqliteStatement, index))
     }
 
     /// Returns a TEXT database value.
@@ -622,7 +624,18 @@ extension String: DatabaseValueConvertible, StatementColumnConvertible {
     }
 
     public func bind(to sqliteStatement: SQLiteStatement, at index: CInt) -> CInt {
-        sqlite3_bind_text(sqliteStatement, index, self, -1, SQLITE_TRANSIENT)
+        var string = self
+        return string.withUTF8 { buffer in
+            guard let baseAddress = buffer.baseAddress else {
+                // baseAddress may be nil for an empty string
+                return sqlite3_bind_text(sqliteStatement, index, "", 0, SQLITE_TRANSIENT)
+            }
+            return sqlite3_bind_text(
+                sqliteStatement, index,
+                UnsafeRawPointer(baseAddress).assumingMemoryBound(to: CChar.self),
+                CInt(buffer.count),
+                SQLITE_TRANSIENT)
+        }
     }
 
     /// Calls the given closure after binding a statement argument.
@@ -633,8 +646,19 @@ extension String: DatabaseValueConvertible, StatementColumnConvertible {
     /// - parameter index: 1-based index to statement arguments.
     /// - parameter body: The closure to execute when argument is bound.
     func withBinding<T>(to sqliteStatement: SQLiteStatement, at index: CInt, do body: () throws -> T) throws -> T {
-        try withCString {
-            let code = sqlite3_bind_text(sqliteStatement, index, $0, -1, nil /* SQLITE_STATIC */)
+        var string = self
+        return try string.withUTF8 { buffer in
+            guard let baseAddress = buffer.baseAddress else {
+                // baseAddress may be nil for an empty string
+                let code = sqlite3_bind_text(sqliteStatement, index, "", 0, SQLITE_TRANSIENT)
+                try checkBindingSuccess(code: code, sqliteStatement: sqliteStatement)
+                return try body()
+            }
+            let code = sqlite3_bind_text(
+                sqliteStatement, index,
+                UnsafeRawPointer(baseAddress).assumingMemoryBound(to: CChar.self),
+                CInt(buffer.count),
+                nil /* SQLITE_STATIC */)
             try checkBindingSuccess(code: code, sqliteStatement: sqliteStatement)
             return try body()
         }

GRDB/Utils/Utils.swift

@@ -16,6 +16,21 @@ extension String {
     }
 }
 
+// MARK: - String and SQLite text
+
+extension String {
+    /// Creates a string from a buffer of UTF-8 bytes, repairing invalid
+    /// UTF-8.
+    ///
+    /// Unlike `String(cString:)`, the result may contain zero bytes: the
+    /// buffer is delimited by `count`, not by a nul terminator.
+    init(utf8Bytes: UnsafePointer<UInt8>, count: CInt) {
+        self = String(
+            decoding: UnsafeRawBufferPointer(start: utf8Bytes, count: Int(count)),
+            as: UTF8.self)
+    }
+}
+
 /// Return as many question marks separated with commas as the *count* argument.
 ///
 ///     databaseQuestionMarks(count: 3) // "?,?,?"

Tests/GRDBTests/Core/DatabaseFunctionTests.swift

@@ -89,7 +89,20 @@ class DatabaseFunctionTests: GRDBTestCase {
             XCTAssertEqual(try String.fetchOne(db, sql: "SELECT f()")!, "foo")
         }
     }
-
+
+    func testFunctionReturningStringWithNulCharacter() throws {
+        let dbQueue = try makeDatabaseQueue()
+        let fn = DatabaseFunction("f", argumentCount: 0) { dbValues in
+            // SQLite supports strings that contain a NUL character:
+            // https://sqlite.org/nulinstr.html
+            return "foo\u{0}bar"
+        }
+        try dbQueue.inDatabase { db in
+            db.add(function: fn)
+            XCTAssertEqual(try String.fetchOne(db, sql: "SELECT f()")!, "foo\u{0}bar")
+        }
+    }
+
     func testFunctionReturningData() throws {
         let dbQueue = try makeDatabaseQueue()
         let fn = DatabaseFunction("f", argumentCount: 0) { dbValues in
@@ -167,7 +180,22 @@ class DatabaseFunctionTests: GRDBTestCase {
             XCTAssertEqual(try String.fetchOne(db, sql: "SELECT f('foo')")!, "foo")
         }
     }
-
+
+    func testFunctionArgumentStringWithNulCharacter() throws {
+        let dbQueue = try makeDatabaseQueue()
+        let fn = DatabaseFunction("f", argumentCount: 1) { (dbValues: [DatabaseValue]) in
+            return String.fromDatabaseValue(dbValues[0])
+        }
+        try dbQueue.inDatabase { db in
+            db.add(function: fn)
+            // A NUL character can not be written in an SQL string literal:
+            // the string is passed as a statement argument.
+            XCTAssertEqual(
+                try String.fetchOne(db, sql: "SELECT f(?)", arguments: ["foo\u{0}bar"])!,
+                "foo\u{0}bar")
+        }
+    }
+
     func testFunctionArgumentBlob() throws {
         let dbQueue = try makeDatabaseQueue()
         let fn = DatabaseFunction("f", argumentCount: 1) { (dbValues: [DatabaseValue]) in

Tests/GRDBTests/Support/StringTests.swift

@@ -0,0 +1,74 @@
+import XCTest
+import GRDB
+
+final class StringTests: GRDBTestCase {
+
+    // A string that contains a NUL character. SQLite supports those
+    // strings: https://sqlite.org/nulinstr.html
+    private let nulString = "foo\u{0}bar"
+
+    func testStringDatabaseRoundTrip() throws {
+        let dbQueue = try makeDatabaseQueue()
+        try dbQueue.inDatabase { db in
+            func roundTrip(_ value: String) throws -> Bool {
+                guard let back = try String.fetchOne(db, sql: "SELECT ?", arguments: [value]) else {
+                    XCTFail("Failed to fetch a String")
+                    return false
+                }
+                return back == value
+            }
+
+            XCTAssertTrue(try roundTrip(""))
+            XCTAssertTrue(try roundTrip("foo"))
+            XCTAssertTrue(try roundTrip("'fooéı👨👨🏿🇫🇷🇨🇮'"))
+            XCTAssertTrue(try roundTrip(nulString))
+        }
+    }
+
+    func testStringDatabaseValueRoundTrip() throws {
+        let dbQueue = try makeDatabaseQueue()
+        try dbQueue.inDatabase { db in
+            func roundTrip(_ value: String) throws -> Bool {
+                let row = try Row.fetchOne(db, sql: "SELECT ?", arguments: [value])!
+                guard let back = String.fromDatabaseValue(row[0]) else {
+                    XCTFail("Failed to convert from DatabaseValue to String")
+                    return false
+                }
+                return back == value
+            }
+
+            XCTAssertTrue(try roundTrip(""))
+            XCTAssertTrue(try roundTrip("foo"))
+            XCTAssertTrue(try roundTrip("'fooéı👨👨🏿🇫🇷🇨🇮'"))
+            XCTAssertTrue(try roundTrip(nulString))
+        }
+    }
+
+    // Statements execute with temporary bindings, and do not copy their
+    // string arguments. See `String.withBinding(to:at:do:)`.
+    func testStringWithTemporaryBinding() throws {
+        let dbQueue = try makeDatabaseQueue()
+        try dbQueue.inDatabase { db in
+            try db.execute(sql: "CREATE TABLE t (value TEXT)")
+            try db.execute(sql: "INSERT INTO t VALUES (?)", arguments: [nulString])
+
+            let fetched = try String.fetchOne(db, sql: "SELECT value FROM t")
+            XCTAssertEqual(fetched, nulString)
+        }
+    }
+
+    // Statements that are given their arguments before execution copy their
+    // string arguments. See `String.bind(to:at:)`.
+    func testStringWithStatementArguments() throws {
+        let dbQueue = try makeDatabaseQueue()
+        try dbQueue.inDatabase { db in
+            try db.execute(sql: "CREATE TABLE t (value TEXT)")
+            let statement = try db.makeStatement(sql: "INSERT INTO t VALUES (?)")
+            statement.arguments = [nulString]
+            try statement.execute()
+
+            let fetched = try String.fetchOne(db, sql: "SELECT value FROM t")
+            XCTAssertEqual(fetched, nulString)
+        }
+    }
+}

Tests/Performance/GRDBPerformance/FetchStringTests.swift

@@ -0,0 +1,118 @@
+import XCTest
+import GRDB
+
+/// Here we test the decoding of string columns. The other fetch tests in
+/// this target use an all-INT model, and never decode a string.
+class FetchStringTests: XCTestCase {
+    static let expectedRowCount = 200_000
+    static let personRowCount = 100_000
+    static let recordFetchesPerIteration = 5
+    static let fetchesPerIteration = 10
+
+    private struct Person: Codable, FetchableRecord, PersistableRecord {
+        static let databaseTableName = "person"
+
+        var firstName: String
+        var lastName: String
+        var email: String
+        var address: String
+        var age: Int
+        var height: Double
+    }
+
+    func test_shortString_fetch_performance() throws {
+        try measureFetch(of: String.self, from: "short")
+    }
+
+    func test_longString_fetch_performance() throws {
+        try measureFetch(of: String.self, from: "long")
+    }
+
+    func test_shortString_databaseValue_performance() throws {
+        try measureFetch(of: DatabaseValue.self, from: "short")
+    }
+
+    func test_longString_databaseValue_performance() throws {
+        try measureFetch(of: DatabaseValue.self, from: "long")
+    }
+
+    func test_record_fetch_performance() throws {
+        let dbQueue = try makePersonDatabase()
+        measure {
+            try! dbQueue.read { db in
+                for _ in 0..<Self.recordFetchesPerIteration {
+                    let people = try Person.fetchAll(db)
+                    XCTAssertEqual(people.count, Self.personRowCount)
+                }
+            }
+        }
+    }
+
+    private func measureFetch<Value: DatabaseValueConvertible>(
+        of type: Value.Type,
+        from column: String) throws
+    {
+        let dbQueue = try makeDatabaseQueue()
+        measure {
+            try! dbQueue.read { db in
+                for _ in 0..<Self.fetchesPerIteration {
+                    let values = try Value.fetchAll(db, sql: "SELECT \(column) FROM item")
+                    XCTAssertEqual(values.count, Self.expectedRowCount)
+                }
+            }
+        }
+    }
+
+    private func makePersonDatabase() throws -> DatabaseQueue {
+        let url = URL(fileURLWithPath: NSTemporaryDirectory())
+            .appendingPathComponent("GRDBPerformancePeople.sqlite")
+        let dbQueue = try DatabaseQueue(path: url.path)
+        try dbQueue.write { db in
+            if try db.tableExists("person"),
+               try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM person") == Self.personRowCount
+            {
+                return
+            }
+            try db.execute(sql: "DROP TABLE IF EXISTS person")
+            try db.create(table: "person") { table in
+                table.column("firstName", .text)
+                table.column("lastName", .text)
+                table.column("email", .text)
+                table.column("address", .text)
+                table.column("age", .integer)
+                table.column("height", .double)
+            }
+            let statement = try db.makeStatement(sql: """
+                INSERT INTO person (firstName, lastName, email, address, age, height)
+                VALUES (?, ?, ?, ?, ?, ?)
+                """)
+            for index in 0..<Self.personRowCount {
+                try statement.execute(arguments: [
+                    "Arthur", "Dubois", "arthur.dubois@example.com",
+                    "12 Rue de la Paix, 75002 Paris", 20 + index % 60, 1.60 + Double(index % 40) / 100,
+                ])
+            }
+        }
+        return dbQueue
+    }
+
+    private func makeDatabaseQueue() throws -> DatabaseQueue {
+        let url = URL(fileURLWithPath: NSTemporaryDirectory())
+            .appendingPathComponent("GRDBPerformanceStrings.sqlite")
+        let dbQueue = try DatabaseQueue(path: url.path)
+        try dbQueue.write { db in
+            if try db.tableExists("item"),
+               try Int.fetchOne(db, sql: "SELECT COUNT(*) FROM item") == Self.expectedRowCount
+            {
+                return
+            }
+            try db.execute(sql: "DROP TABLE IF EXISTS item")
+            try db.execute(sql: "CREATE TABLE item (short TEXT, long TEXT)")
+            let statement = try db.makeStatement(sql: "INSERT INTO item (short, long) VALUES (?, ?)")
+            for _ in 0..<Self.expectedRowCount {
+                try statement.execute(arguments: [ArgumentsTests.shortString, ArgumentsTests.longString])
+            }
+        }
+        return dbQueue
+    }
+}