diff --git a/.github/workflows/scan-validation.yml b/.github/workflows/scan-validation.yml new file mode 100644 index 0000000..a036b12 --- /dev/null +++ b/.github/workflows/scan-validation.yml @@ -0,0 +1,39 @@ +name: Scan validation + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + macos: + runs-on: macos-15 + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - name: Select project toolchain + run: | + sudo xcode-select --switch /Applications/Xcode_26.3.app/Contents/Developer + swift --version + - name: Unit and mounted-volume regression tests + env: + CLEARDISK_TEST_MOUNTS: "1" + run: swift test + - name: Parallel measurement race checks + run: swift test --sanitize=thread --filter DirectoryMeasurementTests.testParallelMeasurementsHaveIndependentState + - name: Universal app and final signature + run: | + bash scripts/build_app.sh + codesign --verify --deep --strict ClearDisk.app + - name: Staging failure tests + run: python3 scripts/test_stage_sign_app.py + - name: Benchmark harness smoke test + run: | + fixture=$(mktemp -d) + trap 'rm -rf "$fixture"' EXIT + mkdir "$fixture/one" "$fixture/two" + dd if=/dev/zero of="$fixture/one/data" bs=4096 count=4 + dd if=/dev/zero of="$fixture/two/data" bs=4096 count=8 + BENCH_RUNS=2 bash scripts/benchmark_scan.sh "$fixture/one" "$fixture/two" diff --git a/Resources/en.lproj/Localizable.strings b/Resources/en.lproj/Localizable.strings index a86e8dc..40d4d90 100644 --- a/Resources/en.lproj/Localizable.strings +++ b/Resources/en.lproj/Localizable.strings @@ -1,2 +1,6 @@ /* ClearDisk — English (en) is the development language. Keys are the English source strings themselves, so this table is intentionally empty. */ + +// Scan completeness and resource policy +"Analysis incomplete: some locations could not be measured." = "Analysis incomplete: some locations could not be measured."; +"Scan stopped to reduce power or thermal load. Start again to scan with fewer workers." = "Scan stopped to reduce power or thermal load. Start again to scan with fewer workers."; diff --git a/Resources/tr.lproj/Localizable.strings b/Resources/tr.lproj/Localizable.strings index 3bd100b..fd30c3e 100644 --- a/Resources/tr.lproj/Localizable.strings +++ b/Resources/tr.lproj/Localizable.strings @@ -590,3 +590,7 @@ "The downloaded update payload, staging copy, updater state, and updater logs." = "İndirilen güncelleme paketi, hazırlık kopyası, güncelleyici durumu ve güncelleyici günlükleri."; "The installed app, its settings, accounts, and documents." = "Kurulu uygulama, ayarları, hesapları ve belgeleri."; "Your account, playlists, local files, and explicitly downloaded offline library." = "Hesabınız, çalma listeleriniz, yerel dosyalarınız ve bilerek indirdiğiniz çevrimdışı kitaplığınız."; + +// Scan completeness and resource policy +"Analysis incomplete: some locations could not be measured." = "Analiz tamamlanamadı: bazı konumlar ölçülemedi."; +"Scan stopped to reduce power or thermal load. Start again to scan with fewer workers." = "Güç tüketimini veya ısınmayı azaltmak için tarama durduruldu. Daha az eşzamanlı işlemle taramak için yeniden başlatın."; diff --git a/Sources/ClearDisk/AppCacheCatalog.swift b/Sources/ClearDisk/AppCacheCatalog.swift index 209191a..99c5348 100644 --- a/Sources/ClearDisk/AppCacheCatalog.swift +++ b/Sources/ClearDisk/AppCacheCatalog.swift @@ -1,6 +1,6 @@ import Foundation -enum CacheSection: String, CaseIterable { +enum CacheSection: String, CaseIterable, Sendable { case app case developer @@ -19,7 +19,7 @@ enum CacheSection: String, CaseIterable { } } -struct CacheSafetyDetails { +struct CacheSafetyDetails: Sendable { let removes: String let keeps: String let note: String diff --git a/Sources/ClearDisk/DirectoryMeasurement.swift b/Sources/ClearDisk/DirectoryMeasurement.swift new file mode 100644 index 0000000..75e4e93 --- /dev/null +++ b/Sources/ClearDisk/DirectoryMeasurement.swift @@ -0,0 +1,86 @@ +import Darwin +import Foundation + +struct DirectoryMeasurement: Sendable { + var bytes: Int64 = 0 + var files: Int = 0 + var errorCount: Int = 0 + var firstError: String? + var isComplete: Bool { errorCount == 0 } + + mutating func recordError(path: String, code: Int32) { + errorCount += 1 + if firstError == nil { + firstError = "\(path): \(NSError(domain: NSPOSIXErrorDomain, code: Int(code)).localizedDescription)" + } + } + + /// Allocated bytes attributed to directory entries, not a promise of space reclaimed. + /// Hard links share an allocation; deleting one of several links need not free any blocks. + static func measure(path: String, isCancelled: () -> Bool = { false }) -> Self { + var result = Self() + guard !isCancelled() else { + result.recordError(path: path, code: ECANCELED) + return result + } + guard let cPath = strdup(path) else { + result.recordError(path: path, code: ENOMEM) + return result + } + defer { free(cPath) } + + // Missing optional cache roots are empty. Other errors must remain visible. + var rootStat = stat() + if lstat(cPath, &rootStat) != 0 { + let code = errno + if code != ENOENT { result.recordError(path: path, code: code) } + return result + } + + var paths: [UnsafeMutablePointer?] = [cPath, nil] + // Keep argv storage alive for the entire traversal; each call owns its FTS handle. + paths.withUnsafeMutableBufferPointer { buffer in + guard let tree = fts_open(buffer.baseAddress!, FTS_PHYSICAL | FTS_NOCHDIR | FTS_XDEV, nil) else { + result.recordError(path: path, code: errno) + return + } + defer { + if fts_close(tree) != 0 { result.recordError(path: path, code: errno) } + } + while true { + if isCancelled() { + result.recordError(path: path, code: ECANCELED) + break + } + errno = 0 + guard let node = fts_read(tree) else { + let code = errno + if code != 0 { result.recordError(path: path, code: code) } + break + } + switch Int32(node.pointee.fts_info) { + case FTS_F: + let metadata = node.pointee.fts_statp.pointee + let (allocated, overflow) = max(0, Int64(metadata.st_blocks)).multipliedReportingOverflow(by: 512) + let (total, sumOverflow) = result.bytes.addingReportingOverflow( + allocated / max(1, Int64(metadata.st_nlink)) + ) + if overflow || sumOverflow { + result.recordError(path: path, code: EOVERFLOW) + return + } + result.bytes = total + result.files += 1 + case FTS_DC: + result.recordError(path: String(cString: node.pointee.fts_path), code: ELOOP) + case FTS_DNR, FTS_ERR, FTS_NS: + let code = node.pointee.fts_errno + result.recordError(path: String(cString: node.pointee.fts_path), code: code == 0 ? EIO : code) + default: + break + } + } + } + return result + } +} diff --git a/Sources/ClearDisk/DiskMonitor.swift b/Sources/ClearDisk/DiskMonitor.swift index b01d025..413db95 100644 --- a/Sources/ClearDisk/DiskMonitor.swift +++ b/Sources/ClearDisk/DiskMonitor.swift @@ -188,11 +188,6 @@ class DiskMonitor: ObservableObject { } } - /// Check if a path is readable before scanning - private func canAccess(path: String) -> Bool { - FileManager.default.isReadableFile(atPath: path) - } - private var isScanInProgress = false private var isCacheScanInProgress = false @@ -211,12 +206,10 @@ class DiskMonitor: ObservableObject { guard !isScanInProgress, !isCacheScanInProgress else { return } // Prevent concurrent scans isScanInProgress = true isScanning = true + inaccessiblePaths = [] scanProgress = 0.03 scanStatusText = L("Preparing storage analysis...") DispatchQueue.global(qos: .utility).async { [weak self] in - // Reset scan status - var inaccessible: [String] = [] - self?.publishScanProgress(0.08, L("Reading disk capacity...")) self?.scanDiskCapacity() self?.publishScanProgress(0.18, L("Analyzing storage categories...")) @@ -230,22 +223,11 @@ class DiskMonitor: ObservableObject { self?.publishScanProgress(0.92, L("Measuring Trash and finalizing results...")) let trashBytes = self?.trashSize() ?? 0 - // Check which known cache paths are inaccessible. - let cachePaths = self?.knownCachePaths() ?? [] - for (name, path) in cachePaths { - let expanded = (path as NSString).expandingTildeInPath - let parent = (expanded as NSString).deletingLastPathComponent - if FileManager.default.fileExists(atPath: parent) && !(self?.canAccess(path: expanded) ?? true) { - inaccessible.append(name) - } - } - DispatchQueue.main.async { self?.isScanning = false self?.isScanInProgress = false self?.scanProgress = 1 - self?.scanStatusText = L("Analysis complete") - self?.inaccessiblePaths = inaccessible + self?.scanStatusText = self?.inaccessiblePaths.isEmpty == true ? L("Analysis complete") : L("Analysis incomplete: some locations could not be measured.") self?.hasCompletedFirstScan = true self?.hasCompletedCacheScan = true self?.lastFullScanAt = Date() @@ -269,6 +251,7 @@ class DiskMonitor: ObservableObject { guard !isScanInProgress, !isCacheScanInProgress else { return } isCacheScanInProgress = true isScanningCaches = true + inaccessiblePaths = [] DispatchQueue.global(qos: .utility).async { [weak self] in self?.scanKnownCaches() @@ -499,16 +482,24 @@ class DiskMonitor: ObservableObject { ("Photos", "photo.fill", ["\(home)/Pictures"]), ] - var cats: [DiskCategory] = [] + let opQueue = ScanOperationScheduler.shared.queue + let collected = ScanResults() + for (name, icon, paths) in categoryPaths { - var totalSize: Int64 = 0 - for path in paths { - totalSize += directorySize(path: path) - } - if totalSize > 0 { - cats.append(DiskCategory(name: name, icon: icon, size: totalSize)) + opQueue.addOperation { [weak self] in + guard let self else { return } + var totalSize: Int64 = 0 + for path in paths { + totalSize += self.directorySize(path: path) + } + if totalSize > 0 { + let category = DiskCategory(name: name, icon: icon, size: totalSize) + collected.append(category) + } } } + opQueue.waitUntilAllOperationsAreFinished() + var cats = collected.snapshot() cats.sort { $0.size > $1.size } @@ -838,37 +829,45 @@ class DiskMonitor: ObservableObject { private func scanKnownCaches() { let definitions = allKnownCacheDefinitions() - var caches: [DevCache] = [] + let opQueue = ScanOperationScheduler.shared.queue + let collected = ScanResults() + for entry in definitions { - let size = directorySize(path: entry.path) - if size > 1_048_576 { // Only show if > 1MB - let lastAccessed = lastModifiedDate(path: entry.path) - let daysSinceAccess = daysSince(lastAccessed) - let suggestion = generateSuggestion(name: entry.name, size: size, daysSinceAccess: daysSinceAccess) - // Resolve DerivedData subfolders to project names - var detail: String? = nil - if entry.rawName == "Xcode DerivedData" { - detail = derivedDataProjectSummary() + opQueue.addOperation { [weak self] in + guard let self else { return } + let size = self.directorySize(path: entry.path) + if size > 1_048_576 { // Only show if > 1MB + let lastAccessed = self.lastModifiedDate(path: entry.path) + let daysSinceAccess = self.daysSince(lastAccessed) + let suggestion = self.generateSuggestion(name: entry.name, size: size, daysSinceAccess: daysSinceAccess) + // Resolve DerivedData subfolders to project names + var detail: String? = nil + if entry.rawName == "Xcode DerivedData" { + detail = self.derivedDataProjectSummary() + } + + let cache = DevCache( + name: entry.name, + rawName: entry.rawName, + icon: entry.icon, + path: entry.path, + size: size, + lastAccessed: lastAccessed, + daysSinceAccess: daysSinceAccess, + suggestion: suggestion, + riskLevel: entry.riskLevel, + cacheDescription: entry.description, + group: entry.group, + section: entry.section, + safetyDetails: entry.safetyDetails, + detail: detail + ) + collected.append(cache) } - - caches.append(DevCache( - name: entry.name, - rawName: entry.rawName, - icon: entry.icon, - path: entry.path, - size: size, - lastAccessed: lastAccessed, - daysSinceAccess: daysSinceAccess, - suggestion: suggestion, - riskLevel: entry.riskLevel, - cacheDescription: entry.description, - group: entry.group, - section: entry.section, - safetyDetails: entry.safetyDetails, - detail: detail - )) } } + opQueue.waitUntilAllOperationsAreFinished() + var caches = collected.snapshot() caches.sort { $0.size > $1.size } @@ -951,8 +950,6 @@ class DiskMonitor: ObservableObject { private func scanLargeFiles() { let home = FileManager.default.homeDirectoryForCurrentUser.path let threshold: Int64 = 100_000_000 // 100MB - var files: [LargeFile] = [] - let scanDirs = [ "\(home)/Downloads", "\(home)/Documents", @@ -962,10 +959,22 @@ class DiskMonitor: ObservableObject { "\(home)/Pictures", ] + let opQueue = ScanOperationScheduler.shared.queue + let collected = ScanResults() + for dir in scanDirs { - let folderName = (dir as NSString).lastPathComponent - findLargeFiles(in: dir, folder: folderName, threshold: threshold, results: &files, maxDepth: 3, currentDepth: 0) + opQueue.addOperation { [weak self] in + guard let self else { return } + let folderName = (dir as NSString).lastPathComponent + var dirFiles: [LargeFile] = [] + self.findLargeFiles(in: dir, folder: folderName, threshold: threshold, results: &dirFiles, maxDepth: 3, currentDepth: 0) + if !dirFiles.isEmpty { + collected.append(contentsOf: dirFiles) + } + } } + opQueue.waitUntilAllOperationsAreFinished() + var files = collected.snapshot() files.sort { $0.size > $1.size } @@ -1078,8 +1087,11 @@ class DiskMonitor: ObservableObject { let fullPath = (path as NSString).appendingPathComponent(item) if let reason = moveToTrash(path: fullPath), failure == nil { failure = reason } } - let remaining = directorySize(path: path) - return (max(0, sizeBefore - remaining), failure) + let remaining = DirectoryMeasurement.measure(path: path) + guard remaining.isComplete else { + return (0, failure ?? remaining.firstError) + } + return (max(0, sizeBefore - remaining.bytes), failure) } func cleanDevCache(_ cache: DevCache) { @@ -1122,21 +1134,29 @@ class DiskMonitor: ObservableObject { DispatchQueue.global(qos: .userInitiated).async { [weak self] in guard let self else { return } let fm = FileManager.default - let sizeBefore = self.directorySize(path: trashPath) - if let contents = try? fm.contentsOfDirectory(atPath: trashPath) { - for item in contents { - let fullPath = (trashPath as NSString).appendingPathComponent(item) - try? fm.removeItem(atPath: fullPath) // Trash empty = permanent delete (intended) + let before = DirectoryMeasurement.measure(path: trashPath) + var failure: String? = before.firstError + do { + for item in try fm.contentsOfDirectory(atPath: trashPath) { + do { + try fm.removeItem(atPath: (trashPath as NSString).appendingPathComponent(item)) + } catch { + if failure == nil { failure = error.localizedDescription } + } } + } catch { + if failure == nil { failure = error.localizedDescription } + } + let remaining = DirectoryMeasurement.measure(path: trashPath) + if failure == nil { failure = remaining.firstError } + if failure == nil && remaining.bytes > 0 { + failure = L("Some items in the Trash could not be removed.") } - let remaining = self.directorySize(path: trashPath) + let freed = before.isComplete && remaining.isComplete + ? max(0, before.bytes - remaining.bytes) : 0 + let finalFailure = failure DispatchQueue.main.async { - self.finishClean( - title: "Trash", - freed: max(0, sizeBefore - remaining), - failure: remaining > 0 ? L("Some items in the Trash could not be removed.") : nil, - outcome: .reclaimedSpace - ) + self.finishClean(title: "Trash", freed: freed, failure: finalFailure, outcome: .reclaimedSpace) } } } @@ -1261,12 +1281,20 @@ class DiskMonitor: ObservableObject { ] private func scanProjectArtifacts() { - var artifacts: [ProjectArtifact] = [] + let opQueue = ScanOperationScheduler.shared.queue + let collected = ScanResults() let fm = FileManager.default for root in projectScanRoots() { guard fm.fileExists(atPath: root) else { continue } - findProjectArtifacts(in: root, results: &artifacts, maxDepth: 5, currentDepth: 0) + opQueue.addOperation { [weak self] in + guard let self else { return } + var localResults: [ProjectArtifact] = [] + self.findProjectArtifacts(in: root, results: &localResults, maxDepth: 5, currentDepth: 0) + if !localResults.isEmpty { + collected.append(contentsOf: localResults) + } + } } // Repositories kept directly in `~` were invisible: every root above is a subdirectory of the @@ -1286,9 +1314,18 @@ class DiskMonitor: ObservableObject { let full = (home as NSString).appendingPathComponent(entry) var isDir: ObjCBool = false guard fm.fileExists(atPath: full, isDirectory: &isDir), isDir.boolValue else { continue } - findProjectArtifacts(in: full, results: &artifacts, maxDepth: 1, currentDepth: 0) + opQueue.addOperation { [weak self] in + guard let self else { return } + var localResults: [ProjectArtifact] = [] + self.findProjectArtifacts(in: full, results: &localResults, maxDepth: 1, currentDepth: 0) + if !localResults.isEmpty { + collected.append(contentsOf: localResults) + } + } } } + opQueue.waitUntilAllOperationsAreFinished() + var artifacts = collected.snapshot() // Defensive: the scan roots are disjoint today, but a duplicate row double-counts its size in // the totals and makes the second clean fail on an already-trashed path. Keep the first sighting. @@ -1462,31 +1499,19 @@ class DiskMonitor: ObservableObject { // MARK: - Helpers func directorySize(path: String) -> Int64 { - let fm = FileManager.default - var totalSize: Int64 = 0 - - guard let enumerator = fm.enumerator( - at: URL(fileURLWithPath: path), - includingPropertiesForKeys: [.totalFileAllocatedSizeKey, .fileAllocatedSizeKey, .isRegularFileKey, .linkCountKey], - options: [], // Don't skip hidden files — caches often contain them - errorHandler: nil - ) else { return 0 } - - for case let fileURL as URL in enumerator { - guard let values = try? fileURL.resourceValues(forKeys: [.totalFileAllocatedSizeKey, .fileAllocatedSizeKey, .isRegularFileKey, .linkCountKey]), - values.isRegularFile == true else { continue } - // Use totalFileAllocatedSize (accounts for sparse files like Docker.raw) - // Falls back to fileAllocatedSize if total isn't available - let size = values.totalFileAllocatedSize ?? values.fileAllocatedSize ?? 0 - // Hardlink-aware: a file with N hard links only frees `size / N` bytes when one link is removed. - // This is critical for pnpm / Bun / Yarn Berry / Cargo registry stores that hardlink into project caches — - // otherwise we wildly overestimate how much disk space cleaning would actually free. - let links = max(values.linkCount ?? 1, 1) - totalSize += Int64(size / links) + let measurement = DirectoryMeasurement.measure(path: path) + if let error = measurement.firstError { + DispatchQueue.main.async { [weak self] in + guard let self else { return } + if self.inaccessiblePaths.count < 100, !self.inaccessiblePaths.contains(error) { + self.inaccessiblePaths.append(error) + } + } } - - return totalSize + // Partial sizes must not become cache candidates or cleanup promises. + return measurement.isComplete ? measurement.bytes : 0 } + } // MARK: - Permission State @@ -1502,14 +1527,14 @@ enum CleanOutcome { } // MARK: - Models -struct DiskCategory: Identifiable { +struct DiskCategory: Identifiable, Sendable { let id = UUID() let name: String let icon: String let size: Int64 } -struct DevCache: Identifiable { +struct DevCache: Identifiable, Sendable { let id: UUID /// Display name, localized for the user's language. let name: String @@ -1584,7 +1609,7 @@ struct DevCache: Identifiable { } } -struct LargeFile: Identifiable { +struct LargeFile: Identifiable, Sendable { let id = UUID() let name: String let path: String @@ -1601,7 +1626,7 @@ struct CleanFailure: Identifiable { let isPermission: Bool // true → the fix is granting Full Disk Access } -struct ProjectArtifact: Identifiable { +struct ProjectArtifact: Identifiable, Sendable { let id = UUID() let projectName: String // e.g. "my-react-app" let projectPath: String // full path to project root diff --git a/Sources/ClearDisk/DiskSpaceWindow.swift b/Sources/ClearDisk/DiskSpaceWindow.swift index db7c90e..44f726e 100644 --- a/Sources/ClearDisk/DiskSpaceWindow.swift +++ b/Sources/ClearDisk/DiskSpaceWindow.swift @@ -115,6 +115,8 @@ final class DiskSpaceStore: ObservableObject { private let scanner = DiskScanner() private var scanTask: Task? + private var resourceObservers: [NSObjectProtocol] = [] + private var scanResourcePolicy = ScanResourcePolicy.current private var activeScanID: UUID? private var scannedRootPath: String? private var activeScanRootURL: URL? @@ -127,9 +129,24 @@ final class DiskSpaceStore: ObservableObject { init() { reloadLocations() + for name in [Notification.Name.NSProcessInfoPowerStateDidChange, ProcessInfo.thermalStateDidChangeNotification] { + resourceObservers.append(NotificationCenter.default.addObserver(forName: name, object: nil, queue: .main) { [weak self] _ in + Task { @MainActor [weak self] in self?.handleResourceChange() } + }) + } + } + + private func handleResourceChange() { + let policy = ScanResourcePolicy.current + guard phase == .scanning, policy.workers < scanResourcePolicy.workers else { return } + // Backend budgets are fixed for one request. Cancel rather than continue at the old + // higher budget; a user-initiated retry takes a fresh, conservative policy snapshot. + stopScan() + phase = .failed(L("Scan stopped to reduce power or thermal load. Start again to scan with fewer workers.")) } deinit { + resourceObservers.forEach { NotificationCenter.default.removeObserver($0) } scanTask?.cancel() } @@ -325,6 +342,7 @@ final class DiskSpaceStore: ObservableObject { let scanID = UUID() activeScanID = scanID phase = .scanning + scanResourcePolicy = .current progress = nil issues = [] // A rescan must not retain the previous full-volume tree while a second @@ -371,6 +389,7 @@ final class DiskSpaceStore: ObservableObject { scanID: UUID ) async throws { var collectedIssues: [DiskScanIssue] = [] + let limits = scanResourcePolicy.backendLimits let request = DiskScanRequest( rootURL: rootURL, includesHiddenItems: true, @@ -379,9 +398,9 @@ final class DiskSpaceStore: ObservableObject { ? preservedDirectoryURLs(for: location) : [], maximumMaterializedDepth: Self.maximumMaterializedDepth, - atomicSummaryWorkerLimit: Self.scanWorkerLimit(for: rootURL), - directoryClassificationWorkerLimit: Self.scanWorkerLimit(for: rootURL), - directoryTraversalWorkerLimit: Self.scanWorkerLimit(for: rootURL) + atomicSummaryWorkerLimit: limits.atomic, + directoryClassificationWorkerLimit: limits.classification, + directoryTraversalWorkerLimit: limits.traversal ) for try await event in scanner.events(for: request) { @@ -427,14 +446,15 @@ final class DiskSpaceStore: ObservableObject { for source in group.sources { var sourceSnapshot: DiskScanSnapshot? + let limits = scanResourcePolicy.backendLimits let request = DiskScanRequest( rootURL: source.url, includesHiddenItems: true, expandsPackages: false, maximumMaterializedDepth: Self.maximumMaterializedDepth, - atomicSummaryWorkerLimit: Self.scanWorkerLimit(for: source.url), - directoryClassificationWorkerLimit: Self.scanWorkerLimit(for: source.url), - directoryTraversalWorkerLimit: Self.scanWorkerLimit(for: source.url) + atomicSummaryWorkerLimit: limits.atomic, + directoryClassificationWorkerLimit: limits.classification, + directoryTraversalWorkerLimit: limits.traversal ) for try await event in scanner.events(for: request) { @@ -951,12 +971,6 @@ final class DiskSpaceStore: ObservableObject { candidate != root && candidate.hasPrefix(root.hasSuffix("/") ? root : root + "/") } - private static func scanWorkerLimit(for rootURL: URL) -> Int { - // A full-volume walk is long-running background work. One worker keeps the Mac usable; - // focused folder scans may use two workers without monopolizing the CPU. - normalizedPath(rootURL.path) == "/" ? 1 : 2 - } - private var locationRootNode: DiskFileNode? { guard let snapshot else { return nil } if selectedLocation.id == scannedRootPath { diff --git a/Sources/ClearDisk/MainView.swift b/Sources/ClearDisk/MainView.swift index 0247f07..768e2df 100644 --- a/Sources/ClearDisk/MainView.swift +++ b/Sources/ClearDisk/MainView.swift @@ -115,6 +115,15 @@ struct MainView: View { .frame(width: Layout.contentWidth, height: Layout.popoverHeight) let base = content + .safeAreaInset(edge: .bottom) { + if !diskMonitor.inaccessiblePaths.isEmpty { + Label("Analysis incomplete: some locations could not be measured.", systemImage: "exclamationmark.triangle") + .font(.caption) + .foregroundStyle(.orange) + .padding(8) + .help(diskMonitor.inaccessiblePaths.joined(separator: "\n")) + } + } .frame(width: Layout.popoverWidth, height: Layout.popoverHeight) .overlay { ZStack { diff --git a/Sources/ClearDisk/ScanResourcePolicy.swift b/Sources/ClearDisk/ScanResourcePolicy.swift new file mode 100644 index 0000000..e8b89b2 --- /dev/null +++ b/Sources/ClearDisk/ScanResourcePolicy.swift @@ -0,0 +1,64 @@ +import Foundation + +/// Conservative per-stage budgets. QoS is a scheduling hint, not CPU affinity. +struct ScanResourcePolicy: Equatable, Sendable { + let workers: Int + + init(cores: Int, lowPower: Bool, thermalState: ProcessInfo.ThermalState) { + // A bounded two-worker default also avoids assuming every volume is an NVMe SSD. + workers = cores <= 2 || lowPower || thermalState != .nominal ? 1 : 2 + } + + var backendLimits: (traversal: Int, classification: Int, atomic: Int) { + // Classification is per directory; keep it serial to avoid multiplying the budget. + (traversal: workers, classification: 1, atomic: workers) + } + + static var current: Self { + let info = ProcessInfo.processInfo + return Self(cores: info.activeProcessorCount, lowPower: info.isLowPowerModeEnabled, + thermalState: info.thermalState) + } +} + +/// The legacy scanner shares one queue across all stages and monitor instances. +/// Changes throttle admission; already running synchronous filesystem calls must finish. +final class ScanOperationScheduler: @unchecked Sendable { + static let shared = ScanOperationScheduler() + let queue = OperationQueue() + private var observers: [NSObjectProtocol] = [] + + private init() { + queue.qualityOfService = .utility + queue.maxConcurrentOperationCount = ScanResourcePolicy.current.workers + for name in [Notification.Name.NSProcessInfoPowerStateDidChange, ProcessInfo.thermalStateDidChangeNotification] { + observers.append(NotificationCenter.default.addObserver(forName: name, object: nil, queue: nil) { [weak self] _ in + self?.queue.maxConcurrentOperationCount = ScanResourcePolicy.current.workers + }) + } + } + + deinit { + observers.forEach { NotificationCenter.default.removeObserver($0) } + } +} + +/// Keep result storage synchronized without capturing a mutable array in @Sendable closures. +final class ScanResults: @unchecked Sendable { + private let lock = NSLock() + private var values: [Element] = [] + + func append(_ value: Element) { append(contentsOf: [value]) } + + func append(contentsOf additions: [Element]) { + lock.lock() + defer { lock.unlock() } + values.append(contentsOf: additions) + } + + func snapshot() -> [Element] { + lock.lock() + defer { lock.unlock() } + return values + } +} diff --git a/Tests/ClearDiskTests/DirectoryMeasurementTests.swift b/Tests/ClearDiskTests/DirectoryMeasurementTests.swift new file mode 100644 index 0000000..9bfbeb7 --- /dev/null +++ b/Tests/ClearDiskTests/DirectoryMeasurementTests.swift @@ -0,0 +1,147 @@ +import Darwin +import Foundation +import XCTest +@testable import ClearDisk + +final class DirectoryMeasurementTests: XCTestCase { + private func fixture() throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + addTeardownBlock { try? FileManager.default.removeItem(at: url) } + return url + } + + private func foundationBytes(_ root: URL) throws -> Int64 { + let keys: Set = [.totalFileAllocatedSizeKey, .fileAllocatedSizeKey, .isRegularFileKey, .linkCountKey] + let iterator = try XCTUnwrap(FileManager.default.enumerator(at: root, includingPropertiesForKeys: Array(keys))) + var total: Int64 = 0 + for case let url as URL in iterator { + let values = try url.resourceValues(forKeys: keys) + if values.isRegularFile == true { + total += Int64((values.totalFileAllocatedSize ?? values.fileAllocatedSize ?? 0) / max(1, values.linkCount ?? 1)) + } + } + return total + } + + func testHiddenFilesHardlinksSparseFilesAndSymlinkCycle() throws { + let root = try fixture() + let original = root.appendingPathComponent(".hidden") + try Data(repeating: 42, count: 16384).write(to: original) + try FileManager.default.linkItem(at: original, to: root.appendingPathComponent("hardlink")) + let sparse = root.appendingPathComponent("sparse") + FileManager.default.createFile(atPath: sparse.path, contents: nil) + let handle = try FileHandle(forWritingTo: sparse) + try handle.truncate(atOffset: 1 << 30) + try handle.close() + try FileManager.default.createSymbolicLink(at: root.appendingPathComponent("cycle"), withDestinationURL: root) + let measured = DirectoryMeasurement.measure(path: root.path) + XCTAssertTrue(measured.isComplete, measured.firstError ?? "") + XCTAssertEqual(measured.files, 3) + XCTAssertEqual(measured.bytes, try foundationBytes(root)) + XCTAssertLessThan(measured.bytes, 1 << 30) + } + + func testMissingRootIsEmptyButInvalidParentIsAnError() throws { + let root = try fixture() + XCTAssertTrue(DirectoryMeasurement.measure(path: root.appendingPathComponent("missing").path).isComplete) + let file = root.appendingPathComponent("file") + try Data([1]).write(to: file) + let result = DirectoryMeasurement.measure(path: file.appendingPathComponent("child").path) + XCTAssertFalse(result.isComplete) + XCTAssertNotNil(result.firstError) + } + + func testResourceForkMatchesFoundationAllocation() throws { + let root = try fixture() + let file = root.appendingPathComponent("forked") + try Data(repeating: 1, count: 4096).write(to: file) + let descriptor = open(file.path + "/..namedfork/rsrc", O_WRONLY | O_CREAT, 0o600) + guard descriptor >= 0 else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(errno)) } + let handle = FileHandle(fileDescriptor: descriptor, closeOnDealloc: true) + try handle.write(contentsOf: Data(repeating: 2, count: 16384)) + try handle.close() + let result = DirectoryMeasurement.measure(path: root.path) + XCTAssertTrue(result.isComplete, result.firstError ?? "") + XCTAssertEqual(result.bytes, try foundationBytes(root)) + } + + func testUnreadableChildDoesNotLookComplete() throws { + guard geteuid() != 0 else { throw XCTSkip("Permission denial requires an unprivileged test user") } + let root = try fixture() + let denied = root.appendingPathComponent("denied") + try FileManager.default.createDirectory(at: denied, withIntermediateDirectories: true) + try Data(repeating: 1, count: 4096).write(to: denied.appendingPathComponent("file")) + XCTAssertEqual(chmod(denied.path, 0), 0) + defer { chmod(denied.path, 0o700) } + let result = DirectoryMeasurement.measure(path: root.path) + XCTAssertFalse(result.isComplete) + XCTAssertGreaterThan(result.errorCount, 0) + } + + func testCancellationDuringTraversalIsIncomplete() throws { + let root = try fixture() + try Data([1]).write(to: root.appendingPathComponent("file")) + var checks = 0 + let result = DirectoryMeasurement.measure(path: root.path) { + checks += 1 + return checks >= 3 + } + XCTAssertFalse(result.isComplete) + XCTAssertEqual(result.errorCount, 1) + } + + func testParallelMeasurementsHaveIndependentState() throws { + let root = try fixture() + try Data(repeating: 7, count: 8192).write(to: root.appendingPathComponent("file")) + let expected = DirectoryMeasurement.measure(path: root.path) + let results = ScanResults() + DispatchQueue.concurrentPerform(iterations: 40) { _ in + results.append(DirectoryMeasurement.measure(path: root.path)) + } + XCTAssertEqual(results.snapshot().count, 40) + XCTAssertTrue(results.snapshot().allSatisfy { $0.isComplete && $0.bytes == expected.bytes && $0.files == expected.files }) + } + + func testMountedChildIsExcluded() throws { + guard ProcessInfo.processInfo.environment["CLEARDISK_TEST_MOUNTS"] == "1" else { + throw XCTSkip("Set CLEARDISK_TEST_MOUNTS=1 to run the disk-image integration test") + } + let root = try fixture() + let image = try fixture().appendingPathComponent("fixture.dmg") + let mount = root.appendingPathComponent("mounted") + try FileManager.default.createDirectory(at: mount, withIntermediateDirectories: true) + func hdiutil(_ arguments: [String]) throws { + let process = Process() + process.executableURL = URL(fileURLWithPath: "/usr/bin/hdiutil") + process.arguments = arguments + try process.run() + process.waitUntilExit() + XCTAssertEqual(process.terminationStatus, 0) + if process.terminationStatus != 0 { throw NSError(domain: "hdiutil", code: Int(process.terminationStatus)) } + } + try hdiutil(["create", "-size", "32m", "-fs", "HFS+", "-volname", "ScanTest", image.path]) + try hdiutil(["attach", "-nobrowse", "-mountpoint", mount.path, image.path]) + defer { try? hdiutil(["detach", mount.path]) } + try Data(repeating: 1, count: 1 << 20).write(to: mount.appendingPathComponent("excluded")) + let result = DirectoryMeasurement.measure(path: root.path) + XCTAssertTrue(result.isComplete, result.firstError ?? "") + XCTAssertEqual(result.files, 0) + XCTAssertEqual(result.bytes, 0) + } + + func testPolicyThrottlesAllNonNominalStatesAndLowPower() { + for cores in [1, 2, 4, 8, 24] { + let lowPower = ScanResourcePolicy(cores: cores, lowPower: true, thermalState: .nominal) + XCTAssertEqual(lowPower.workers, 1) + XCTAssertEqual(lowPower.backendLimits.traversal, 1) + XCTAssertEqual(lowPower.backendLimits.classification, 1) + XCTAssertEqual(lowPower.backendLimits.atomic, 1) + for state in [ProcessInfo.ThermalState.fair, .serious, .critical] { + XCTAssertEqual(ScanResourcePolicy(cores: cores, lowPower: false, thermalState: state).workers, 1) + } + } + XCTAssertEqual(ScanResourcePolicy(cores: 2, lowPower: false, thermalState: .nominal).workers, 1) + XCTAssertEqual(ScanResourcePolicy(cores: 8, lowPower: false, thermalState: .nominal).workers, 2) + } +} diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 0000000..2f6b986 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,44 @@ +# Scan performance validation + +No end-to-end speedup, battery-life improvement, E-core affinity, idle-core reservation, +or absence of fan noise is claimed. Utility QoS expresses scheduling intent only. +The earlier PR numbers had no reproducible logs and mixed incompatible baselines. + +## Traversal microbenchmark (macOS) + +Run `BENCH_RUNS=7 bash scripts/benchmark_scan.sh /absolute/root /another/disjoint/root > samples.jsonl`. +Use a static, fully readable fixture. The harness alternates Foundation, serial FTS, and +bounded parallel FTS order and rejects unequal file counts, allocated bytes, or errors. +One root cannot benefit from root-level parallelism. This is not a full-volume backend benchmark. +Report medians and dispersion from the JSONL samples, not the best isolated run. +These runs share filesystem caches; do not label them cold-cache measurements. + +For a base/head app comparison, build both commits in Release with the same Swift toolchain. +Record commit SHAs, Mac model/SoC, macOS/toolchain, storage and filesystem, dataset file count, +power source, Low Power Mode, initial thermal state, and Full Disk Access. Alternate build order; +compare the same categories or the same full-volume root and count all warnings. Retain raw logs. +Use separate first-run and repeated-run series with an explicitly documented cache-reset method. + +Measure elapsed time, user/system CPU time, peak RSS, UI responsiveness, and cancellation latency. +Use Instruments/System Trace and Energy Log (or available power instrumentation on that Mac) +to measure total energy over the scan, not just peak CPU percentage. Repeat on Intel and Apple +Silicon, battery and AC, with Low Power Mode on/off. Test a non-NVMe/external or network volume +before making storage-independent claims. Do not infer a speedup from worker counts. + +## Resource limits and failure semantics + +- Legacy stages share an OperationQueue: 2 workers normally; 1 on <=2 processors, Low Power Mode, + or any non-nominal thermal state. Notifications update admission limits; in-flight calls drain. +- Backend requests use traversal=2, classification=1 per directory, atomic=2 normally; all 1 in + the conservative profile. These are per-stage limits, not a process-wide thread cap. Leaf + preparation and the separate legacy scanner may also be active. +- A power/thermal change requiring fewer backend workers cancels the current scan. The UI + explains that a retry uses fewer workers. Blocking filesystem calls are not forcibly interrupted. +- FTS is physical, does not change cwd, and stays on the root device. Missing optional roots are + empty; unreadable or failed traversals are incomplete and excluded from cleanup estimates. +- `st_blocks * 512 / st_nlink` is attributed allocation, not exact reclaimable space. Hard links, + APFS clones, snapshots and open files prevent interpreting it as guaranteed free space. + +Run `CLEARDISK_TEST_MOUNTS=1 swift test` on macOS for the mounted-image regression test. +Run the measurement tests with Thread Sanitizer separately. Compressed files, +iCloud placeholders and live filesystem mutation also require macOS validation before merge. diff --git a/benchmarks/ScanBenchmark.swift b/benchmarks/ScanBenchmark.swift new file mode 100644 index 0000000..ec68a7e --- /dev/null +++ b/benchmarks/ScanBenchmark.swift @@ -0,0 +1,78 @@ +import Foundation + +@main +enum ScanBenchmark { + static func foundation(_ path: String) -> DirectoryMeasurement { + var result = DirectoryMeasurement() + let keys: Set = [.totalFileAllocatedSizeKey, .fileAllocatedSizeKey, .isRegularFileKey, .linkCountKey] + guard let iterator = FileManager.default.enumerator( + at: URL(fileURLWithPath: path), includingPropertiesForKeys: Array(keys), + errorHandler: { url, error in + result.errorCount += 1 + result.firstError = result.firstError ?? "\(url.path): \(error.localizedDescription)" + return true + } + ) else { result.errorCount += 1; return result } + for case let url as URL in iterator { + do { + let values = try url.resourceValues(forKeys: keys) + if values.isRegularFile == true { + result.files += 1 + result.bytes += Int64((values.totalFileAllocatedSize ?? values.fileAllocatedSize ?? 0) / max(1, values.linkCount ?? 1)) + } + } catch { result.errorCount += 1 } + } + return result + } + + static func emit(_ record: [String: Any]) throws { + let data = try JSONSerialization.data(withJSONObject: record, options: [.sortedKeys]) + print(String(decoding: data, as: UTF8.self)) + } + + static func main() throws { + let paths = Array(CommandLine.arguments.dropFirst()) + guard !paths.isEmpty else { + fputs("Usage: scan-benchmark /absolute/directory [other-disjoint-directory ...]\n", stderr) + exit(2) + } + let info = ProcessInfo.processInfo + let iterations = max(1, Int(info.environment["BENCH_RUNS"] ?? "7") ?? 7) + try emit(["type": "environment", "os": info.operatingSystemVersionString, + "cores": info.activeProcessorCount, "memory": info.physicalMemory, + "lowPower": info.isLowPowerModeEnabled, "thermal": info.thermalState.rawValue, + "roots": paths, "note": "Alternating order; runs share filesystem caches. No cold-cache or energy claim."]) + var reference: (Int, Int64)? + for run in 0..() + let workers = mode == "fts-parallel" ? ScanResourcePolicy.current.workers : 1 + let queue = OperationQueue() + queue.qualityOfService = .utility + queue.maxConcurrentOperationCount = workers + for path in paths { + queue.addOperation { + results.append(mode == "foundation" ? foundation(path) : DirectoryMeasurement.measure(path: path)) + } + } + queue.waitUntilAllOperationsAreFinished() + let seconds = Double(DispatchTime.now().uptimeNanoseconds - start) / 1e9 + let values = results.snapshot() + let count = values.reduce(0) { $0 + $1.files } + let bytes = values.reduce(Int64(0)) { $0 + $1.bytes } + let errors = values.reduce(0) { $0 + $1.errorCount } + if reference == nil { reference = (count, bytes) } + let matches = reference!.0 == count && reference!.1 == bytes && errors == 0 + try emit(["type": "sample", "run": run, "mode": mode, "seconds": seconds, + "files": count, "bytes": bytes, "errors": errors, "workers": workers, + "matchesReference": matches, "filesPerSecond": Double(count) / seconds]) + if !matches { + fputs("Counts/bytes differ or scan failed; performance comparison is invalid.\n", stderr) + exit(1) + } + } + } + } +} diff --git a/scripts/benchmark_scan.sh b/scripts/benchmark_scan.sh new file mode 100644 index 0000000..2884df8 --- /dev/null +++ b/scripts/benchmark_scan.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BENCH_DIR="$(mktemp -d /tmp/cleardisk-bench.XXXXXX)" +trap 'rm -rf "$BENCH_DIR"' EXIT +swiftc -O -o "$BENCH_DIR/scan-benchmark" \ + "$ROOT/Sources/ClearDisk/DirectoryMeasurement.swift" \ + "$ROOT/Sources/ClearDisk/ScanResourcePolicy.swift" \ + "$ROOT/benchmarks/ScanBenchmark.swift" +"$BENCH_DIR/scan-benchmark" "$@" diff --git a/scripts/build_app.sh b/scripts/build_app.sh index 98979a6..4ea7cb3 100755 --- a/scripts/build_app.sh +++ b/scripts/build_app.sh @@ -172,7 +172,7 @@ cat > "$APP_BUNDLE/Contents/Info.plist" << EOF EOF # Ad-hoc code sign the entire bundle (better Gatekeeper handling than linker-signed) -codesign --force --deep -s - "$APP_BUNDLE" +bash "$SCRIPTS_DIR/stage_sign_app.sh" "$APP_BUNDLE" echo "Code signed (ad-hoc)." echo "Done! App bundle created at: $APP_BUNDLE" diff --git a/scripts/stage_sign_app.sh b/scripts/stage_sign_app.sh new file mode 100644 index 0000000..fbdca75 --- /dev/null +++ b/scripts/stage_sign_app.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Sign away from synced folders; preserve the input bundle until replacement is verified. +set -euo pipefail + +APP_BUNDLE="${1:?Usage: stage_sign_app.sh /absolute/path/App.app}" +[[ "$APP_BUNDLE" = /* && -d "$APP_BUNDLE" ]] || exit 1 +APP_PARENT="$(dirname "$APP_BUNDLE")" +APP_LEAF="$(basename "$APP_BUNDLE")" +STAGE_DIR="" +REPLACE_DIR="" +BACKUP_DIR="" +COMMITTED=0 + +cleanup() { + local status=$? + trap - EXIT + if [ "$COMMITTED" -eq 0 ] && [ -n "$BACKUP_DIR" ] && [ -d "$BACKUP_DIR/$APP_LEAF" ]; then + rm -rf "$APP_BUNDLE" + if ! mv "$BACKUP_DIR/$APP_LEAF" "$APP_BUNDLE"; then + echo "error: restore failed; original bundle retained at $BACKUP_DIR/$APP_LEAF" >&2 + BACKUP_DIR="" + status=1 + fi + fi + [ -z "$STAGE_DIR" ] || rm -rf "$STAGE_DIR" + [ -z "$REPLACE_DIR" ] || rm -rf "$REPLACE_DIR" + [ -z "$BACKUP_DIR" ] || rm -rf "$BACKUP_DIR" + exit "$status" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +STAGE_DIR="$(mktemp -d /tmp/cleardisk-sign.XXXXXX)" +ditto "$APP_BUNDLE" "$STAGE_DIR/$APP_LEAF" +xattr -cr "$STAGE_DIR/$APP_LEAF" +codesign --force --deep -s - "$STAGE_DIR/$APP_LEAF" +codesign --verify --deep --strict "$STAGE_DIR/$APP_LEAF" + +# All fallible copying happens before the old bundle is moved. Sibling paths allow renames. +REPLACE_DIR="$(mktemp -d "$APP_PARENT/.cleardisk-replace.XXXXXX")" +ditto "$STAGE_DIR/$APP_LEAF" "$REPLACE_DIR/$APP_LEAF" +codesign --verify --deep --strict "$REPLACE_DIR/$APP_LEAF" +BACKUP_DIR="$(mktemp -d "$APP_PARENT/.cleardisk-backup.XXXXXX")" +mv "$APP_BUNDLE" "$BACKUP_DIR/$APP_LEAF" +mv "$REPLACE_DIR/$APP_LEAF" "$APP_BUNDLE" +codesign --verify --deep --strict "$APP_BUNDLE" +COMMITTED=1 diff --git a/scripts/test_stage_sign_app.py b/scripts/test_stage_sign_app.py new file mode 100644 index 0000000..1dc0804 --- /dev/null +++ b/scripts/test_stage_sign_app.py @@ -0,0 +1,73 @@ +"""Exercise bundle preservation with fake macOS tools; runnable on Linux and macOS.""" +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +class StagingTests(unittest.TestCase): + def check_staging(self, failure): + script = Path(__file__).with_name("stage_sign_app.sh").resolve() + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + app = root / "ClearDisk.app" + app.mkdir() + (app / "original").write_text("preserve me") + (app / "link").symlink_to("original") + bin_dir = root / "bin" + bin_dir.mkdir() + mock = '''#!/usr/bin/env python3 +import os, pathlib, shutil, sys +tool = pathlib.Path(sys.argv[0]).name +failure = os.environ.get("FAILURE", "") +target = sys.argv[-1] +if tool == "ditto": + if failure == "copy-back" and ".cleardisk-replace." in target: + pathlib.Path(target).mkdir() + sys.exit(1) + shutil.copytree(sys.argv[1], target, symlinks=True) +elif tool == "codesign": + signing = "--force" in sys.argv + if signing and failure == "sign": sys.exit(1) + if not signing and failure == "verify-candidate" and ".cleardisk-replace." in target: sys.exit(1) + if not signing and failure == "verify-final" and target == os.environ["APP"]: sys.exit(1) + if signing: pathlib.Path(target, "signed").write_text("yes") +elif tool == "xattr" and failure == "xattr": + sys.exit(1) +''' + for tool in ["ditto", "codesign", "xattr"]: + executable = bin_dir / tool + executable.write_text(mock) + executable.chmod(0o755) + environment = dict(os.environ, PATH=f"{bin_dir}:{os.environ['PATH']}", FAILURE=failure, APP=str(app)) + result = subprocess.run(["bash", str(script), str(app)], env=environment, + capture_output=True, text=True) + self.assertEqual(result.returncode == 0, not failure, result.stderr) + self.assertEqual((app / "original").read_text(), "preserve me") + self.assertTrue((app / "link").is_symlink()) + self.assertEqual((app / "signed").exists(), not failure) + self.assertEqual(list(root.glob(".cleardisk-*")), []) + + def test_success(self): + self.check_staging("") + + def test_signing_failure_preserves_bundle(self): + self.check_staging("sign") + + def test_xattr_failure_preserves_bundle(self): + self.check_staging("xattr") + + def test_partial_copy_back_preserves_bundle(self): + self.check_staging("copy-back") + + def test_candidate_verification_failure_preserves_bundle(self): + self.check_staging("verify-candidate") + + def test_final_verification_failure_rolls_back(self): + self.check_staging("verify-final") + + +if __name__ == "__main__": + unittest.main()