Skip to content

Commit 39e6df9

Browse files
committed
Auto-updating of remotely opened CSV files
1 parent 2275f12 commit 39e6df9

2 files changed

Lines changed: 114 additions & 5 deletions

File tree

‎renderer/components/dbComponents/dataTableFromDB.jsx‎

Lines changed: 112 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ import { Message } from "primereact/message"
1212
import { Skeleton } from "primereact/skeleton"
1313
import React, { useEffect, useState } from "react"
1414
import { toast } from "react-toastify"
15-
import { connectToMongoDB, getCollectionTags, insertMEDDataObjectIfNotExists } from "../mongoDB/mongoDBUtils"
15+
import { connectToMongoDB, getCollectionTags, getPathFromMEDDataObject, insertMEDDataObjectIfNotExists, insertObjectIntoCollection, overwriteMEDDataObjectProperties } from "../mongoDB/mongoDBUtils"
16+
import { getTunnelState } from "../../utilities/tunnelState"
17+
import { lstat } from "../../utilities/fileManagement/fileOps"
1618
import { MEDDataObject } from "../workspace/NewMedDataObject"
1719
import InputToolsComponent from "./InputToolsComponent"
1820
import { collectionExists, getCollectionData } from "./utils"
@@ -90,10 +92,38 @@ const DataTableFromDB = ({ data, tablePropsData, tablePropsColumn, isReadOnly })
9092
// Fetch data from MongoDB on component mount
9193
useEffect(() => {
9294
const getData = async () => {
93-
// Get total count of documents in the collection
9495
const db = await connectToMongoDB()
9596
const collection = db.collection(data.id)
96-
const count = await collection.countDocuments()
97+
98+
let count = await collection.countDocuments()
99+
const objectPath = await resolveObjectPath()
100+
const tunnel = getTunnelState()
101+
const isRemoteMode = !!(tunnel && tunnel.tunnelActive)
102+
const medDataObject = await db.collection("medDataObjects").findOne({ id: data.id })
103+
const currentMTimeMs = await getSourceMTimeMs(objectPath, isRemoteMode)
104+
const lastIngestedMTimeMs = medDataObject?.sourceMTimeMs
105+
const shouldReingestForChange =
106+
!!objectPath &&
107+
!!currentMTimeMs &&
108+
!!lastIngestedMTimeMs &&
109+
Number(lastIngestedMTimeMs) !== Number(currentMTimeMs)
110+
const shouldReingestForEmpty = !!objectPath && count === 0
111+
112+
if (shouldReingestForChange || shouldReingestForEmpty) {
113+
try {
114+
await reingestFromSourceFile({
115+
silent: true,
116+
resolvedPath: objectPath,
117+
sourceMTimeMs: currentMTimeMs
118+
})
119+
count = await collection.countDocuments()
120+
} catch (error) {
121+
console.error("Auto re-ingest failed:", error)
122+
}
123+
} else if (objectPath && currentMTimeMs && !lastIngestedMTimeMs && count > 0) {
124+
await overwriteMEDDataObjectProperties(data.id, { sourceMTimeMs: currentMTimeMs })
125+
}
126+
97127
setCollectionSize(count)
98128
console.log("Fetching data with:", data)
99129
let collectionName = data.extension === "view" ? data.name : data.id
@@ -380,6 +410,85 @@ const DataTableFromDB = ({ data, tablePropsData, tablePropsColumn, isReadOnly })
380410
setLoadingData(false)
381411
}
382412

413+
const resolveObjectPath = async () => {
414+
if (data?.path) {
415+
return data.path
416+
}
417+
if (!data?.id) {
418+
return null
419+
}
420+
try {
421+
const dbPath = await getPathFromMEDDataObject(data.id)
422+
return dbPath || null
423+
} catch (error) {
424+
console.error("Failed to resolve object path:", error)
425+
return null
426+
}
427+
}
428+
429+
const getSourceMTimeMs = async (objectPath, isRemoteMode) => {
430+
if (!objectPath) return null
431+
try {
432+
const fileInfo = await lstat(objectPath, { isRemote: isRemoteMode })
433+
if (!fileInfo || !fileInfo.stats) return null
434+
const stats = fileInfo.stats
435+
if (typeof stats.mtimeMs === "number") return stats.mtimeMs
436+
if (stats.mtime instanceof Date) return stats.mtime.getTime()
437+
if (typeof stats.mtime === "number") return stats.mtime * 1000
438+
return null
439+
} catch (error) {
440+
console.error("Failed to read source file mtime:", error)
441+
return null
442+
}
443+
}
444+
445+
446+
const reingestFromSourceFile = async ({ silent = false, resolvedPath = null, sourceMTimeMs = null } = {}) => {
447+
if (!data?.id) {
448+
if (!silent) toast.error("No collection ID available for re-ingest.")
449+
return
450+
}
451+
setLoadingData(true)
452+
try {
453+
const objectPath = resolvedPath || (await resolveObjectPath())
454+
if (!objectPath) {
455+
if (!silent) toast.error("Could not find source file path for this dataset.")
456+
return
457+
}
458+
459+
const inferredType = (data?.extension || data?.name?.split(".")?.pop() || "csv").toLowerCase()
460+
const medDataObject = { id: data.id, type: inferredType }
461+
const tunnel = getTunnelState()
462+
463+
if (tunnel && tunnel.tunnelActive && tunnel.localExpressPort) {
464+
const response = await window.backend.requestExpress({
465+
method: "post",
466+
path: "/insert-object-into-collection",
467+
host: tunnel.host,
468+
port: tunnel.localExpressPort,
469+
body: { objectPath, medDataObject }
470+
})
471+
if (!response?.data?.success) {
472+
throw new Error(response?.data?.error || "Remote re-ingest failed")
473+
}
474+
} else {
475+
await insertObjectIntoCollection({ objectPath, medDataObject })
476+
}
477+
478+
if (sourceMTimeMs) {
479+
await overwriteMEDDataObjectProperties(data.id, { sourceMTimeMs })
480+
}
481+
482+
await refreshData()
483+
if (!silent) toast.success("Re-ingest completed. Data refreshed.")
484+
} catch (error) {
485+
console.error("Re-ingest failed:", error)
486+
if (!silent) toast.error("Re-ingest failed. See console for details.")
487+
} finally {
488+
setLoadingData(false)
489+
}
490+
}
491+
383492
// Function to generate a random UUID
384493
const usePersistentUUID = () => {
385494
const [id, setId] = useState("")

‎renderer/components/mongoDB/mongoDBUtils.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,9 +160,9 @@ export async function insertMEDDataObjectIfNotExists(medData, path = null, jsonD
160160
} else if (path) {
161161
const tunnel = getTunnelState()
162162
console.log("tunnel before insert-object-into-collection:", tunnel)
163-
if (tunnel && tunnel.tunnelActive && tunnel.localDBPort) { // run remotely
163+
if (tunnel && tunnel.tunnelActive && tunnel.localExpressPort) { // run remotely
164164
try {
165-
const response = await window.backend.requestExpress({ method: 'post', path: '/insert-object-into-collection', host: tunnel.host, port: tunnel.localDBPort, body: { objectPath: path, medDataObject: medData } })
165+
const response = await window.backend.requestExpress({ method: 'post', path: '/insert-object-into-collection', host: tunnel.host, port: tunnel.localExpressPort, body: { objectPath: path, medDataObject: medData } })
166166
if (response?.data?.success) {
167167
console.log(`${medData.type} object successfully inserted remotely.`)
168168
} else {

0 commit comments

Comments
 (0)