@@ -12,7 +12,9 @@ import { Message } from "primereact/message"
1212import { Skeleton } from "primereact/skeleton"
1313import React , { useEffect , useState } from "react"
1414import { toast } from "react-toastify"
15- import { connectToMongoDB , getCollectionTags , insertMEDDataObjectIfNotExists } from "../mongoDB/mongoDBUtils"
15+ import { connectToMongoDB , getCollectionTags , getPathFromMEDDataObject , insertMEDDataObjectIfNotExists , insertObjectIntoCollection , overwriteMEDDataObjectProperties } from "../mongoDB/mongoDBUtils"
16+ import { getTunnelState } from "../../utilities/tunnelState"
17+ import { lstat } from "../../utilities/fileManagement/fileOps"
1618import { MEDDataObject } from "../workspace/NewMedDataObject"
1719import InputToolsComponent from "./InputToolsComponent"
1820import { collectionExists , getCollectionData } from "./utils"
@@ -90,10 +92,38 @@ const DataTableFromDB = ({ data, tablePropsData, tablePropsColumn, isReadOnly })
9092 // Fetch data from MongoDB on component mount
9193 useEffect ( ( ) => {
9294 const getData = async ( ) => {
93- // Get total count of documents in the collection
9495 const db = await connectToMongoDB ( )
9596 const collection = db . collection ( data . id )
96- const count = await collection . countDocuments ( )
97+
98+ let count = await collection . countDocuments ( )
99+ const objectPath = await resolveObjectPath ( )
100+ const tunnel = getTunnelState ( )
101+ const isRemoteMode = ! ! ( tunnel && tunnel . tunnelActive )
102+ const medDataObject = await db . collection ( "medDataObjects" ) . findOne ( { id : data . id } )
103+ const currentMTimeMs = await getSourceMTimeMs ( objectPath , isRemoteMode )
104+ const lastIngestedMTimeMs = medDataObject ?. sourceMTimeMs
105+ const shouldReingestForChange =
106+ ! ! objectPath &&
107+ ! ! currentMTimeMs &&
108+ ! ! lastIngestedMTimeMs &&
109+ Number ( lastIngestedMTimeMs ) !== Number ( currentMTimeMs )
110+ const shouldReingestForEmpty = ! ! objectPath && count === 0
111+
112+ if ( shouldReingestForChange || shouldReingestForEmpty ) {
113+ try {
114+ await reingestFromSourceFile ( {
115+ silent : true ,
116+ resolvedPath : objectPath ,
117+ sourceMTimeMs : currentMTimeMs
118+ } )
119+ count = await collection . countDocuments ( )
120+ } catch ( error ) {
121+ console . error ( "Auto re-ingest failed:" , error )
122+ }
123+ } else if ( objectPath && currentMTimeMs && ! lastIngestedMTimeMs && count > 0 ) {
124+ await overwriteMEDDataObjectProperties ( data . id , { sourceMTimeMs : currentMTimeMs } )
125+ }
126+
97127 setCollectionSize ( count )
98128 console . log ( "Fetching data with:" , data )
99129 let collectionName = data . extension === "view" ? data . name : data . id
@@ -380,6 +410,85 @@ const DataTableFromDB = ({ data, tablePropsData, tablePropsColumn, isReadOnly })
380410 setLoadingData ( false )
381411 }
382412
413+ const resolveObjectPath = async ( ) => {
414+ if ( data ?. path ) {
415+ return data . path
416+ }
417+ if ( ! data ?. id ) {
418+ return null
419+ }
420+ try {
421+ const dbPath = await getPathFromMEDDataObject ( data . id )
422+ return dbPath || null
423+ } catch ( error ) {
424+ console . error ( "Failed to resolve object path:" , error )
425+ return null
426+ }
427+ }
428+
429+ const getSourceMTimeMs = async ( objectPath , isRemoteMode ) => {
430+ if ( ! objectPath ) return null
431+ try {
432+ const fileInfo = await lstat ( objectPath , { isRemote : isRemoteMode } )
433+ if ( ! fileInfo || ! fileInfo . stats ) return null
434+ const stats = fileInfo . stats
435+ if ( typeof stats . mtimeMs === "number" ) return stats . mtimeMs
436+ if ( stats . mtime instanceof Date ) return stats . mtime . getTime ( )
437+ if ( typeof stats . mtime === "number" ) return stats . mtime * 1000
438+ return null
439+ } catch ( error ) {
440+ console . error ( "Failed to read source file mtime:" , error )
441+ return null
442+ }
443+ }
444+
445+
446+ const reingestFromSourceFile = async ( { silent = false , resolvedPath = null , sourceMTimeMs = null } = { } ) => {
447+ if ( ! data ?. id ) {
448+ if ( ! silent ) toast . error ( "No collection ID available for re-ingest." )
449+ return
450+ }
451+ setLoadingData ( true )
452+ try {
453+ const objectPath = resolvedPath || ( await resolveObjectPath ( ) )
454+ if ( ! objectPath ) {
455+ if ( ! silent ) toast . error ( "Could not find source file path for this dataset." )
456+ return
457+ }
458+
459+ const inferredType = ( data ?. extension || data ?. name ?. split ( "." ) ?. pop ( ) || "csv" ) . toLowerCase ( )
460+ const medDataObject = { id : data . id , type : inferredType }
461+ const tunnel = getTunnelState ( )
462+
463+ if ( tunnel && tunnel . tunnelActive && tunnel . localExpressPort ) {
464+ const response = await window . backend . requestExpress ( {
465+ method : "post" ,
466+ path : "/insert-object-into-collection" ,
467+ host : tunnel . host ,
468+ port : tunnel . localExpressPort ,
469+ body : { objectPath, medDataObject }
470+ } )
471+ if ( ! response ?. data ?. success ) {
472+ throw new Error ( response ?. data ?. error || "Remote re-ingest failed" )
473+ }
474+ } else {
475+ await insertObjectIntoCollection ( { objectPath, medDataObject } )
476+ }
477+
478+ if ( sourceMTimeMs ) {
479+ await overwriteMEDDataObjectProperties ( data . id , { sourceMTimeMs } )
480+ }
481+
482+ await refreshData ( )
483+ if ( ! silent ) toast . success ( "Re-ingest completed. Data refreshed." )
484+ } catch ( error ) {
485+ console . error ( "Re-ingest failed:" , error )
486+ if ( ! silent ) toast . error ( "Re-ingest failed. See console for details." )
487+ } finally {
488+ setLoadingData ( false )
489+ }
490+ }
491+
383492 // Function to generate a random UUID
384493 const usePersistentUUID = ( ) => {
385494 const [ id , setId ] = useState ( "" )
0 commit comments