Merge pull request #492 from nightscout/minor-fixes-data-migration

[Part 6 of 6] Minor fixes for JSON migration to CoreData

Model/JSONImporter.swift

@@ -56,74 +56,42 @@ class JSONImporter {
         return try decoder.decode(T.self, from: data)
     }
 
-    /// Retrieves the set of dates for all glucose values currently stored in CoreData.
+    /// Fetches a set of unique `Date` values for a specific `NSManagedObject` type from Core Data.
     ///
-    /// - Parameters: the start and end dates to fetch glucose values, inclusive
-    /// - Returns: A set of dates corresponding to existing glucose readings.
-    /// - Throws: An error if the fetch operation fails.
-    private func fetchGlucoseDates(start: Date, end: Date) async throws -> Set<Date> {
-        let allReadings = try await coreDataStack.fetchEntitiesAsync(
-            ofType: GlucoseStored.self,
-            onContext: context,
-            predicate: .predicateForDateBetween(start: start, end: end),
-            key: "date",
-            ascending: false
-        ) as? [GlucoseStored] ?? []
-
-        return Set(allReadings.compactMap(\.date))
-    }
-
-    /// Retrieves the set of timestamps for all pump events currently stored in CoreData.
+    /// This helper function is used to retrieve all existing date-like values (e.g., `date`, `timestamp`, `deliverAt`)
+    /// from a given entity type within a specified time range. It wraps the fetch and transformation
+    /// in a `context.perform` block to ensure thread safety when used on private background contexts.
     ///
-    /// - Parameters: the start and end dates to fetch pump events, inclusive
-    /// - Returns: A set of dates corresponding to existing pump events.
-    /// - Throws: An error if the fetch operation fails.
-    private func fetchPumpTimestamps(start: Date, end: Date) async throws -> Set<Date> {
-        let allPumpEvents = try await coreDataStack.fetchEntitiesAsync(
-            ofType: PumpEventStored.self,
-            onContext: context,
-            predicate: .predicateForTimestampBetween(start: start, end: end),
-            key: "timestamp",
-            ascending: false
-        ) as? [PumpEventStored] ?? []
-
-        return Set(allPumpEvents.compactMap(\.timestamp))
-    }
-
-    /// Retrieves the set of timestamps for all carb entries currently stored in CoreData.
+    /// - Parameters:
+    ///   - type: The `NSManagedObject` subclass to fetch (e.g., `GlucoseStored.self`, `PumpEventStored.self`)
+    ///   - predicate: A preconstructed predicate that filters the entity by date/timestamp range.
+    ///   - sortKey: The string name of the date-like field used to sort the fetch results. **This must match the key used in Core Data.**
+    ///   - dateKeyPath: A key path pointing to the `Date?` property on the entity used to extract the actual date value from each record.
     ///
-    /// - Parameters: the start and end dates to fetch carb entries, inclusive
-    /// - Returns: A set of dates corresponding to existing carb entries.
-    /// - Throws: An error if the fetch operation fails.
-    private func fetchCarbEntryDates(start: Date, end: Date) async throws -> Set<Date> {
-        let allCarbEntryDates = try await coreDataStack.fetchEntitiesAsync(
-            ofType: CarbEntryStored.self,
+    /// - Returns: A `Set<Date>` containing all non-nil date values from the fetched entities.
+    /// - Throws: `CoreDataError.fetchError` if casting the fetched objects fails, or if the fetch itself fails.
+
+    private func fetchDates<T: NSManagedObject>(
+        ofType type: T.Type,
+        predicate: NSPredicate,
+        sortKey: String,
+        dateKeyPath: KeyPath<T, Date?>
+    ) async throws -> Set<Date> {
+        let fetched = try await coreDataStack.fetchEntitiesAsync(
+            ofType: type,
             onContext: context,
-            predicate: .predicateForDateBetween(start: start, end: end),
-            key: "date",
+            predicate: predicate,
+            key: sortKey,
             ascending: false
-        ) as? [CarbEntryStored] ?? []
-
-        return Set(allCarbEntryDates.compactMap(\.date))
-    }
+        )
 
-    /// Retrieves the set of dates for all oref determinations currently stored in CoreData.
-    ///
-    /// - Parameters:
-    ///   - start: the start to fetch from; inclusive
-    ///   - end: the end date to fetch to; inclusive
-    /// - Returns: A set of dates corresponding to existing determinations.
-    /// - Throws: An error if the fetch operation fails.
-    private func fetchDeterminationDates(start: Date, end: Date) async throws -> Set<Date> {
-        let determinations = try await coreDataStack.fetchEntitiesAsync(
-            ofType: OrefDetermination.self,
-            onContext: context,
-            predicate: .predicateForDeliverAtBetween(start: start, end: end),
-            key: "deliverAt",
-            ascending: false
-        ) as? [OrefDetermination] ?? []
+        return try await context.perform {
+            guard let typed = fetched as? [T] else {
+                throw CoreDataError.fetchError(function: #function, file: #file)
+            }
 
-        return Set(determinations.compactMap(\.deliverAt))
+            return Set(typed.compactMap { $0[keyPath: dateKeyPath] })
+        }
     }
 
     /// Imports glucose history from a JSON file into CoreData.
@@ -141,7 +109,12 @@ class JSONImporter {
     func importGlucoseHistory(url: URL, now: Date) async throws {
         let twentyFourHoursAgo = now - 24.hours.timeInterval
         let glucoseHistoryFull: [BloodGlucose] = try readJsonFile(url: url)
-        let existingDates = try await fetchGlucoseDates(start: twentyFourHoursAgo, end: now)
+        let existingDates = try await fetchDates(
+            ofType: GlucoseStored.self,
+            predicate: .predicateForDateBetween(start: twentyFourHoursAgo, end: now),
+            sortKey: "date",
+            dateKeyPath: \.date
+        )
 
         // only import glucose values from the last 24 hours that don't exist
         let glucoseHistory = glucoseHistoryFull
@@ -232,7 +205,12 @@ class JSONImporter {
     func importPumpHistory(url: URL, now: Date) async throws {
         let twentyFourHoursAgo = now - 24.hours.timeInterval
         let pumpHistoryRaw: [PumpHistoryEvent] = try readJsonFile(url: url)
-        let existingTimestamps = try await fetchPumpTimestamps(start: twentyFourHoursAgo, end: now)
+        let existingTimestamps = try await fetchDates(
+            ofType: PumpEventStored.self,
+            predicate: .predicateForTimestampBetween(start: twentyFourHoursAgo, end: now),
+            sortKey: "timestamp",
+            dateKeyPath: \.timestamp
+        )
         let pumpHistoryFiltered = pumpHistoryRaw
             .filter { $0.timestamp >= twentyFourHoursAgo && $0.timestamp <= now && !existingTimestamps.contains($0.timestamp) }
 
@@ -272,7 +250,12 @@ class JSONImporter {
     func importCarbHistory(url: URL, now: Date) async throws {
         let twentyFourHoursAgo = now - 24.hours.timeInterval
         let carbHistoryFull: [CarbsEntry] = try readJsonFile(url: url)
-        let existingDates = try await fetchCarbEntryDates(start: twentyFourHoursAgo, end: now)
+        let existingDates = try await fetchDates(
+            ofType: CarbEntryStored.self,
+            predicate: .predicateForDateBetween(start: twentyFourHoursAgo, end: now),
+            sortKey: "date",
+            dateKeyPath: \.date
+        )
 
         // Only import carb entries from the last 24 hours that do not exist yet in Core Data
         // Only import "true" carb entries; ignore all FPU entries (aka carb equivalents)
@@ -314,7 +297,12 @@ class JSONImporter {
         let twentyFourHoursAgo = now - 24.hours.timeInterval
         let enactedDetermination: Determination = try readJsonFile(url: enactedUrl)
         let suggestedDetermination: Determination = try readJsonFile(url: suggestedUrl)
-        let existingDates = try await fetchDeterminationDates(start: twentyFourHoursAgo, end: now)
+        let existingDates = try await fetchDates(
+            ofType: OrefDetermination.self,
+            predicate: .predicateForDeliverAtBetween(start: twentyFourHoursAgo, end: now),
+            sortKey: "deliverAt",
+            dateKeyPath: \.deliverAt
+        )
 
         /// Helper function to check if entries are from within the last 24 hours that do not yet exist in Core Data
         func checkDeterminationDate(_ date: Date) -> Bool {
@@ -396,7 +384,7 @@ extension PumpHistoryEvent {
             let bolusEntry = BolusStored(context: context)
             bolusEntry.amount = NSDecimalNumber(decimal: amount)
             bolusEntry.isSMB = isSMB ?? false
-            bolusEntry.isExternal = isExternal ?? false
+            bolusEntry.isExternal = isExternal ?? isExternalInsulin ?? false
             pumpEntry.bolus = bolusEntry
         } else if type == .tempBasal {
             guard let rate = rate, let duration = duration else {
@@ -486,18 +474,14 @@ extension CarbsEntry: Codable {
         carbEntry.id = id
             .flatMap({ UUID(uuidString: $0) }) ?? UUID() /// The `CodingKey` of `id` is `_id`, so this fine to use here
         carbEntry.date = actualDate ?? createdAt
-        carbEntry.carbs = Double(truncating: NSDecimalNumber(decimal: carbs))
-        carbEntry.fat = Double(truncating: NSDecimalNumber(decimal: fat ?? 0))
-        carbEntry.protein = Double(truncating: NSDecimalNumber(decimal: protein ?? 0))
+        carbEntry.carbs = Double(truncating: NSDecimalNumber(decimal: carbs.rounded(toPlaces: 0)))
+        carbEntry.fat = Double(truncating: NSDecimalNumber(decimal: fat?.rounded(toPlaces: 0) ?? 0))
+        carbEntry.protein = Double(truncating: NSDecimalNumber(decimal: protein?.rounded(toPlaces: 0) ?? 0))
         carbEntry.note = note ?? ""
         carbEntry.isFPU = false
         carbEntry.isUploadedToNS = true
         carbEntry.isUploadedToHealth = true
         carbEntry.isUploadedToTidepool = true
-
-        if fat != nil, protein != nil, let fpuId = fpuID {
-            carbEntry.fpuID = UUID(uuidString: fpuId)
-        }
     }
 }
 

Trio.xcodeproj/project.pbxproj

@@ -249,6 +249,7 @@
 		3B997DD32DC02AEF006B6BB2 /* glucose.json in Resources */ = {isa = PBXBuildFile; fileRef = 3B997DD12DC02AEF006B6BB2 /* glucose.json */; };
 		3BAD36B22D7CDC1A00CC298D /* MainLoadingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3BAD36B12D7CDC1400CC298D /* MainLoadingView.swift */; };
 		3BAD36CC2D7D420E00CC298D /* CoreDataInitializationCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3BAD36CB2D7D420500CC298D /* CoreDataInitializationCoordinator.swift */; };
+		3BCA5F7C2DC7B16400A7EAC7 /* pumphistory-with-external.json in Resources */ = {isa = PBXBuildFile; fileRef = 3BCA5F7B2DC7B15400A7EAC7 /* pumphistory-with-external.json */; };
 		3BD6CE262DC24CFD00FA0472 /* pumphistory-24h-zoned.json in Resources */ = {isa = PBXBuildFile; fileRef = 3BD6CE252DC24CFD00FA0472 /* pumphistory-24h-zoned.json */; };
 		3BD9687C2D8DDD4600899469 /* SlideButton in Frameworks */ = {isa = PBXBuildFile; productRef = 3BD9687B2D8DDD4600899469 /* SlideButton */; };
 		3BD9687F2D8DDD8800899469 /* CryptoSwift in Frameworks */ = {isa = PBXBuildFile; productRef = 3BD9687E2D8DDD8800899469 /* CryptoSwift */; };
@@ -1056,6 +1057,7 @@
 		3B997DD12DC02AEF006B6BB2 /* glucose.json */ = {isa = PBXFileReference; lastKnownFileType = text.json; path = glucose.json; sourceTree = "<group>"; };
 		3BAD36B12D7CDC1400CC298D /* MainLoadingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MainLoadingView.swift; sourceTree = "<group>"; };
 		3BAD36CB2D7D420500CC298D /* CoreDataInitializationCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CoreDataInitializationCoordinator.swift; sourceTree = "<group>"; };
+		3BCA5F7B2DC7B15400A7EAC7 /* pumphistory-with-external.json */ = {isa = PBXFileReference; lastKnownFileType = text.json; path = "pumphistory-with-external.json"; sourceTree = "<group>"; };
 		3BD6CE252DC24CFD00FA0472 /* pumphistory-24h-zoned.json */ = {isa = PBXFileReference; lastKnownFileType = text.json; path = "pumphistory-24h-zoned.json"; sourceTree = "<group>"; };
 		3BDEA2DC60EDE0A3CA54DC73 /* TargetsEditorProvider.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = TargetsEditorProvider.swift; sourceTree = "<group>"; };
 		3BF768BD6264FF7D71D66767 /* NightscoutConfigProvider.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = NightscoutConfigProvider.swift; sourceTree = "<group>"; };
@@ -2567,6 +2569,7 @@
 		3B997DD22DC02AEF006B6BB2 /* JSONImporterData */ = {
 			isa = PBXGroup;
 			children = (
+				3BCA5F7B2DC7B15400A7EAC7 /* pumphistory-with-external.json */,
 				DD3C47B22DC5608A003DD20D /* newerSuggested.json */,
 				DDD78AD72DC421B500AC63F3 /* enacted.json */,
 				DDD78AD82DC421B500AC63F3 /* suggested.json */,
@@ -3926,6 +3929,7 @@
 				3B997DD32DC02AEF006B6BB2 /* glucose.json in Resources */,
 				DDD78AD92DC421B500AC63F3 /* enacted.json in Resources */,
 				DD3C47B32DC5608A003DD20D /* newerSuggested.json in Resources */,
+				3BCA5F7C2DC7B16400A7EAC7 /* pumphistory-with-external.json in Resources */,
 				DDD78ADA2DC421B500AC63F3 /* suggested.json in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;

Trio/Sources/Models/PumpHistoryEvent.swift

@@ -16,6 +16,7 @@ struct PumpHistoryEvent: JSON, Equatable, Identifiable {
     let note: String?
     let isSMB: Bool?
     let isExternal: Bool?
+    let isExternalInsulin: Bool?
 
     init(
         id: String,
@@ -31,7 +32,8 @@ struct PumpHistoryEvent: JSON, Equatable, Identifiable {
         proteinInput: Int? = nil,
         note: String? = nil,
         isSMB: Bool? = nil,
-        isExternal: Bool? = nil
+        isExternal: Bool? = nil,
+        isExternalInsulin: Bool? = nil
     ) {
         self.id = id
         self.type = type
@@ -47,6 +49,7 @@ struct PumpHistoryEvent: JSON, Equatable, Identifiable {
         self.note = note
         self.isSMB = isSMB
         self.isExternal = isExternal
+        self.isExternalInsulin = isExternalInsulin
     }
 }
 
@@ -101,6 +104,7 @@ extension PumpHistoryEvent {
         case note
         case isSMB
         case isExternal
+        case isExternalInsulin
     }
 }
 

TrioTests/JSONImporterData/pumphistory-with-external.json

@@ -0,0 +1,9 @@
+[
+    {
+      "amount" : 0.88,
+      "timestamp" : "2025-05-04T04:37:43.654Z",
+      "isExternalInsulin" : true,
+      "id" : "98134B68-6B46-4814-96C1-68A745173D03",
+      "_type" : "Bolus"
+    }
+]

TrioTests/JSONImporterTests.swift

@@ -155,6 +155,41 @@ class BundleReference {}
         #expect(allReadings.isEmpty)
     }
 
+    @Test("Import pump history with external insulin") func testImportPumpHistoryWithExternalInsulin() async throws {
+        let testBundle = Bundle(for: BundleReference.self)
+        let path = testBundle.path(forResource: "pumphistory-with-external", ofType: "json")!
+        let url = URL(filePath: path)
+
+        let now = Date("2025-05-04T04:37:44.654Z")!
+        try await importer.importPumpHistory(url: url, now: now)
+
+        let allReadings = try await coreDataStack.fetchEntitiesAsync(
+            ofType: PumpEventStored.self,
+            onContext: context,
+            predicate: NSPredicate(format: "TRUEPREDICATE"),
+            key: "timestamp",
+            ascending: false
+        ) as? [PumpEventStored] ?? []
+
+        let objectIds = allReadings.map(\.objectID)
+        let parsedHistory = OpenAPS.loadAndMapPumpEvents(objectIds, from: context)
+
+        #expect(parsedHistory.count == 1)
+
+        let bolus: BolusDTO? = {
+            switch parsedHistory.first! {
+            case let .bolus(bolus):
+                return bolus
+            default:
+                return nil
+            }
+        }()
+
+        #expect(bolus != nil)
+        #expect(bolus!.isExternal)
+        #expect(bolus!.amount.isApproximatelyEqual(to: 0.88, epsilon: 0.01))
+    }
+
     @Test("Import carb history with value checks") func testImportCarbHistoryDetails() async throws {
         let testBundle = Bundle(for: BundleReference.self)
         let path = testBundle.path(forResource: "carbhistory", ofType: "json")!