Merge pull request #484 from nightscout/carb-data-migration

[Part 3 of 6] Carb entries migration for importing 0.2.x JSON entries to CoreData

Model/JSONImporter.swift

@@ -4,6 +4,7 @@ import Foundation
 /// Migration-specific errors that might happen during migration
 enum JSONImporterError: Error {
     case missingGlucoseValueInGlucoseEntry
+    case missingCarbsValueInCarbEntry
 }
 
 // MARK: - JSONImporter Class
@@ -55,6 +56,23 @@ class JSONImporter {
         return Set(allReadings.compactMap(\.date))
     }
 
+    /// Retrieves the set of timestamps for all carb entries currently stored in CoreData.
+    ///
+    /// - Parameters: the start and end dates to fetch carb entries, inclusive
+    /// - Returns: A set of dates corresponding to existing carb entries.
+    /// - Throws: An error if the fetch operation fails.
+    private func fetchCarbEntryDates(start: Date, end: Date) async throws -> Set<Date> {
+        let allCarbEntryDates = try await coreDataStack.fetchEntitiesAsync(
+            ofType: CarbEntryStored.self,
+            onContext: context,
+            predicate: .predicateForDateBetween(start: start, end: end),
+            key: "date",
+            ascending: false
+        ) as? [CarbEntryStored] ?? []
+
+        return Set(allCarbEntryDates.compactMap(\.date))
+    }
+
     /// Imports glucose history from a JSON file into CoreData.
     ///
     /// The function reads glucose data from the provided JSON file and stores new entries
@@ -91,6 +109,49 @@ class JSONImporter {
             try self.context.save()
         }
     }
+
+    /// Imports carb history from a JSON file into CoreData.
+    ///
+    /// The function reads carb entries data from the provided JSON file and stores new entries
+    /// in CoreData, skipping entries with dates that already exist in the database.
+    /// We ignore all FPU entries (aka carb equivalents) when performing an import.
+    ///
+    /// - Parameters:
+    ///   - url: The URL of the JSON file containing glucose history.
+    ///   - now: The current datetime
+    /// - Throws:
+    ///   - JSONImporterError.missingCarbsValueInCarbEntry if a carb entry is missing a `carbs: Decimal` value.
+    ///   - An error if the file cannot be read or decoded.
+    ///   - An error if the CoreData operation fails.
+    func importCarbHistory(url: URL, now: Date) async throws {
+        let twentyFourHoursAgo = now - 24.hours.timeInterval
+        let carbHistoryFull: [CarbsEntry] = try readJsonFile(url: url)
+        let existingDates = try await fetchCarbEntryDates(start: twentyFourHoursAgo, end: now)
+
+        // Only import carb entries from the last 24 hours that do not exist yet in Core Data
+        // Only import "true" carb entries; ignore all FPU entries (aka carb equivalents)
+        let carbHistory = carbHistoryFull
+            .filter {
+                let dateToCheck = $0.actualDate ?? $0.createdAt
+                return dateToCheck >= twentyFourHoursAgo && dateToCheck <= now && !existingDates.contains(dateToCheck) && $0
+                    .isFPU ?? false == false }
+
+        // Create a background context for batch processing
+        let backgroundContext = NSManagedObjectContext(concurrencyType: .privateQueueConcurrencyType)
+        backgroundContext.parent = context
+
+        try await backgroundContext.perform {
+            for carbEntry in carbHistory {
+                try carbEntry.store(in: backgroundContext)
+            }
+
+            try backgroundContext.save()
+        }
+
+        try await context.perform {
+            try self.context.save()
+        }
+    }
 }
 
 // MARK: - Extension for Specific Import Functions
@@ -114,6 +175,97 @@ extension BloodGlucose {
     }
 }
 
+/// Extension to support decoding `CarbsEntry` from JSON with multiple possible key formats for entry notes.
+///
+/// This is needed because some JSON sources (e.g., Trio v0.2.5) use the singular key `"note"`
+/// for the `note` field, while others (e.g., Nightscout or oref) use the plural `"notes"`.
+///
+/// To ensure compatibility across all sources without duplicating models or requiring upstream fixes,
+/// this custom implementation attempts to decode the `note` field first from `"note"`, then from `"notes"`.
+/// Encoding will always use the canonical `"notes"` key to preserve consistency in output,
+/// as this is what's established throughout the backend now.
+extension CarbsEntry: Codable {
+    init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: CodingKeys.self)
+
+        id = try container.decodeIfPresent(String.self, forKey: .id)
+        createdAt = try container.decode(Date.self, forKey: .createdAt)
+        actualDate = try container.decodeIfPresent(Date.self, forKey: .actualDate)
+        carbs = try container.decode(Decimal.self, forKey: .carbs)
+        fat = try container.decodeIfPresent(Decimal.self, forKey: .fat)
+        protein = try container.decodeIfPresent(Decimal.self, forKey: .protein)
+
+        // Handle both `note` and `notes`
+        if let noteValue = try? container.decodeIfPresent(String.self, forKey: .note) {
+            note = noteValue
+        } else if let notesValue = try? container.decodeIfPresent(String.self, forKey: .noteAlt) {
+            note = notesValue
+        } else {
+            note = nil
+        }
+
+        enteredBy = try container.decodeIfPresent(String.self, forKey: .enteredBy)
+        isFPU = try container.decodeIfPresent(Bool.self, forKey: .isFPU)
+        fpuID = try container.decodeIfPresent(String.self, forKey: .fpuID)
+    }
+
+    func encode(to encoder: Encoder) throws {
+        var container = encoder.container(keyedBy: CodingKeys.self)
+
+        try container.encodeIfPresent(id, forKey: .id)
+        try container.encode(createdAt, forKey: .createdAt)
+        try container.encodeIfPresent(actualDate, forKey: .actualDate)
+        try container.encode(carbs, forKey: .carbs)
+        try container.encodeIfPresent(fat, forKey: .fat)
+        try container.encodeIfPresent(protein, forKey: .protein)
+        try container.encodeIfPresent(note, forKey: .note)
+        try container.encodeIfPresent(enteredBy, forKey: .enteredBy)
+        try container.encodeIfPresent(isFPU, forKey: .isFPU)
+        try container.encodeIfPresent(fpuID, forKey: .fpuID)
+    }
+
+    private enum CodingKeys: String, CodingKey {
+        case id = "_id"
+        case createdAt = "created_at"
+        case actualDate
+        case carbs
+        case fat
+        case protein
+        case note = "notes" // standard key
+        case noteAlt = "note" // import key
+        case enteredBy
+        case isFPU
+        case fpuID
+    }
+
+    /// Helper function to convert `CarbsStored` to `CarbEntryStored` while importing JSON carb entries
+    func store(in context: NSManagedObjectContext) throws {
+        guard carbs >= 0 else {
+            throw JSONImporterError.missingCarbsValueInCarbEntry
+        }
+
+        // skip FPU entries for now
+
+        let carbEntry = CarbEntryStored(context: context)
+        carbEntry.id = id
+            .flatMap({ UUID(uuidString: $0) }) ?? UUID() /// The `CodingKey` of `id` is `_id`, so this fine to use here
+        carbEntry.date = actualDate ?? createdAt
+        carbEntry.carbs = Double(truncating: NSDecimalNumber(decimal: carbs))
+        carbEntry.fat = Double(truncating: NSDecimalNumber(decimal: fat ?? 0))
+        carbEntry.protein = Double(truncating: NSDecimalNumber(decimal: protein ?? 0))
+        carbEntry.note = note ?? ""
+        carbEntry.isFPU = false
+        carbEntry.isUploadedToNS = true
+        carbEntry.isUploadedToHealth = true
+        carbEntry.isUploadedToTidepool = true
+
+        if fat != nil, protein != nil, let fpuId = fpuID {
+            carbEntry.fpuID = UUID(uuidString: fpuId)
+        }
+    }
+}
+
 extension JSONImporter {
     func importGlucoseHistoryIfNeeded() async {}
+    func importCarbHistoryIfNeeded() async {}
 }

Trio.xcodeproj/project.pbxproj

@@ -621,6 +621,7 @@
 		DDD1631C2C4C697400CD525A /* AddOverrideForm.swift in Sources */ = {isa = PBXBuildFile; fileRef = DDD1631B2C4C697400CD525A /* AddOverrideForm.swift */; };
 		DDD1631F2C4C6F6900CD525A /* TrioCoreDataPersistentContainer.xcdatamodeld in Sources */ = {isa = PBXBuildFile; fileRef = DDD1631D2C4C6F6900CD525A /* TrioCoreDataPersistentContainer.xcdatamodeld */; };
 		DDD6D4D32CDE90720029439A /* EstimatedA1cDisplayUnit.swift in Sources */ = {isa = PBXBuildFile; fileRef = DDD6D4D22CDE90720029439A /* EstimatedA1cDisplayUnit.swift */; };
+		DDD78A912DC4064800AC63F3 /* carbhistory.json in Resources */ = {isa = PBXBuildFile; fileRef = DDD78A902DC4064800AC63F3 /* carbhistory.json */; };
 		DDE179522C910127003CDDB7 /* MealPresetStored+CoreDataClass.swift in Sources */ = {isa = PBXBuildFile; fileRef = DDE179322C910127003CDDB7 /* MealPresetStored+CoreDataClass.swift */; };
 		DDE179532C910127003CDDB7 /* MealPresetStored+CoreDataProperties.swift in Sources */ = {isa = PBXBuildFile; fileRef = DDE179332C910127003CDDB7 /* MealPresetStored+CoreDataProperties.swift */; };
 		DDE179542C910127003CDDB7 /* LoopStatRecord+CoreDataClass.swift in Sources */ = {isa = PBXBuildFile; fileRef = DDE179342C910127003CDDB7 /* LoopStatRecord+CoreDataClass.swift */; };
@@ -1428,6 +1429,7 @@
 		DDD1631B2C4C697400CD525A /* AddOverrideForm.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AddOverrideForm.swift; sourceTree = "<group>"; };
 		DDD1631E2C4C6F6900CD525A /* TrioCoreDataPersistentContainer.xcdatamodel */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcdatamodel; path = TrioCoreDataPersistentContainer.xcdatamodel; sourceTree = "<group>"; };
 		DDD6D4D22CDE90720029439A /* EstimatedA1cDisplayUnit.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EstimatedA1cDisplayUnit.swift; sourceTree = "<group>"; };
+		DDD78A902DC4064800AC63F3 /* carbhistory.json */ = {isa = PBXFileReference; lastKnownFileType = text.json; path = carbhistory.json; sourceTree = "<group>"; };
 		DDE179322C910127003CDDB7 /* MealPresetStored+CoreDataClass.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MealPresetStored+CoreDataClass.swift"; sourceTree = "<group>"; };
 		DDE179332C910127003CDDB7 /* MealPresetStored+CoreDataProperties.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "MealPresetStored+CoreDataProperties.swift"; sourceTree = "<group>"; };
 		DDE179342C910127003CDDB7 /* LoopStatRecord+CoreDataClass.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "LoopStatRecord+CoreDataClass.swift"; sourceTree = "<group>"; };
@@ -2554,6 +2556,7 @@
 		3B997DD22DC02AEF006B6BB2 /* JSONImporterData */ = {
 			isa = PBXGroup;
 			children = (
+				DDD78A902DC4064800AC63F3 /* carbhistory.json */,
 				3B997DD12DC02AEF006B6BB2 /* glucose.json */,
 			);
 			path = JSONImporterData;
@@ -3903,6 +3906,7 @@
 			isa = PBXResourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				DDD78A912DC4064800AC63F3 /* carbhistory.json in Resources */,
 				3B997DD32DC02AEF006B6BB2 /* glucose.json in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;

TrioTests/JSONImporterData/carbhistory.json

@@ -0,0 +1,82 @@
+[
+  {
+    "carbs" : 10,
+    "_id" : "767AB5E3-D494-47B1-A057-084E26C4673A",
+    "fat" : 0,
+    "protein" : 0,
+    "created_at" : "2025-04-28T18:36:06.968Z",
+    "enteredBy" : "Trio",
+    "isFPU" : false,
+    "note" : "Snack 🍪"
+  },
+  {
+    "created_at" : "2025-04-28T18:10:32.027Z",
+    "_id" : "7DF8723D-F6BB-4C15-8761-1714E9269B87",
+    "note" : "",
+    "fat" : 0,
+    "isFPU" : false,
+    "protein" : 0,
+    "enteredBy" : "Trio",
+    "carbs" : 45
+  },
+  {
+    "created_at" : "2025-04-28T14:37:24.802Z",
+    "carbs" : 10,
+    "_id" : "3EB9212B-0BF6-4842-87B6-4204C28F64DD",
+    "isFPU" : false,
+    "fat" : 0,
+    "note" : "",
+    "enteredBy" : "Trio",
+    "protein" : 0
+  },
+  {
+    "protein" : 0,
+    "fat" : 0,
+    "created_at" : "2025-04-28T14:19:06.544Z",
+    "note" : "",
+    "isFPU" : false,
+    "_id" : "841383CF-BBE2-449F-B43A-7E3877B335C5",
+    "enteredBy" : "Trio",
+    "carbs" : 10
+  },
+  {
+    "_id" : "5D872E78-00D1-43B8-B38B-44DB07E3C7AD",
+    "carbs" : 20,
+    "protein" : 0,
+    "isFPU" : false,
+    "fat" : 0,
+    "note" : "",
+    "enteredBy" : "Trio",
+    "created_at" : "2025-04-28T14:04:45.343Z"
+  },
+  {
+    "note" : "",
+    "fat" : 0,
+    "created_at" : "2025-04-28T12:59:03.344Z",
+    "_id" : "637DA05B-5761-46C6-9862-84953CA99F70",
+    "isFPU" : false,
+    "protein" : 0,
+    "enteredBy" : "Trio",
+    "carbs" : 40
+  },
+  {
+    "protein" : 0,
+    "fat" : 0,
+    "note" : "",
+    "isFPU" : false,
+    "enteredBy" : "Trio",
+    "carbs" : 35,
+    "created_at" : "2025-04-28T08:38:33.763Z",
+    "_id" : "4EF0E6CA-BC02-4C02-AE28-4F719B327AAB"
+  },
+  {
+    "fat" : 0,
+    "protein" : 0,
+    "carbs" : 25,
+    "_id" : "100DAC30-7C9D-42F8-8D69-451A8660A27A",
+    "created_at" : "2025-04-28T05:03:43.332Z",
+    "note" : "",
+    "enteredBy" : "Trio",
+    "isFPU" : false
+  }
+]

TrioTests/JSONImporterTests.swift

@@ -72,4 +72,50 @@ class BundleReference {}
 
         #expect(allReadings.isEmpty)
     }
+
+    @Test("Import carb history with value checks") func testImportCarbHistoryDetails() async throws {
+        let testBundle = Bundle(for: BundleReference.self)
+        let path = testBundle.path(forResource: "carbhistory", ofType: "json")!
+        let url = URL(filePath: path)
+
+        let now = Date("2025-04-28T19:32:52.000Z")!
+        try await importer.importCarbHistory(url: url, now: now)
+        // run the import againt to check our deduplication logic
+        try await importer.importCarbHistory(url: url, now: now)
+
+        let allCarbEntries = try await coreDataStack.fetchEntitiesAsync(
+            ofType: CarbEntryStored.self,
+            onContext: context,
+            predicate: NSPredicate(format: "TRUEPREDICATE"),
+            key: "date",
+            ascending: false
+        ) as? [CarbEntryStored] ?? []
+
+        #expect(allCarbEntries.count == 8)
+        #expect(allCarbEntries.first?.carbs == 10)
+        #expect(allCarbEntries.first?.note == "Snack 🍪")
+        #expect(allCarbEntries.first?.date == Date("2025-04-28T18:36:06.968Z"))
+        #expect(allCarbEntries.last?.carbs == 25)
+        #expect(allCarbEntries.last?.date == Date("2025-04-28T05:03:43.332Z"))
+    }
+
+    @Test("Skip importing old carb entries") func testSkipImportOldCarbEntries() async throws {
+        let testBundle = Bundle(for: BundleReference.self)
+        let path = testBundle.path(forResource: "carbhistory", ofType: "json")!
+        let url = URL(filePath: path)
+
+        // more than 24 hours in the future from the most recent entry
+        let now = Date("2025-04-29T19:32:52.000Z")!
+        try await importer.importCarbHistory(url: url, now: now)
+
+        let allCarbEntries = try await coreDataStack.fetchEntitiesAsync(
+            ofType: CarbEntryStored.self,
+            onContext: context,
+            predicate: NSPredicate(format: "TRUEPREDICATE"),
+            key: "date",
+            ascending: false
+        ) as? [CarbEntryStored] ?? []
+
+        #expect(allCarbEntries.isEmpty)
+    }
 }