Merge pull request #479 from nightscout/glucose-data-migration

[Part 1 of 6] Glucose JSON to CoreData migration

Model/Helper/NSPredicates.swift

@@ -120,4 +120,8 @@ extension NSPredicate {
         let date = Date.threeMonthsAgo
         return NSPredicate(format: "date >= %@", date as NSDate)
     }
+
+    static func predicateForDateBetween(start: Date, end: Date) -> NSPredicate {
+        NSPredicate(format: "date >= %@ AND date <= %@", start as NSDate, end as NSDate)
+    }
 }

Model/JSONImporter.swift

@@ -0,0 +1,119 @@
+import CoreData
+import Foundation
+
+/// Migration-specific errors that might happen during migration
+enum JSONImporterError: Error {
+    case missingGlucoseValueInGlucoseEntry
+}
+
+// MARK: - JSONImporter Class
+
+/// Responsible for importing JSON data into Core Data.
+///
+/// The importer handles two important states:
+/// - JSON files stored in the file system that contain data to import
+/// - Existing entries in CoreData that should not be duplicated
+///
+/// Imports are performed when a JSON file exists. The importer checks
+/// CoreData for existing entries to avoid duplicating records from partial imports.
+class JSONImporter {
+    private let context: NSManagedObjectContext
+    private let coreDataStack: CoreDataStack
+
+    /// Initializes the importer with a Core Data context.
+    init(context: NSManagedObjectContext, coreDataStack: CoreDataStack) {
+        self.context = context
+        self.coreDataStack = coreDataStack
+    }
+
+    /// Reads and parses a JSON file from the file system.
+    ///
+    /// - Parameters:
+    ///   - url: The URL of the JSON file to read.
+    /// - Returns: A decoded object of the specified type.
+    /// - Throws: An error if the file cannot be read or decoded.
+    private func readJsonFile<T: Decodable>(url: URL) throws -> T {
+        let data = try Data(contentsOf: url)
+        let decoder = JSONCoding.decoder
+        return try decoder.decode(T.self, from: data)
+    }
+
+    /// Retrieves the set of dates for all glucose values currently stored in CoreData.
+    ///
+    /// - Parameters: the start and end dates to fetch glucose values, inclusive
+    /// - Returns: A set of dates corresponding to existing glucose readings.
+    /// - Throws: An error if the fetch operation fails.
+    private func fetchGlucoseDates(start: Date, end: Date) async throws -> Set<Date> {
+        let allReadings = try await coreDataStack.fetchEntitiesAsync(
+            ofType: GlucoseStored.self,
+            onContext: context,
+            predicate: .predicateForDateBetween(start: start, end: end),
+            key: "date",
+            ascending: false
+        ) as? [GlucoseStored] ?? []
+
+        return Set(allReadings.compactMap(\.date))
+    }
+
+    /// Imports glucose history from a JSON file into CoreData.
+    ///
+    /// The function reads glucose data from the provided JSON file and stores new entries
+    /// in CoreData, skipping entries with dates that already exist in the database.
+    ///
+    /// - Parameters:
+    ///   - url: The URL of the JSON file containing glucose history.
+    /// - Throws:
+    ///   - JSONImporterError.missingGlucoseValueInGlucoseEntry if a glucose entry is missing a value.
+    ///   - An error if the file cannot be read or decoded.
+    ///   - An error if the CoreData operation fails.
+    func importGlucoseHistory(url: URL, now: Date) async throws {
+        let twentyFourHoursAgo = now - 24.hours.timeInterval
+        let glucoseHistoryFull: [BloodGlucose] = try readJsonFile(url: url)
+        let existingDates = try await fetchGlucoseDates(start: twentyFourHoursAgo, end: now)
+
+        // only import glucose values from the last 24 hours that don't exist
+        let glucoseHistory = glucoseHistoryFull
+            .filter { $0.dateString >= twentyFourHoursAgo && $0.dateString <= now && !existingDates.contains($0.dateString) }
+
+        // Create a background context for batch processing
+        let backgroundContext = NSManagedObjectContext(concurrencyType: .privateQueueConcurrencyType)
+        backgroundContext.parent = context
+
+        try await backgroundContext.perform {
+            for glucoseEntry in glucoseHistory {
+                try glucoseEntry.store(in: backgroundContext)
+            }
+
+            try backgroundContext.save()
+        }
+
+        try await context.perform {
+            try self.context.save()
+        }
+    }
+}
+
+// MARK: - Extension for Specific Import Functions
+
+extension BloodGlucose {
+    /// Helper function to convert `BloodGlucose` to `GlucoseStored` while importing JSON glucose entries
+    func store(in context: NSManagedObjectContext) throws {
+        guard let glucoseValue = glucose ?? sgv else {
+            throw JSONImporterError.missingGlucoseValueInGlucoseEntry
+        }
+
+        let glucoseEntry = GlucoseStored(context: context)
+        glucoseEntry.id = _id.flatMap({ UUID(uuidString: $0) }) ?? UUID()
+        glucoseEntry.date = dateString
+        glucoseEntry.glucose = Int16(glucoseValue)
+        glucoseEntry.direction = direction?.rawValue
+        glucoseEntry.isManual = type == "Manual"
+        glucoseEntry.isUploadedToNS = true
+        glucoseEntry.isUploadedToHealth = true
+        glucoseEntry.isUploadedToTidepool = true
+    }
+}
+
+extension JSONImporter {
+    func importGlucoseHistoryIfNeeded() async {}
+}

Trio.xcodeproj/project.pbxproj

@@ -104,7 +104,6 @@
 		383420D625FFE38C002D46C1 /* LoopView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 383420D525FFE38C002D46C1 /* LoopView.swift */; };
 		383420D925FFEB3F002D46C1 /* Popup.swift in Sources */ = {isa = PBXBuildFile; fileRef = 383420D825FFEB3F002D46C1 /* Popup.swift */; };
 		383948D625CD4D8900E91849 /* FileStorage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 383948D525CD4D8900E91849 /* FileStorage.swift */; };
-		383948DA25CD64D500E91849 /* Glucose.swift in Sources */ = {isa = PBXBuildFile; fileRef = 383948D925CD64D500E91849 /* Glucose.swift */; };
 		384E803425C385E60086DB71 /* JavaScriptWorker.swift in Sources */ = {isa = PBXBuildFile; fileRef = 384E803325C385E60086DB71 /* JavaScriptWorker.swift */; };
 		384E803825C388640086DB71 /* Script.swift in Sources */ = {isa = PBXBuildFile; fileRef = 384E803725C388640086DB71 /* Script.swift */; };
 		38569347270B5DFB0002C50D /* CGMType.swift in Sources */ = {isa = PBXBuildFile; fileRef = 38569344270B5DFA0002C50D /* CGMType.swift */; };
@@ -245,6 +244,9 @@
 		3B4BA78F2D8DC0EC0069D5B8 /* TidepoolServiceKit.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 3B4BA7882D8DC0EC0069D5B8 /* TidepoolServiceKit.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
 		3B4BA7902D8DC0EC0069D5B8 /* TidepoolServiceKitUI.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 3B4BA7892D8DC0EC0069D5B8 /* TidepoolServiceKitUI.framework */; };
 		3B4BA7912D8DC0EC0069D5B8 /* TidepoolServiceKitUI.framework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 3B4BA7892D8DC0EC0069D5B8 /* TidepoolServiceKitUI.framework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; };
+		3B997DCB2DC00849006B6BB2 /* JSONImporter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3B997DCA2DC00849006B6BB2 /* JSONImporter.swift */; };
+		3B997DCF2DC00A3A006B6BB2 /* JSONImporterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3B997DCE2DC00A3A006B6BB2 /* JSONImporterTests.swift */; };
+		3B997DD32DC02AEF006B6BB2 /* glucose.json in Resources */ = {isa = PBXBuildFile; fileRef = 3B997DD12DC02AEF006B6BB2 /* glucose.json */; };
 		3BAD36B22D7CDC1A00CC298D /* MainLoadingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3BAD36B12D7CDC1400CC298D /* MainLoadingView.swift */; };
 		3BAD36CC2D7D420E00CC298D /* CoreDataInitializationCoordinator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3BAD36CB2D7D420500CC298D /* CoreDataInitializationCoordinator.swift */; };
 		3BD9687C2D8DDD4600899469 /* SlideButton in Frameworks */ = {isa = PBXBuildFile; productRef = 3BD9687B2D8DDD4600899469 /* SlideButton */; };
@@ -922,7 +924,6 @@
 		383420D525FFE38C002D46C1 /* LoopView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LoopView.swift; sourceTree = "<group>"; };
 		383420D825FFEB3F002D46C1 /* Popup.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Popup.swift; sourceTree = "<group>"; };
 		383948D525CD4D8900E91849 /* FileStorage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FileStorage.swift; sourceTree = "<group>"; };
-		383948D925CD64D500E91849 /* Glucose.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Glucose.swift; sourceTree = "<group>"; };
 		384E803325C385E60086DB71 /* JavaScriptWorker.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = JavaScriptWorker.swift; sourceTree = "<group>"; };
 		384E803725C388640086DB71 /* Script.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Script.swift; sourceTree = "<group>"; };
 		38569344270B5DFA0002C50D /* CGMType.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CGMType.swift; sourceTree = "<group>"; };
@@ -1044,6 +1045,9 @@
 		3B4BA7692D8DBD690069D5B8 /* RileyLinkKitUI.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; path = RileyLinkKitUI.framework; sourceTree = BUILT_PRODUCTS_DIR; };
 		3B4BA7882D8DC0EC0069D5B8 /* TidepoolServiceKit.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; path = TidepoolServiceKit.framework; sourceTree = BUILT_PRODUCTS_DIR; };
 		3B4BA7892D8DC0EC0069D5B8 /* TidepoolServiceKitUI.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; path = TidepoolServiceKitUI.framework; sourceTree = BUILT_PRODUCTS_DIR; };
+		3B997DCA2DC00849006B6BB2 /* JSONImporter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = JSONImporter.swift; sourceTree = "<group>"; };
+		3B997DCE2DC00A3A006B6BB2 /* JSONImporterTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = JSONImporterTests.swift; sourceTree = "<group>"; };
+		3B997DD12DC02AEF006B6BB2 /* glucose.json */ = {isa = PBXFileReference; lastKnownFileType = text.json; path = glucose.json; sourceTree = "<group>"; };
 		3BAD36B12D7CDC1400CC298D /* MainLoadingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MainLoadingView.swift; sourceTree = "<group>"; };
 		3BAD36CB2D7D420500CC298D /* CoreDataInitializationCoordinator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CoreDataInitializationCoordinator.swift; sourceTree = "<group>"; };
 		3BDEA2DC60EDE0A3CA54DC73 /* TargetsEditorProvider.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = TargetsEditorProvider.swift; sourceTree = "<group>"; };
@@ -2321,7 +2325,6 @@
 				3811DF0125CA9FEA00A708ED /* Credentials.swift */,
 				E592A36F2CEEC01E009A472C /* ContactTrickEntry.swift */,
 				38AEE73C25F0200C0013F05B /* TrioSettings.swift */,
-				383948D925CD64D500E91849 /* Glucose.swift */,
 				382C133625F13A1E00715CE1 /* InsulinSensitivities.swift */,
 				38887CCD25F5725200944304 /* IOBEntry.swift */,
 				DD68889C2C386E17006E3C44 /* NightscoutExercise.swift */,
@@ -2535,17 +2538,27 @@
 		38FCF3EE25E9028E0078B0D1 /* TrioTests */ = {
 			isa = PBXGroup;
 			children = (
+				3B997DD22DC02AEF006B6BB2 /* JSONImporterData */,
 				BD8FC05C2D6618BE00B95AED /* BolusCalculatorTests */,
 				BD8FC0552D66187700B95AED /* CoreDataTests */,
 				38FCF3F125E9028E0078B0D1 /* Info.plist */,
+				CEE9A65D2BBC9F6500EB5194 /* CalibrationsTests.swift */,
 				38FCF3F825E902C20078B0D1 /* FileStorageTests.swift */,
+				3B997DCE2DC00A3A006B6BB2 /* JSONImporterTests.swift */,
 				CE1F6DD82BADF4620064EB8D /* PluginManagerTests.swift */,
-				CEE9A65D2BBC9F6500EB5194 /* CalibrationsTests.swift */,
 				BD8FC0532D66186000B95AED /* TestError.swift */,
 			);
 			path = TrioTests;
 			sourceTree = "<group>";
 		};
+		3B997DD22DC02AEF006B6BB2 /* JSONImporterData */ = {
+			isa = PBXGroup;
+			children = (
+				3B997DD12DC02AEF006B6BB2 /* glucose.json */,
+			);
+			path = JSONImporterData;
+			sourceTree = "<group>";
+		};
 		4E8C7B59F8065047ECE20965 /* View */ = {
 			isa = PBXGroup;
 			children = (
@@ -2643,6 +2656,7 @@
 				3BAD36CB2D7D420500CC298D /* CoreDataInitializationCoordinator.swift */,
 				BDF34F8F2C10CF8C00D51995 /* CoreDataStack.swift */,
 				BD4064D02C4ED26900582F43 /* CoreDataObserver.swift */,
+				3B997DCA2DC00849006B6BB2 /* JSONImporter.swift */,
 				DDD1631D2C4C6F6900CD525A /* TrioCoreDataPersistentContainer.xcdatamodeld */,
 				DDE179112C9100FA003CDDB7 /* Classes+Properties */,
 				5825D1622BD405AE00F36E9B /* Helper */,
@@ -3889,6 +3903,7 @@
 			isa = PBXResourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				3B997DD32DC02AEF006B6BB2 /* glucose.json in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -4060,7 +4075,6 @@
 				DDBD53FC2DAA903100F940A6 /* OverviewStepView.swift in Sources */,
 				38A0364225ED069400FCBB52 /* TempBasal.swift in Sources */,
 				3811DE1725C9D40400A708ED /* Screen.swift in Sources */,
-				383948DA25CD64D500E91849 /* Glucose.swift in Sources */,
 				CE94598029E9E3BD0047C9C6 /* WatchConfigDataFlow.swift in Sources */,
 				388E596C25AD95110019842D /* OpenAPS.swift in Sources */,
 				E00EEC0527368630002FF094 /* StorageAssembly.swift in Sources */,
@@ -4414,6 +4428,7 @@
 				DD17453A2C55BFA600211FAC /* AlgorithmAdvancedSettingsDataFlow.swift in Sources */,
 				5075C1608E6249A51495C422 /* TargetsEditorProvider.swift in Sources */,
 				E13B7DAB2A435F57066AF02E /* TargetsEditorStateModel.swift in Sources */,
+				3B997DCB2DC00849006B6BB2 /* JSONImporter.swift in Sources */,
 				BD249D992D42FCCD00412DEB /* BolusStatsSetup.swift in Sources */,
 				9702FF92A09C53942F20D7EA /* TargetsEditorRootView.swift in Sources */,
 				1967DFBE29D052C200759F30 /* Icons.swift in Sources */,
@@ -4577,6 +4592,7 @@
 				CEE9A65E2BBC9F6500EB5194 /* CalibrationsTests.swift in Sources */,
 				BD8FC0622D6619E600B95AED /* OverrideStorageTests.swift in Sources */,
 				BD8FC0592D66189700B95AED /* TestAssembly.swift in Sources */,
+				3B997DCF2DC00A3A006B6BB2 /* JSONImporterTests.swift in Sources */,
 				BD8FC0662D661A0000B95AED /* GlucoseStorageTests.swift in Sources */,
 				BD8FC05B2D6618AF00B95AED /* DeterminationStorageTests.swift in Sources */,
 				CE1F6DD92BADF4620064EB8D /* PluginManagerTests.swift in Sources */,

Trio/Sources/Models/BloodGlucose.swift

@@ -79,11 +79,13 @@ struct BloodGlucose: JSON, Identifiable, Hashable, Codable {
         let container = try decoder.container(keyedBy: CodingKeys.self)
         _id = try container.decode(String.self, forKey: ._id)
 
-        do {
-            sgv = try container.decode(Int.self, forKey: .sgv)
-        } catch {
-            // The nightscout API returns a double instead of an int
-            sgv = Int(try container.decode(Double.self, forKey: .sgv))
+        sgv = try? container.decodeIfPresent(Int.self, forKey: .sgv)
+        if sgv == nil {
+            // The nightscout API might return a double instead of an int, or the key might be missing
+            if let doubleValue = try? container.decodeIfPresent(Double.self, forKey: .sgv) {
+                sgv = Int(doubleValue)
+            }
+            // If both attempts fail, sgv remains nil
         }
 
         direction = try container.decodeIfPresent(Direction.self, forKey: .direction)

Trio/Sources/Models/Glucose.swift

@@ -1,32 +0,0 @@
-import Foundation
-
-struct Glucose: JSON {
-    let sgv: Int?
-    let glucose: Int?
-    let type: GlucoseType
-    let noise: Int?
-    let date: Date
-    let filtered: Double?
-    let direction: Direction?
-}
-
-enum GlucoseType: String, JSON {
-    case sgv
-    case cal
-    case manual = "Manual"
-}
-
-enum Direction: String, JSON {
-    case tripleUp = "TripleUp"
-    case doubleUp = "DoubleUp"
-    case singleUp = "SingleUp"
-    case fortyFiveUp = "FortyFiveUp"
-    case flat = "Flat"
-    case fortyFiveDown = "FortyFiveDown"
-    case singleDown = "SingleDown"
-    case doubleDown = "DoubleDown"
-    case tripleDown = "TripleDown"
-    case none = "NONE"
-    case notComputable = "NOT COMPUTABLE"
-    case rateOutOfRange = "RATE OUT OF RANGE"
-}

TrioTests/JSONImporterTests.swift

@@ -0,0 +1,75 @@
+//
+//  JSONImporterTests.swift
+//  Trio
+//
+//  Created by Cengiz Deniz on 21.04.25.
+//
+import CoreData
+import Foundation
+import Swinject
+import Testing
+
+@testable import Trio
+
+class BundleReference {}
+
+@Suite("JSON Importer Tests", .serialized) struct JSONImporterTests: Injectable {
+    var coreDataStack: CoreDataStack!
+    var context: NSManagedObjectContext!
+    var importer: JSONImporter!
+
+    init() async throws {
+        // In-memory Core Data for tests
+        coreDataStack = try await CoreDataStack.createForTests()
+        context = coreDataStack.newTaskContext()
+        importer = JSONImporter(context: context, coreDataStack: coreDataStack)
+    }
+
+    @Test("Import glucose history with value checks") func testImportGlucoseHistoryDetails() async throws {
+        let testBundle = Bundle(for: BundleReference.self)
+        let path = testBundle.path(forResource: "glucose", ofType: "json")!
+        let url = URL(filePath: path)
+
+        let now = Date("2025-04-28T19:32:52.000Z")!
+        try await importer.importGlucoseHistory(url: url, now: now)
+        // run the import againt to check our deduplication logic
+        try await importer.importGlucoseHistory(url: url, now: now)
+
+        let allReadings = try await coreDataStack.fetchEntitiesAsync(
+            ofType: GlucoseStored.self,
+            onContext: context,
+            predicate: NSPredicate(format: "TRUEPREDICATE"),
+            key: "date",
+            ascending: false
+        ) as? [GlucoseStored] ?? []
+
+        #expect(allReadings.count == 274)
+        #expect(allReadings.first?.glucose == 115)
+        #expect(allReadings.first?.date == Date("2025-04-28T19:32:51.727Z"))
+        #expect(allReadings.last?.glucose == 127)
+        #expect(allReadings.last?.date == Date("2025-04-27T19:37:50.327Z"))
+
+        let manualCount = allReadings.filter({ $0.isManual }).count
+        #expect(manualCount == 1)
+    }
+
+    @Test("Skip importing old glucose values") func testSkipImportOldGlucoseValues() async throws {
+        let testBundle = Bundle(for: BundleReference.self)
+        let path = testBundle.path(forResource: "glucose", ofType: "json")!
+        let url = URL(filePath: path)
+
+        // more than 24 hours in the future from the most recent entry
+        let now = Date("2025-04-29T19:32:52.000Z")!
+        try await importer.importGlucoseHistory(url: url, now: now)
+
+        let allReadings = try await coreDataStack.fetchEntitiesAsync(
+            ofType: GlucoseStored.self,
+            onContext: context,
+            predicate: NSPredicate(format: "TRUEPREDICATE"),
+            key: "date",
+            ascending: false
+        ) as? [GlucoseStored] ?? []
+
+        #expect(allReadings.isEmpty)
+    }
+}