From 41cc5335900f4c3833c921e338c112d0f0ed93fb Mon Sep 17 00:00:00 2001 From: Vishwa Karthik Date: Sun, 23 Feb 2025 21:31:01 +0530 Subject: [PATCH 1/4] Auto Clear History via WidgetBindingObserver & Riverpod for Dynamic Clean Up --- history_service_implementation_readme.md | 44 ++++++++++++++ lib/app.dart | 56 ++++++++++++++++- lib/consts.dart | 2 + lib/main.dart | 2 +- lib/services/history_service.dart | 76 +++++++++++++++--------- lib/utils/history_utils.dart | 29 ++++++++- 6 files changed, 177 insertions(+), 32 deletions(-) create mode 100644 history_service_implementation_readme.md diff --git a/history_service_implementation_readme.md b/history_service_implementation_readme.md new file mode 100644 index 000000000..de10dad26 --- /dev/null +++ b/history_service_implementation_readme.md @@ -0,0 +1,44 @@ +# History Service Implementation for Auto-Clearing API Requests + +## Team Dart Knight + ++ User - `Vishwa Karthik` ++ Mail - `vishwa.prarthana@gmail.com` + + +## Overview + +This document outlines the implementation of the autoClearHistory feature in APIDASH to efficiently manage stored API request history while maintaining a smooth user experience across all supported platforms. + +## Features +### Auto-Clear History on App Launch + ++ When the app is launched, it automatically clears 50 old API records to prevent excessive local storage usage. + ++ This helps in keeping the app responsive and prevents unnecessary memory consumption. + ++ Triggered using autoClearHistory() from HistoryServiceImpl. + +### Dynamic Auto-Clear Trigger (State Management-Based) + ++ Uses Riverpod StateNotifier to monitor the API request list dynamically. + ++ If the length exceeds 50, the history is automatically trimmed, keeping only the latest 50 requests. + ++ This ensures automatic cleanup without relying on platform-dependent lifecycle events. + +### Platform-Specific Cleanup Handling + ++ For Android & iOS: Uses AppLifecycleState.paused via WidgetsBindingObserver to trigger autoClearHistory() when the app goes to the background. + ++ For Windows, macOS, Linux:Uses window_manager to detect app minimize/close events and trigger cleanup accordingly. + ++ Ensures proper handling since AppLifecycleState does not work on desktops. + +### Batch Deletion Strategy + ++ Deleting 50 records at a time minimizes performance issues. + ++ Ensures smooth UX by avoiding excessive database transactions. + ++ Users can continue adding new requests seamlessly without experiencing lag. \ No newline at end of file diff --git a/lib/app.dart b/lib/app.dart index fcc83b50e..b8331b90f 100644 --- a/lib/app.dart +++ b/lib/app.dart @@ -1,5 +1,6 @@ // ignore_for_file: use_build_context_synchronously +import 'package:apidash/models/history_meta_model.dart'; import 'package:apidash_design_system/apidash_design_system.dart'; import 'package:flutter/material.dart'; import 'package:flutter_portal/flutter_portal.dart'; @@ -101,11 +102,62 @@ class _AppState extends ConsumerState with WindowListener { } } -class DashApp extends ConsumerWidget { +class DashApp extends ConsumerStatefulWidget { const DashApp({super.key}); @override - Widget build(BuildContext context, WidgetRef ref) { + ConsumerState createState() => _DashAppState(); +} + +class _DashAppState extends ConsumerState + with WidgetsBindingObserver, WindowListener { + @override + void initState() { + super.initState(); + WidgetsBinding.instance.addObserver(this); + if (kIsDesktop) { + windowManager.addListener(this); + } + } + + @override + void dispose() { + WidgetsBinding.instance.removeObserver(this); + if (kIsDesktop) { + windowManager.removeListener(this); + } + super.dispose(); + } + + // Mobile LifeCyclse (Android, IOS) + @override + void didChangeAppLifecycleState(AppLifecycleState state) { + switch (state) { + case AppLifecycleState.paused: + clearHistoryService(); + default: + break; + } + super.didChangeAppLifecycleState(state); + } + + // Desktop Lifecycle (Windows, macOS, Linux) + @override + void onWindowMinimize() { + clearHistoryService(); + } + + Future clearHistoryService() async { + final Map? historyMetas = + ref.watch(historyMetaStateNotifier); + if ((historyMetas?.length ?? 0) > kCleaHistoryBackgroundThreshold) { + var settingsModel = await getSettingsFromSharedPrefs(); + await HistoryServiceImpl().autoClearHistory(settingsModel: settingsModel); + } + } + + @override + Widget build(BuildContext context) { final isDarkMode = ref.watch(settingsProvider.select((value) => value.isDark)); final workspaceFolderPath = ref diff --git a/lib/consts.dart b/lib/consts.dart index 7eb1ae95f..cac34f47c 100644 --- a/lib/consts.dart +++ b/lib/consts.dart @@ -70,6 +70,7 @@ final kIconRemoveLight = Icon( const kCodePreviewLinesLimit = 500; enum HistoryRetentionPeriod { + fiveSeconds("5 Seconds", Icons.calendar_view_week_rounded), oneWeek("1 Week", Icons.calendar_view_week_rounded), oneMonth("1 Month", Icons.calendar_view_month_rounded), threeMonths("3 Months", Icons.calendar_month_rounded), @@ -482,3 +483,4 @@ const kMsgClearHistory = const kMsgClearHistorySuccess = 'History cleared successfully'; const kMsgClearHistoryError = 'Error clearing history'; const kMsgShareError = "Unable to share"; +const int kCleaHistoryBackgroundThreshold = 10; diff --git a/lib/main.dart b/lib/main.dart index 8b5fab32a..edee291ae 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -47,7 +47,7 @@ Future initApp( ); debugPrint("openBoxesStatus: $openBoxesStatus"); if (openBoxesStatus) { - await autoClearHistory(settingsModel: settingsModel); + await HistoryServiceImpl().autoClearHistory(settingsModel: settingsModel); } return openBoxesStatus; } catch (e) { diff --git a/lib/services/history_service.dart b/lib/services/history_service.dart index 487b91640..7f6309e66 100644 --- a/lib/services/history_service.dart +++ b/lib/services/history_service.dart @@ -1,42 +1,62 @@ +import 'dart:developer'; + +import 'dart:async'; import 'package:apidash/models/models.dart'; import 'package:apidash/utils/utils.dart'; + import 'hive_services.dart'; -Future autoClearHistory({SettingsModel? settingsModel}) async { - final historyRetentionPeriod = settingsModel?.historyRetentionPeriod; - DateTime? retentionDate = getRetentionDate(historyRetentionPeriod); +abstract class HistoryService { + Future autoClearHistory({required SettingsModel? settingsModel}); +} - if (retentionDate == null) { - return; - } else { - List? historyIds = hiveHandler.getHistoryIds(); - List toRemoveIds = []; +class HistoryServiceImpl implements HistoryService { + @override + Future autoClearHistory({required SettingsModel? settingsModel}) async { + try { + final historyRetentionPeriod = settingsModel?.historyRetentionPeriod; + DateTime? retentionDate = getRetentionDate(historyRetentionPeriod); + if (retentionDate == null) return; - if (historyIds == null || historyIds.isEmpty) { - return; - } + List? historyIds = hiveHandler.getHistoryIds(); + if (historyIds == null || historyIds.isEmpty) return; - for (var historyId in historyIds) { - var jsonModel = hiveHandler.getHistoryMeta(historyId); - if (jsonModel != null) { - var jsonMap = Map.from(jsonModel); - HistoryMetaModel historyMetaModelFromJson = - HistoryMetaModel.fromJson(jsonMap); - if (historyMetaModelFromJson.timeStamp.isBefore(retentionDate)) { - toRemoveIds.add(historyId); + List toRemoveIds = historyIds.where((historyId) { + var jsonModel = hiveHandler.getHistoryMeta(historyId); + if (jsonModel != null) { + var jsonMap = Map.from(jsonModel); + HistoryMetaModel historyMetaModelFromJson = + HistoryMetaModel.fromJson(jsonMap); + return historyMetaModelFromJson.timeStamp.isBefore(retentionDate); } + return false; + }).toList(); + + if (toRemoveIds.isEmpty) return; + + int batchSize = calculateOptimalBatchSize(toRemoveIds.length); + final List> batches = createBatches(toRemoveIds, batchSize); + + for (var batch in batches) { + await deleteRecordsInBatches(batch); } - } - if (toRemoveIds.isEmpty) { - return; + hiveHandler.setHistoryIds(historyIds..removeWhere(toRemoveIds.contains)); + } catch (e, st) { + log("Error clearing history records", + name: "autoClearHistory", error: e, stackTrace: st); } + } - for (var id in toRemoveIds) { - await hiveHandler.deleteHistoryRequest(id); - hiveHandler.deleteHistoryMeta(id); + static Future deleteRecordsInBatches(List batch) async { + try { + for (var id in batch) { + hiveHandler.deleteHistoryMeta(id); + hiveHandler.deleteHistoryRequest(id); + } + } catch (e, st) { + log("Error deleting records in batches", + name: "deleteRecordsInBatches", error: e, stackTrace: st); } - hiveHandler.setHistoryIds( - historyIds..removeWhere((id) => toRemoveIds.contains(id))); } -} +} \ No newline at end of file diff --git a/lib/utils/history_utils.dart b/lib/utils/history_utils.dart index d1c3dbbc3..d2851c1f4 100644 --- a/lib/utils/history_utils.dart +++ b/lib/utils/history_utils.dart @@ -1,9 +1,17 @@ import 'package:apidash/models/models.dart'; import 'package:apidash/consts.dart'; + import 'convert_utils.dart'; DateTime stripTime(DateTime dateTime) { - return DateTime(dateTime.year, dateTime.month, dateTime.day); + return DateTime( + dateTime.year, + dateTime.month, + dateTime.day, + dateTime.hour, + dateTime.minute, + dateTime.second, + ); } RequestModel getRequestModelFromHistoryModel(HistoryRequestModel model) { @@ -115,11 +123,30 @@ List getRequestGroup( return requestGroup; } +int calculateOptimalBatchSize(int totalRecords) { + if (totalRecords < 100) return 50; + if (totalRecords < 500) return 80; + if (totalRecords < 5000) return 100; + return 500; +} + +List> createBatches(List items, int batchSize) { + return List.generate( + (items.length / batchSize).ceil(), + (index) => items.sublist( + index * batchSize, + (index * batchSize + batchSize).clamp(0, items.length), + ), + ); +} + DateTime? getRetentionDate(HistoryRetentionPeriod? retentionPeriod) { DateTime now = DateTime.now(); DateTime today = stripTime(now); switch (retentionPeriod) { + case HistoryRetentionPeriod.fiveSeconds: + return today.subtract(const Duration(seconds: 5)); case HistoryRetentionPeriod.oneWeek: return today.subtract(const Duration(days: 7)); case HistoryRetentionPeriod.oneMonth: From df011f2d35802ea7bc8db3be2b62bad31339ea22 Mon Sep 17 00:00:00 2001 From: Vishwa Karthik Date: Sun, 23 Feb 2025 21:45:27 +0530 Subject: [PATCH 2/4] update readme --- history_service_implementation_readme.md | 42 +++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/history_service_implementation_readme.md b/history_service_implementation_readme.md index de10dad26..30b0c02b5 100644 --- a/history_service_implementation_readme.md +++ b/history_service_implementation_readme.md @@ -41,4 +41,44 @@ This document outlines the implementation of the autoClearHistory feature in API + Ensures smooth UX by avoiding excessive database transactions. -+ Users can continue adding new requests seamlessly without experiencing lag. \ No newline at end of file ++ Users can continue adding new requests seamlessly without experiencing lag. + +## [Issues with Clearing via Isolates](https://github.com/foss42/apidash/pull/604) + +## My Findings... + +Local Database Hive do not support concurrent connections or transactions due to Native OS locking the hive files on the user desired directory, (hive.lock) files. + ++ Linux Based Arch Distributions and Apple System follow strict file lock system and wouldn't allow opening or accessing boxes from multiple isolates. + +## Reasoning ++ `Hive.init` and `hive.initFlutter` are both method-channel and main UI thread operations which needs many data accessibility which are only available in main thread isolate. + +## Cheap Work Around Solution +1. Close Hive in the main thread +2. Start an isolate +3. Initialize Hive in that isolate +4. Offload tasks to new isolate & Close Hive +5. Re-Open Hive Box in the main thread + +## Problems ++ Although the database transactions are fast, there are high chances the database behavior becomes highly unpredictable. ++ The cleaning service job trigger logic had to changed, since calling it main function may become stupidity. + +## Technical Issues ++ With issues stated, frequent switches between threads will make too many open/close hive boxes to hinder performance of the app. ++ App may stop working abruptly ++ IOS Production app may not allow these operations to do so, and may kill the app. ++ The Hive documentation clearly states that it is not designed for multi-threaded use. ++ Simultaneous reads/writes across isolates may lead to inconsistencies. + +## What about... [PR 604](https://github.com/foss42/apidash/pull/604) ++ The reason why it could have worked for Android, is due to its lenient OS behavior although its Linux-based distribution. ++ Even though Android doesn't throw an error, it's still not safe to open Hive in multiple isolates. + +## Note ++ There is another database called 'Isar' which probably supports multi-threaded concurrent transactions which could have been possible to resolve this functionality. + +## Conclusion ++ The issue opened is very real but the way it has to be tackled is just to clear them in main isolate using optimization techniques like batch request, clearing history in frequent intervals and few more everything in Main Thread ONLY. + From b5019d3bed22d84d8c78a0d6c43a392b7529dddd Mon Sep 17 00:00:00 2001 From: Vishwa Karthik Date: Sun, 23 Feb 2025 21:57:20 +0530 Subject: [PATCH 3/4] update readme --- history_service_implementation_readme.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/history_service_implementation_readme.md b/history_service_implementation_readme.md index 30b0c02b5..6d71d168b 100644 --- a/history_service_implementation_readme.md +++ b/history_service_implementation_readme.md @@ -5,6 +5,9 @@ + User - `Vishwa Karthik` + Mail - `vishwa.prarthana@gmail.com` +## Do Checkout ++ [Issues with Isolate & Compute Methodology](https://github.com/foss42/apidash/pull/604#issuecomment-2676863276) + ## Overview @@ -23,15 +26,15 @@ This document outlines the implementation of the autoClearHistory feature in API + Uses Riverpod StateNotifier to monitor the API request list dynamically. -+ If the length exceeds 50, the history is automatically trimmed, keeping only the latest 50 requests. ++ If the length exceeds 50, the history clearance is automatically triggered, just to avoid heavy duty on app launch. -+ This ensures automatic cleanup without relying on platform-dependent lifecycle events. ++ Reason to use State management here is to resist unnecessary local database call because user may keep switching apps/window during development cycle. ### Platform-Specific Cleanup Handling + For Android & iOS: Uses AppLifecycleState.paused via WidgetsBindingObserver to trigger autoClearHistory() when the app goes to the background. -+ For Windows, macOS, Linux:Uses window_manager to detect app minimize/close events and trigger cleanup accordingly. ++ For Windows, macOS, Linux:Uses window_manager to detect app minimize events and trigger cleanup accordingly. + Ensures proper handling since AppLifecycleState does not work on desktops. From 95172872cf4b6c397727a799efec2184c708b13d Mon Sep 17 00:00:00 2001 From: Ashita Prasad Date: Mon, 24 Feb 2025 00:54:21 +0530 Subject: [PATCH 4/4] Delete history_service_implementation_readme.md --- history_service_implementation_readme.md | 87 ------------------------ 1 file changed, 87 deletions(-) delete mode 100644 history_service_implementation_readme.md diff --git a/history_service_implementation_readme.md b/history_service_implementation_readme.md deleted file mode 100644 index 6d71d168b..000000000 --- a/history_service_implementation_readme.md +++ /dev/null @@ -1,87 +0,0 @@ -# History Service Implementation for Auto-Clearing API Requests - -## Team Dart Knight - -+ User - `Vishwa Karthik` -+ Mail - `vishwa.prarthana@gmail.com` - -## Do Checkout -+ [Issues with Isolate & Compute Methodology](https://github.com/foss42/apidash/pull/604#issuecomment-2676863276) - - -## Overview - -This document outlines the implementation of the autoClearHistory feature in APIDASH to efficiently manage stored API request history while maintaining a smooth user experience across all supported platforms. - -## Features -### Auto-Clear History on App Launch - -+ When the app is launched, it automatically clears 50 old API records to prevent excessive local storage usage. - -+ This helps in keeping the app responsive and prevents unnecessary memory consumption. - -+ Triggered using autoClearHistory() from HistoryServiceImpl. - -### Dynamic Auto-Clear Trigger (State Management-Based) - -+ Uses Riverpod StateNotifier to monitor the API request list dynamically. - -+ If the length exceeds 50, the history clearance is automatically triggered, just to avoid heavy duty on app launch. - -+ Reason to use State management here is to resist unnecessary local database call because user may keep switching apps/window during development cycle. - -### Platform-Specific Cleanup Handling - -+ For Android & iOS: Uses AppLifecycleState.paused via WidgetsBindingObserver to trigger autoClearHistory() when the app goes to the background. - -+ For Windows, macOS, Linux:Uses window_manager to detect app minimize events and trigger cleanup accordingly. - -+ Ensures proper handling since AppLifecycleState does not work on desktops. - -### Batch Deletion Strategy - -+ Deleting 50 records at a time minimizes performance issues. - -+ Ensures smooth UX by avoiding excessive database transactions. - -+ Users can continue adding new requests seamlessly without experiencing lag. - -## [Issues with Clearing via Isolates](https://github.com/foss42/apidash/pull/604) - -## My Findings... - -Local Database Hive do not support concurrent connections or transactions due to Native OS locking the hive files on the user desired directory, (hive.lock) files. - -+ Linux Based Arch Distributions and Apple System follow strict file lock system and wouldn't allow opening or accessing boxes from multiple isolates. - -## Reasoning -+ `Hive.init` and `hive.initFlutter` are both method-channel and main UI thread operations which needs many data accessibility which are only available in main thread isolate. - -## Cheap Work Around Solution -1. Close Hive in the main thread -2. Start an isolate -3. Initialize Hive in that isolate -4. Offload tasks to new isolate & Close Hive -5. Re-Open Hive Box in the main thread - -## Problems -+ Although the database transactions are fast, there are high chances the database behavior becomes highly unpredictable. -+ The cleaning service job trigger logic had to changed, since calling it main function may become stupidity. - -## Technical Issues -+ With issues stated, frequent switches between threads will make too many open/close hive boxes to hinder performance of the app. -+ App may stop working abruptly -+ IOS Production app may not allow these operations to do so, and may kill the app. -+ The Hive documentation clearly states that it is not designed for multi-threaded use. -+ Simultaneous reads/writes across isolates may lead to inconsistencies. - -## What about... [PR 604](https://github.com/foss42/apidash/pull/604) -+ The reason why it could have worked for Android, is due to its lenient OS behavior although its Linux-based distribution. -+ Even though Android doesn't throw an error, it's still not safe to open Hive in multiple isolates. - -## Note -+ There is another database called 'Isar' which probably supports multi-threaded concurrent transactions which could have been possible to resolve this functionality. - -## Conclusion -+ The issue opened is very real but the way it has to be tackled is just to clear them in main isolate using optimization techniques like batch request, clearing history in frequent intervals and few more everything in Main Thread ONLY. -