DjangoQ2 Scan Queue QA + Updated defaults

Author: ajinabraham

mobsf/MobSF/settings.py

@@ -345,8 +345,8 @@
 ASYNC_ANALYSIS_TIMEOUT = int(os.getenv('MOBSF_ASYNC_ANALYSIS_TIMEOUT', '60'))
 Q_CLUSTER = {
     'name': 'scan_queue',
-    'workers': int(os.getenv('MOBSF_ASYNC_WORKERS', 3)),
-    'recycle': 5,
+    'workers': int(os.getenv('MOBSF_ASYNC_WORKERS', '2')),
+    'recycle': 100,
     'timeout': ASYNC_ANALYSIS_TIMEOUT * 60,
     'retry': (ASYNC_ANALYSIS_TIMEOUT * 60) + 100,
     'compress': True,

mobsf/StaticAnalyzer/views/android/apk.py

@@ -76,8 +76,8 @@
 )
 from mobsf.StaticAnalyzer.views.common.async_task import (
     async_analysis,
-    enqueued_task_init,
-    update_enqueued_task,
+    mark_task_completed,
+    mark_task_started,
 )
 from mobsf.MobSF.views.authorization import (
     Permissions,
@@ -148,7 +148,7 @@ def apk_analysis_task(checksum, app_dic, rescan, queue=False):
     try:
         if queue:
             settings.ASYNC_ANALYSIS = True
-            enqueued_task_init(checksum)
+            mark_task_started(checksum)
         append_scan_status(checksum, 'init')
         get_size_and_hashes(app_dic)
         msg = 'Extracting APK'
@@ -232,12 +232,12 @@ def apk_analysis_task(checksum, app_dic, rescan, queue=False):
             rescan,
         )
         if queue:
-            return update_enqueued_task(
+            return mark_task_completed(
                 checksum, app_dic['subject'], 'Success')
         return context, None
     except Exception as exp:
         if queue:
-            return update_enqueued_task(
+            return mark_task_completed(
                 checksum, 'Failed', repr(exp))
         return context, repr(exp)
     finally:
@@ -288,7 +288,7 @@ def src_analysis_task(checksum, app_dic, rescan, pro_type, queue=False):
     try:
         if queue:
             settings.ASYNC_ANALYSIS = True
-            enqueued_task_init(checksum)
+            mark_task_started(checksum)
         cert_dic = {
             'certificate_info': '',
             'certificate_status': '',
@@ -353,11 +353,11 @@ def src_analysis_task(checksum, app_dic, rescan, pro_type, queue=False):
             rescan,
         )
         if queue:
-            return update_enqueued_task(
+            return mark_task_completed(
                 checksum, app_dic['subject'], 'Success')
     except Exception as exp:
         if queue:
-            return update_enqueued_task(
+            return mark_task_completed(
                 checksum, 'Failed', repr(exp))
     return context
 

mobsf/StaticAnalyzer/views/android/icon_analysis.py

@@ -190,6 +190,7 @@ def get_icon_apk_res(app_dic):
             icon_src = icon_name
 
         if icon_name and icon_name.endswith('.xml'):
+            # Handle XML icon case
             apktool_res = False
             # Can be vector XML/XML pointing to vector files
             # Convert AXML to XML for vector
@@ -216,10 +217,11 @@ def get_icon_apk_res(app_dic):
             else:
                 # if we cannot find from xml
                 icon_src = guess_icon_path(str(res_path))
-        else:
+        elif icon_name:
             # We found png icon, the easy path
             icon_src = (app_dir / icon_name).as_posix()
 
+        # Log warning if correct icon is not found
         if icon_src.endswith('.xml'):
             logger.warning('Cannot find icon file from xml')
             icon_src = ''

mobsf/StaticAnalyzer/views/common/async_task.py

@@ -48,35 +48,52 @@ def detect_timeout(sender, task, **kwargs):
 
 def async_analysis(checksum, api, file_name, func, *args, **kwargs):
     """Async Analysis Task."""
-    # Check if the task is already completed
+    # Check if the scan is already completed and successful
+    scan_completed = False
     recent = RecentScansDB.objects.filter(MD5=checksum)
-    scan_completed = recent[0].APP_NAME or recent[0].PACKAGE_NAME
-    # Check if the task is updated within the last 60 minutes
-    active_recently = recent[0].TIMESTAMP >= timezone.now() - timedelta(
-        minutes=settings.ASYNC_ANALYSIS_TIMEOUT)
+    if recent.exists() and (recent[0].APP_NAME or recent[0].PACKAGE_NAME):
+        # Successful scan will have APP_NAME or PACKAGE_NAME
+        scan_completed = True
+    # Check if task is already completed within the last 60 minutes
+    # Can be success or failed
+    completed_at_recently = EnqueuedTask.objects.filter(
+        checksum=checksum,
+        completed_at__gte=timezone.now() - timedelta(
+            minutes=settings.ASYNC_ANALYSIS_TIMEOUT),
+    ).exists()
+
     # Check if the task is already enqueued within the last 60 minutes
     queued_recently = EnqueuedTask.objects.filter(
         checksum=checksum,
         created_at__gte=timezone.now() - timedelta(
             minutes=settings.ASYNC_ANALYSIS_TIMEOUT),
     ).exists()
+    # Check if the task is already started within the last 60 minutes
+    started_at_recently = EnqueuedTask.objects.filter(
+        checksum=checksum,
+        started_at__gte=timezone.now() - timedelta(
+            minutes=settings.ASYNC_ANALYSIS_TIMEOUT),
+    ).exists()
 
-    # Additional checks on recent queue
-    if queued_recently:
-        if scan_completed:
-            # scan already completed recently
-            msg = 'Analysis already completed in the last 60 minutes'
-            logger.warning(msg)
-            if api:
-                return {'task_id': None, 'message': msg}
-            return HttpResponseRedirect('/tasks?q=completed')
-        elif active_recently:
-            # scan not completed but active recently
-            msg = 'Analysis already enqueued in the last 60 minutes'
-            logger.warning(msg)
-            if api:
-                return {'task_id': None, 'message': msg}
-            return HttpResponseRedirect('/tasks?q=queued')
+    # Prevent duplicate scans in the last 60 minutes
+    if scan_completed and completed_at_recently:
+        # scan task already completed with success/failure recently
+        msg = ('Analysis already completed/failed in the '
+               f'last {settings.ASYNC_ANALYSIS_TIMEOUT} minutes. '
+               'Please try again later.')
+        logger.warning(msg)
+        if api:
+            return {'task_id': None, 'message': msg}
+        return HttpResponseRedirect('/tasks?q=completed')
+    elif queued_recently or started_at_recently:
+        # scan not completed but queued or started recently
+        msg = ('Analysis already enqueued in the '
+               f'last {settings.ASYNC_ANALYSIS_TIMEOUT} minutes. '
+               'Please wait for the current scan to complete.')
+        logger.warning(msg)
+        if api:
+            return {'task_id': None, 'message': msg}
+        return HttpResponseRedirect('/tasks?q=queued')
 
     # Clear old tasks
     queue_size = settings.QUEUE_MAX_SIZE
@@ -108,15 +125,15 @@ def async_analysis(checksum, api, file_name, func, *args, **kwargs):
     return HttpResponseRedirect('/tasks')
 
 
-def enqueued_task_init(checksum):
-    """Store Task start."""
+def mark_task_started(checksum):
+    """Register the enqued task and others that matches the checksum as started."""
     EnqueuedTask.objects.filter(checksum=checksum).update(
         started_at=timezone.now(),
     )
 
 
-def update_enqueued_task(checksum, app_name, status):
-    """Update the Enqueued Task and others that matches the checksum."""
+def mark_task_completed(checksum, app_name, status):
+    """Update the enqueued task and others that matches the checksum as completed."""
     EnqueuedTask.objects.filter(checksum=checksum).update(
         app_name=app_name[:254],
         completed_at=timezone.now(),

mobsf/StaticAnalyzer/views/ios/ipa.py

@@ -55,8 +55,8 @@
 )
 from mobsf.StaticAnalyzer.views.common.async_task import (
     async_analysis,
-    enqueued_task_init,
-    update_enqueued_task,
+    mark_task_completed,
+    mark_task_started,
 )
 from mobsf.MalwareAnalyzer.views.MalwareDomainCheck import (
     MalwareDomainCheck,
@@ -170,7 +170,7 @@ def ipa_analysis_task(checksum, app_dic, rescan, queue=False):
     try:
         if queue:
             settings.ASYNC_ANALYSIS = True
-            enqueued_task_init(checksum)
+            mark_task_started(checksum)
         append_scan_status(checksum, 'init')
         msg = 'iOS Binary (IPA) Analysis Started'
         logger.info(msg)
@@ -181,11 +181,12 @@ def ipa_analysis_task(checksum, app_dic, rescan, queue=False):
             msg = ('IPA is malformed! MobSF cannot find Payload directory')
             append_scan_status(checksum, 'IPA is malformed', msg)
             if queue:
-                return update_enqueued_task(
+                return mark_task_completed(
                     checksum, 'Failed', msg)
             return context, msg
-        common_analysis('ipa', app_dic, checksum)
 
+        # Common Analysis
+        common_analysis('ipa', app_dic, checksum)
         # IPA Binary Analysis
         bin_dict = binary_analysis(
             checksum,
@@ -224,12 +225,12 @@ def ipa_analysis_task(checksum, app_dic, rescan, queue=False):
             rescan)
         if queue:
             subject = get_scan_subject(app_dic, bin_dict)
-            return update_enqueued_task(
+            return mark_task_completed(
                 checksum, subject, 'Success')
         return context, None
     except Exception as exp:
         if queue:
-            return update_enqueued_task(
+            return mark_task_completed(
                 checksum, 'Failed', repr(exp))
         return context, repr(exp)
 
@@ -275,7 +276,7 @@ def ios_analysis_task(checksum, app_dic, rescan, queue=False):
     try:
         if queue:
             settings.ASYNC_ANALYSIS = True
-            enqueued_task_init(checksum)
+            mark_task_started(checksum)
         logger.info('iOS Source Code Analysis Started')
         get_size_and_hashes(app_dic)
 
@@ -316,11 +317,11 @@ def ios_analysis_task(checksum, app_dic, rescan, queue=False):
             rescan)
         if queue:
             subject = get_scan_subject(app_dic, bin_dict)
-            return update_enqueued_task(
+            return mark_task_completed(
                 checksum, subject, 'Success')
     except Exception as exp:
         if queue:
-            return update_enqueued_task(
+            return mark_task_completed(
                 checksum, 'Failed', repr(exp))
     return context
 

mobsf/templates/general/tasks.html

@@ -275,7 +275,7 @@ <h3 class="box-title"><i class="fa fa-rocket"></i> Scan Queue</h3>
     if (q && q === 'completed') {
         Swal.fire(
             'Scan Task Recently Completed!',
-            'The scan was recently completed.',
+            'The scan was recently processed.',
             'warning'
         )
     } else if (q && q === 'queued') {