Skip to content

Commit b4cf9b7

Browse files
authored
4.2.5 (#2457)
* Unified async scan timeout * Allow incomplete scan delete after async scan timeout duration * Added support for Android SBOM analysis * Make dependencies unpinned (Address #2458)
1 parent 4b394a2 commit b4cf9b7

16 files changed

+204
-43
lines changed

mobsf/MobSF/init.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818

1919
logger = logging.getLogger(__name__)
2020

21-
VERSION = '4.2.4'
21+
VERSION = '4.2.5'
2222
BANNER = r"""
2323
__ __ _ ____ _____ _ _ ____
2424
| \/ | ___ | |__/ ___|| ___|_ _| || | |___ \

mobsf/MobSF/settings.py

+4-3
Original file line numberDiff line numberDiff line change
@@ -341,12 +341,14 @@
341341
},
342342
},
343343
}
344+
ASYNC_ANALYSIS = bool(os.getenv('MOBSF_ASYNC_ANALYSIS', '0') == '1')
345+
ASYNC_ANALYSIS_TIMEOUT = int(os.getenv('MOBSF_ASYNC_ANALYSIS_TIMEOUT', '60'))
344346
Q_CLUSTER = {
345347
'name': 'scan_queue',
346348
'workers': int(os.getenv('MOBSF_ASYNC_WORKERS', 3)),
347349
'recycle': 5,
348-
'timeout': 3600,
349-
'retry': 3700,
350+
'timeout': ASYNC_ANALYSIS_TIMEOUT * 60,
351+
'retry': (ASYNC_ANALYSIS_TIMEOUT * 60) + 100,
350352
'compress': True,
351353
'label': 'scan_queue',
352354
'orm': 'default',
@@ -355,7 +357,6 @@
355357
'ack_failures': True,
356358
}
357359
QUEUE_MAX_SIZE = 100
358-
ASYNC_ANALYSIS = bool(os.getenv('MOBSF_ASYNC_ANALYSIS', '0') == '1')
359360
MULTIPROCESSING = os.getenv('MOBSF_MULTIPROCESSING')
360361
JADX_TIMEOUT = int(os.getenv('MOBSF_JADX_TIMEOUT', 1000))
361362
SAST_TIMEOUT = int(os.getenv('MOBSF_SAST_TIMEOUT', 1000))

mobsf/MobSF/utils.py

+2
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@
6161
USERNAME_REGEX = re.compile(r'^\w[\w\-\@\.]{1,35}$')
6262
GOOGLE_API_KEY_REGEX = re.compile(r'AIza[0-9A-Za-z-_]{35}$')
6363
GOOGLE_APP_ID_REGEX = re.compile(r'\d{1,2}:\d{1,50}:android:[a-f0-9]{1,50}')
64+
PKG_REGEX = re.compile(
65+
r'package\s+([a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z_][a-zA-Z0-9_]*)*);')
6466

6567

6668
class Color(object):

mobsf/MobSF/views/api/api_static_analysis.py

-1
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,6 @@ def api_scan_logs(request):
113113
return make_api_response({'logs': resp}, 200)
114114

115115

116-
117116
@request_method(['POST'])
118117
@csrf_exempt
119118
def api_tasks(request):

mobsf/MobSF/views/home.py

+11-5
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,11 @@
77
import re
88
import shutil
99
from pathlib import Path
10+
from datetime import timedelta
1011
from wsgiref.util import FileWrapper
1112

1213
from django.conf import settings
14+
from django.utils.timezone import now
1315
from django.core.paginator import Paginator
1416
from django.http import HttpResponse, HttpResponseRedirect
1517
from django.views.decorators.http import require_http_methods
@@ -541,10 +543,14 @@ def delete_scan(request, api=False):
541543
if settings.ASYNC_ANALYSIS:
542544
# Handle Async Tasks
543545
et = EnqueuedTask.objects.filter(checksum=md5_hash).first()
544-
if et and not et.completed_at:
545-
# Queue is in progress, cannot delete the task
546-
return send_response({
547-
'deleted': 'A scan can only be deleted after it is completed'}, api)
546+
if et:
547+
max_time_passed = now() - et.created_at > timedelta(
548+
minutes=settings.ASYNC_ANALYSIS_TIMEOUT)
549+
if not (et.completed_at or max_time_passed):
550+
# Queue is in progress, cannot delete the task
551+
return send_response(
552+
{'deleted': 'A scan can only be deleted after it is completed'},
553+
api)
548554
# Delete all related DB entries
549555
EnqueuedTask.objects.filter(checksum=md5_hash).all().delete()
550556
RecentScansDB.objects.filter(MD5=md5_hash).delete()
@@ -565,7 +571,7 @@ def delete_scan(request, api=False):
565571
os.remove(item_path)
566572
# Delete related directories
567573
if is_dir_exists(item_path) and valid_item:
568-
shutil.rmtree(item_path)
574+
shutil.rmtree(item_path, ignore_errors=True)
569575
return send_response({'deleted': 'yes'}, api)
570576
except Exception as exp:
571577
msg = str(exp)

mobsf/StaticAnalyzer/models.py

+1
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ class Meta:
8282
PLAYSTORE_DETAILS = models.TextField(default={})
8383
NETWORK_SECURITY = models.TextField(default=[])
8484
SECRETS = models.TextField(default=[])
85+
SBOM = models.TextField(default={})
8586

8687

8788
class StaticAnalyzerIOS(models.Model):

mobsf/StaticAnalyzer/views/android/code_analysis.py

+13-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@
2626
from mobsf.MalwareAnalyzer.views.android import (
2727
behaviour_analysis,
2828
)
29+
from mobsf.StaticAnalyzer.views.android import (
30+
sbom_analysis,
31+
)
2932

3033
logger = logging.getLogger(__name__)
3134

@@ -83,6 +86,7 @@ def code_analysis(checksum, app_dir, typ, manifest_file, android_permissions):
8386
email_n_file = []
8487
url_n_file = []
8588
url_list = []
89+
sbom = {}
8690
app_dir = Path(app_dir)
8791
src = get_android_src_dir(app_dir, typ).as_posix() + '/'
8892
skp = settings.SKIP_CLASS_PATH
@@ -95,10 +99,17 @@ def code_analysis(checksum, app_dir, typ, manifest_file, android_permissions):
9599
'match_extensions': {'.java', '.kt'},
96100
'ignore_paths': skp,
97101
}
98-
# Code Analysis
99102
sast = SastEngine(options, src)
100103
# Read data once and pass it to all the analysis
101104
file_data = sast.read_files()
105+
106+
# SBOM Analysis
107+
sbom = sbom_analysis.sbom(app_dir, file_data)
108+
msg = 'Android SBOM Analysis Completed'
109+
logger.info(msg)
110+
append_scan_status(checksum, msg)
111+
112+
# Code Analysis
102113
code_findings = sast.run_rules(file_data, code_rules.as_posix())
103114
msg = 'Android SAST Completed'
104115
logger.info(msg)
@@ -186,6 +197,7 @@ def code_analysis(checksum, app_dir, typ, manifest_file, android_permissions):
186197
'urls_list': url_list,
187198
'urls': url_n_file,
188199
'emails': email_n_file,
200+
'sbom': sbom,
189201
}
190202
return code_an_dic
191203
except Exception as exp:

mobsf/StaticAnalyzer/views/android/db_interaction.py

+3
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ def get_context_from_db_entry(db_entry: QuerySet) -> dict:
8888
'playstore_details': python_dict(db_entry[0].PLAYSTORE_DETAILS),
8989
'secrets': python_list(db_entry[0].SECRETS),
9090
'logs': get_scan_logs(db_entry[0].MD5),
91+
'sbom': python_dict(db_entry[0].SBOM),
9192
}
9293
return context
9394
except Exception:
@@ -161,6 +162,7 @@ def get_context_from_analysis(app_dic,
161162
'playstore_details': app_dic['playstore'],
162163
'secrets': code_an_dic['secrets'],
163164
'logs': get_scan_logs(app_dic['md5']),
165+
'sbom': code_an_dic['sbom'],
164166
}
165167
return context
166168
except Exception as exp:
@@ -226,6 +228,7 @@ def save_or_update(update_type,
226228
'PLAYSTORE_DETAILS': app_dic['playstore'],
227229
'NETWORK_SECURITY': man_an_dic['network_security'],
228230
'SECRETS': code_an_dic['secrets'],
231+
'SBOM': code_an_dic['sbom'],
229232
}
230233
if update_type == 'save':
231234
db_entry = StaticAnalyzerAndroid.objects.filter(
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# -*- coding: utf_8 -*-
2+
"""Extract packages from APK."""
3+
import logging
4+
5+
from mobsf.MobSF.utils import (
6+
PKG_REGEX,
7+
)
8+
9+
10+
logger = logging.getLogger(__name__)
11+
12+
13+
def merge_common_packages(items):
14+
"""Merge common packages."""
15+
items = list(items)
16+
items.sort() # Sort items lexicographically
17+
merged = []
18+
for item in items:
19+
if not merged or not item.startswith(merged[-1] + '.'):
20+
merged.append(item)
21+
return merged
22+
23+
24+
def extract_packages(file_data):
25+
"""Extract package names from file data."""
26+
packages = set()
27+
try:
28+
for item in file_data:
29+
# tuple has file path and file content
30+
# we are interested in file content's first line
31+
pkg = item[1].split('\n')[0]
32+
match = PKG_REGEX.search(pkg)
33+
if match and match.group(1) != '_COROUTINE':
34+
packages.add(match.group(1))
35+
packages = merge_common_packages(packages)
36+
except Exception:
37+
logger.exception('Extracting packages from file data')
38+
return sorted(packages)
39+
40+
41+
def get_group_name(file_name, group):
42+
"""Get group and name from file name."""
43+
parts = file_name.split('_')
44+
45+
if parts and len(parts) == 2:
46+
group, name = parts[0], parts[1]
47+
else:
48+
name = file_name.replace('_', '-')
49+
if name.startswith('kotlinx-'):
50+
group = 'org.jetbrains.kotlinx'
51+
52+
return group, name
53+
54+
55+
def android_sbom(app_dir):
56+
"""Extract SBOM from files."""
57+
sbom = set()
58+
for vfile in app_dir.rglob('*.version'):
59+
try:
60+
dependency = vfile.stem
61+
group, name = '', ''
62+
version = vfile.read_text().strip() or ''
63+
version = 'dynamic' if version.startswith('task') else version
64+
65+
if '_' in dependency:
66+
group, name = get_group_name(dependency, group)
67+
sbom.add(f'{group}:{name}@{version}')
68+
except Exception:
69+
pass
70+
return sorted(sbom)
71+
72+
73+
def sbom(app_dir, file_data):
74+
"""Extract SBOM from version files and decompiled source code."""
75+
return {
76+
'sbom_versioned': android_sbom(app_dir),
77+
'sbom_packages': extract_packages(file_data),
78+
}

mobsf/StaticAnalyzer/views/android/so.py

+1
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ def so_analysis(request, app_dic, rescan, api):
123123
'urls_list': [],
124124
'urls': [],
125125
'emails': [],
126+
'sbom': {},
126127
}
127128
# Get the strings and metadata from shared object
128129
get_strings_metadata(

mobsf/StaticAnalyzer/views/common/async_task.py

+4-2
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,13 @@ def async_analysis(checksum, api, file_name, func, *args, **kwargs):
5252
recent = RecentScansDB.objects.filter(MD5=checksum)
5353
scan_completed = recent[0].APP_NAME or recent[0].PACKAGE_NAME
5454
# Check if the task is updated within the last 60 minutes
55-
active_recently = recent[0].TIMESTAMP >= timezone.now() - timedelta(minutes=60)
55+
active_recently = recent[0].TIMESTAMP >= timezone.now() - timedelta(
56+
minutes=settings.ASYNC_ANALYSIS_TIMEOUT)
5657
# Check if the task is already enqueued within the last 60 minutes
5758
queued_recently = EnqueuedTask.objects.filter(
5859
checksum=checksum,
59-
created_at__gte=timezone.now() - timedelta(minutes=60),
60+
created_at__gte=timezone.now() - timedelta(
61+
minutes=settings.ASYNC_ANALYSIS_TIMEOUT),
6062
).exists()
6163

6264
# Additional checks on recent queue

mobsf/templates/static_analysis/android_binary_analysis.html

+28
Original file line numberDiff line numberDiff line change
@@ -324,6 +324,12 @@
324324
<i class="fab fa-buffer nav-icon"></i>
325325
<p>Libraries</p>
326326
</a>
327+
</li>
328+
<li class="nav-item">
329+
<a href="#sbom" class="nav-link">
330+
<i class="fa fa-archive nav-icon"></i>
331+
<p>SBOM</p>
332+
</a>
327333
</li>
328334
<li class="nav-item">
329335
<a href="#files" class="nav-link">
@@ -2337,6 +2343,28 @@ <h5 class="description-header">{{ code_analysis.summary.suppressed }}</h5>
23372343
</div>
23382344
</section>
23392345
<!-- ===========================end libraries ================================== -->
2346+
<a id="sbom" class="anchor"></a>
2347+
<section class="content">
2348+
<div class="container-fluid">
2349+
<div class="row">
2350+
<div class="col-lg-12">
2351+
<div class="card">
2352+
<div class="card-body">
2353+
<p>
2354+
<strong><i class="fa fa-archive"></i> SBOM</strong>
2355+
</p>
2356+
<div class="list-group">
2357+
{% include 'base/list.html' with list=sbom.sbom_versioned type="Versioned Packages" limit=100 %}
2358+
{% include 'base/list.html' with list=sbom.sbom_packages type="Packages" limit=100 %}
2359+
</div>
2360+
</div>
2361+
</div>
2362+
</div><!-- /.card -->
2363+
</div>
2364+
<!-- end row -->
2365+
</div>
2366+
</section>
2367+
<!-- ===========================end sbom ================================== -->
23402368
<a id="files" class="anchor"></a>
23412369
<section class="content">
23422370
<div class="container-fluid">

mobsf/templates/static_analysis/android_source_analysis.html

+28
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,12 @@
241241
<i class="fab fa-buffer nav-icon"></i>
242242
<p>Libraries</p>
243243
</a>
244+
</li>
245+
<li class="nav-item">
246+
<a href="#sbom" class="nav-link">
247+
<i class="fa fa-archive nav-icon"></i>
248+
<p>SBOM</p>
249+
</a>
244250
</li>
245251
<li class="nav-item">
246252
<a href="#files" class="nav-link">
@@ -1744,6 +1750,28 @@ <h5 class="description-header">{{ code_analysis.summary.suppressed }}</h5>
17441750
</div>
17451751
</section>
17461752
<!-- ===========================end libraries ================================== -->
1753+
<a id="sbom" class="anchor"></a>
1754+
<section class="content">
1755+
<div class="container-fluid">
1756+
<div class="row">
1757+
<div class="col-lg-12">
1758+
<div class="card">
1759+
<div class="card-body">
1760+
<p>
1761+
<strong><i class="fa fa-archive"></i> SBOM</strong>
1762+
</p>
1763+
<div class="list-group">
1764+
{% include 'base/list.html' with list=sbom.sbom_versioned type="Versioned Packages" limit=100 %}
1765+
{% include 'base/list.html' with list=sbom.sbom_packages type="Packages" limit=100 %}
1766+
</div>
1767+
</div>
1768+
</div>
1769+
</div><!-- /.card -->
1770+
</div>
1771+
<!-- end row -->
1772+
</div>
1773+
</section>
1774+
<!-- ===========================end sbom ================================== -->
17471775
<a id="files" class="anchor"></a>
17481776
<section class="content">
17491777
<div class="container-fluid">

mobsf/templates/static_analysis/ios_source_analysis.html

-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77
<!-- DataTables -->
88
<link href="{% static "datatables/css/datatables.combined.min.css" %}" rel="stylesheet">
99
<link href="{% static "adminlte/plugins/sweetalert2/sweetalert2.min.css" %}" rel="stylesheet">
10-
1110
<style type="text/css" media="print">
1211
@page { size: landscape; }
1312
@media print {

0 commit comments

Comments
 (0)