Merge branch 'master' into Overview

Author: ananya-agarwal

.github/workflows/pr-comments.yml

@@ -1,13 +1,11 @@
 name: PR Comments
 
 on:
-  pull_request:
+  pull_request_target:
     branches:
       - master
 
 permissions:
-  contents: write
-  checks: write
   pull-requests: write
 
 jobs:
@@ -67,7 +65,7 @@ jobs:
           remove-link-from-badge: true
           default-branch: master
 
-  ut-files-comment:
+  python-ut-files-comment:
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout code

apps/beeswax/src/beeswax/common.py

@@ -125,7 +125,7 @@ def compute_check(x):
     # Pick the most probable compute object
     selected_compute = (cluster if compute_check(cluster)
                         else compute if compute_check(compute)
-                        else connector if compute_check(connector) else None)
+                        else connector if compute_check(connector) else cluster)
 
     # If found, we will attempt to reload it, first by id then by name
     if selected_compute:
@@ -134,8 +134,11 @@ def compute_check(x):
         if c:
           return c.to_dict()
 
-      if selected_compute.get('name'):
-        c = Compute.objects.filter(name=selected_compute['name']).first()
+      # Compute name is sometimes passed in the type and dialect fields.
+      # So, we will attempt to load compute using name, type and dialect as name.
+      compute_name = selected_compute.get('name', selected_compute.get('type', selected_compute.get('dialect')))
+      if compute_name:
+        c = Compute.objects.filter(name=compute_name).first()
         if c:
           return c.to_dict()
 

apps/filebrowser/src/filebrowser/api.py

@@ -165,7 +165,7 @@ def get_all_filesystems(request):
     user_home_dir = fs_home_dir_mapping[fs](request.user)
     config = _get_config(fs, request)
 
-    filesystems.append({'file_system': fs, 'user_home_directory': user_home_dir, 'config': config})
+    filesystems.append({'name': fs, 'user_home_directory': user_home_dir, 'config': config})
 
   return JsonResponse(filesystems, safe=False)
 

apps/filebrowser/src/filebrowser/api_test.py

@@ -748,8 +748,8 @@ def test_get_all_filesystems_without_hdfs(self):
 
           assert response.status_code == 200
           assert response_data == [
-            {'file_system': 's3a', 'user_home_directory': 's3a://test-bucket/test-user-home-dir/', 'config': {}},
-            {'file_system': 'ofs', 'user_home_directory': 'ofs://', 'config': {}},
+            {'name': 's3a', 'user_home_directory': 's3a://test-bucket/test-user-home-dir/', 'config': {}},
+            {'name': 'ofs', 'user_home_directory': 'ofs://', 'config': {}},
           ]
 
   def test_get_all_filesystems_success(self):
@@ -776,7 +776,7 @@ def test_get_all_filesystems_success(self):
               assert response.status_code == 200
               assert response_data == [
                 {
-                  'file_system': 'hdfs',
+                  'name': 'hdfs',
                   'user_home_directory': '/user/test-user',
                   'config': {
                     'is_trash_enabled': False,
@@ -787,8 +787,8 @@ def test_get_all_filesystems_success(self):
                     'supergroup': 'test-supergroup',
                   },
                 },
-                {'file_system': 's3a', 'user_home_directory': 's3a://test-bucket/test-user-home-dir/', 'config': {}},
-                {'file_system': 'ofs', 'user_home_directory': 'ofs://', 'config': {}},
+                {'name': 's3a', 'user_home_directory': 's3a://test-bucket/test-user-home-dir/', 'config': {}},
+                {'name': 'ofs', 'user_home_directory': 'ofs://', 'config': {}},
               ]
 
 

desktop/core/base_requirements.txt

@@ -5,28 +5,28 @@ avro-python3==1.8.2
 Babel==2.9.1
 celery[redis]==5.4.0
 cffi==1.15.0
-channels==3.0.3
-channels-redis==3.2.0
+channels==4.0.0
+channels-redis==4.0.0
 configobj==5.0.9
 cx-Oracle==8.3.0
 django-auth-ldap==4.3.0
-Django==3.2.25
+Django==4.1.13
 daphne==3.0.2
 django-redis==5.4.0
 django-celery-beat==2.6.0
 django-celery-results==2.5.1
-django-cors-headers==3.7.0
+django-cors-headers==3.13.0
 django-crequest==2018.5.11
 django-debug-panel==0.8.3
-django-debug-toolbar==1.11.1
-django-extensions==3.1.3
+django-debug-toolbar==3.6.0
+django-extensions==3.2.1
 django-ipware==3.0.2
 django_opentracing==1.1.0
 django_prometheus==1.0.15
 django-webpack-loader==1.0.0
 djangomako==1.3.2
-djangorestframework-simplejwt==5.2.0
-djangorestframework==3.12.2
+djangorestframework-simplejwt==5.2.1
+djangorestframework==3.14.0
 future==0.18.3
 gcs-oauth2-boto-plugin==3.0
 greenlet==3.1.1

desktop/core/ext-py3/django-babel/.travis.yml

@@ -17,6 +17,7 @@ env:
   - TOX_ENV=py35-django30
   - TOX_ENV=py35-django31
   - TOX_ENV=py35-django32
+  - TOX_ENV=py38-django41
 matrix:
   include:
     - env: TOX_ENV=py36-django18
@@ -35,6 +36,8 @@ matrix:
       python: "3.6"
     - env: TOX_ENV=py36-django32
       python: "3.6"
+    - env: TOX_ENV=py38-django41
+      python: "3.8"
     - env: TOX_ENV=py36-djangomaster
       python: "3.6"
 install: pip install tox

desktop/core/ext-py3/django-babel/setup.py

@@ -24,7 +24,7 @@ def read(*parts):
     url='https://github.com/python-babel/django-babel/',
     packages=find_packages(exclude=('tests',)),
     install_requires=[
-        'django>=1.8,<3.3',
+        'django>=1.8,<4.2',
         'babel>=1.3',
     ],
     classifiers=[

desktop/core/ext-py3/django-babel/tox.ini

@@ -1,6 +1,6 @@
 [tox]
 envlist = py{27,34,35,36}-django{18,111},
-          py{35,36,37}-django{20,21,22,30,31,32},
+          py{35,36,37,38}-django{20,21,22,30,31,32,41},
           py{36,37}-django{master},
           lint, docs
 
@@ -19,6 +19,7 @@ deps =
     django30: Django>=3.0,<3.1
     django31: Django>=3.1,<3.2
     django32: Django>=3.2,<3.3
+    django41: Django>=4.1,<4.2
     djangomaster: https://github.com/django/django/archive/master.tar.gz#egg=Django
 commands = py.test {posargs}
 

desktop/core/src/desktop/api2.py

@@ -93,7 +93,6 @@
 from metadata.conf import has_catalog
 from notebook.connectors.base import Notebook, get_interpreter
 from notebook.management.commands import notebook_setup
-from oozie.management.commands import oozie_setup
 from pig.management.commands import pig_setup
 from search.management.commands import search_setup
 from useradmin.models import Group, User
@@ -1450,6 +1449,9 @@ def _setup_pig_examples(request):
 
 
 def _setup_oozie_examples(request):
+  # Import dynamically to avoid oozie INSTALLED_APPS error
+  from oozie.management.commands import oozie_setup
+
   oozie_setup.Command().handle()
 
 

desktop/core/src/desktop/conf.py

@@ -2649,7 +2649,6 @@ def config_validator(user):
   from desktop.lib import i18n
   from desktop.models import Document, Document2  # Avoid cyclic loop
   from desktop.settings import DOCUMENT2_MAX_ENTRIES  # Avoid cyclic loop
-  from oozie.models import Job
 
   res = []
 
@@ -2678,10 +2677,14 @@ def config_validator(user):
     res.append(('SavedQuery_CLEANUP_WARNING', str(_('Saved Query has more than %d entries: %d, '
                 'please run "hue desktop_document_cleanup --cm-managed" to remove old entries' % (DOCUMENT2_MAX_ENTRIES, sq_count)))))
 
-  job_count = Job.objects.count()
-  if job_count > DOCUMENT2_MAX_ENTRIES:
-    res.append(('OOZIEJOB_CLEANUP_WARNING', str(_('Oozie Job has more than %d entries: %d, '
-                'please run "hue desktop_document_cleanup --cm-managed" to remove old entries' % (DOCUMENT2_MAX_ENTRIES, job_count)))))
+  # Check if oozie is enabled and present in INSTALLED_APPS
+  from desktop.settings import INSTALLED_APPS  # Avoid circular import
+  if 'oozie' in INSTALLED_APPS:
+    from oozie.models import Job
+    job_count = Job.objects.count()
+    if job_count > DOCUMENT2_MAX_ENTRIES:
+      res.append(('OOZIEJOB_CLEANUP_WARNING', str(_('Oozie Job has more than %d entries: %d, '
+                  'please run "hue desktop_document_cleanup --cm-managed" to remove old entries' % (DOCUMENT2_MAX_ENTRIES, job_count)))))
 
   if not get_secret_key():
     res.append((SECRET_KEY, str(_("Secret key should be configured as a random string. All sessions will be lost on restart"))))

desktop/core/src/desktop/js/apps/admin/ServerLogs/ServerLogsTab.scss

@@ -32,11 +32,16 @@
         background-color: $fluidx-white;
         margin: 0;
         padding: 2px;
-      }
-    }
+        white-space: nowrap;
+        font-family: 'Roboto Mono', monospace;
+        font-size: 12px;
 
-    .server_wrap {
-      white-space: nowrap;
+        &--wrap {
+          word-break: break-all;
+          word-wrap: break-word;
+          white-space: pre-wrap;
+        }
+      }
     }
 
     .server__no-logs-found {

desktop/core/src/desktop/js/apps/admin/ServerLogs/ServerLogsTab.test.tsx

@@ -24,12 +24,12 @@ import useLoadData from '../../../utils/hooks/useLoadData/useLoadData';
 
 const mockData = jest.fn().mockReturnValue({
   logs: ['Log entry 1', 'Log entry 2'],
-  hue_hostname: 'test-hostname'
+  hueHostname: 'test-hostname'
 });
 
 const emptyMockData = jest.fn().mockReturnValue({
   logs: [],
-  hue_hostname: 'test-hostname'
+  hueHostname: 'test-hostname'
 });
 
 jest.mock('../../../utils/hooks/useLoadData/useLoadData');
@@ -94,10 +94,11 @@ describe('ServerLogs Component', () => {
 
     render(<ServerLogs />);
 
-    expect(screen.getByText('Log entry 1')).toHaveClass('server_wrap');
+    expect(screen.getByLabelText('Wrap logs')).toBeChecked();
+    expect(screen.getByText('Log entry 1')).toHaveClass('server__log-line--wrap');
 
     fireEvent.click(screen.getByLabelText('Wrap logs'));
-
-    expect(screen.getByText('Log entry 1')).not.toHaveClass('server_wrap');
+    expect(screen.getByLabelText('Wrap logs')).not.toBeChecked();
+    expect(screen.getByText('Log entry 1')).not.toHaveClass('server__log-line--wrap');
   });
 });

desktop/core/src/desktop/js/apps/admin/ServerLogs/ServerLogsTab.tsx

@@ -15,28 +15,25 @@
 // limitations under the License.
 
 import React, { useState } from 'react';
-import { Spin, Alert } from 'antd';
+import { Alert } from 'antd';
 import ServerLogsHeader from './ServerLogsHeader';
 import { i18nReact } from '../../../utils/i18nReact';
 import useLoadData from '../../../utils/hooks/useLoadData/useLoadData';
+import LoadingErrorWrapper from '../../../reactComponents/LoadingErrorWrapper/LoadingErrorWrapper';
 import HighlightText from '../Components/HighlightText';
 import { SERVER_LOGS_API_URL } from '../Components/utils';
 import './ServerLogsTab.scss';
 
 interface ServerLogsData {
   logs: string[];
-  hue_hostname: string;
+  hueHostname: string;
 }
 
 const ServerLogs: React.FC = (): JSX.Element => {
   const [filter, setFilter] = useState<string>('');
   const [wrapLogs, setWrapLogs] = useState(true);
   const { t } = i18nReact.useTranslation();
-  const {
-    data: logsData,
-    loading,
-    error
-  } = useLoadData<ServerLogsData>(SERVER_LOGS_API_URL, {
+  const { data, loading, error } = useLoadData<ServerLogsData>(SERVER_LOGS_API_URL, {
     params: {
       reverse: true
     }
@@ -54,35 +51,31 @@ const ServerLogs: React.FC = (): JSX.Element => {
     );
   }
 
+  const isEmptyLogs = !data?.logs || !data?.logs?.some(log => log.length);
+
   return (
     <div className="hue-server-logs-component">
-      <Spin spinning={loading}>
-        {!loading && (
-          <>
-            <ServerLogsHeader
-              onFilterChange={setFilter}
-              onWrapLogsChange={setWrapLogs}
-              hostName={logsData?.hue_hostname ?? ''}
-            />
-            {logsData && (logsData.logs.length === 0 || logsData.logs[0] === '') && (
-              <pre className="server__no-logs-found">No logs found!</pre>
-            )}
-
-            {logsData && logsData.logs.length > 0 && logsData.logs[0] !== '' && (
-              <div className="server__display-logs">
-                {logsData.logs.map((line, index) => (
-                  <div
-                    className={`server__log-line ${wrapLogs ? 'server_wrap' : ''}`}
-                    key={'logs_' + index}
-                  >
-                    <HighlightText text={line} searchValue={filter} />
-                  </div>
-                ))}
+      <LoadingErrorWrapper loading={loading} errors={[]}>
+        <ServerLogsHeader
+          onFilterChange={setFilter}
+          onWrapLogsChange={setWrapLogs}
+          hostName={data?.hueHostname ?? ''}
+        />
+        {isEmptyLogs ? (
+          <pre className="server__no-logs-found">No logs found!</pre>
+        ) : (
+          <div className="server__display-logs">
+            {data.logs.map((line, index) => (
+              <div
+                className={`server__log-line ${wrapLogs ? 'server__log-line--wrap' : ''}`}
+                key={'logs_' + index}
+              >
+                <HighlightText text={line} searchValue={filter} />
               </div>
-            )}
-          </>
+            ))}
+          </div>
         )}
-      </Spin>
+      </LoadingErrorWrapper>
     </div>
   );
 };

desktop/core/src/desktop/js/apps/storageBrowser/StorageBrowserPage.tsx

@@ -55,11 +55,11 @@ const StorageBrowserPage = (): JSX.Element => {
         <Tabs
           className="hue-storage-browser__tab"
           destroyInactiveTabPane
-          defaultActiveKey={urlFileSystem ?? data?.[0]?.file_system}
-          items={data?.map(fs => ({
-            label: fs.file_system.toUpperCase(),
-            key: fs.file_system,
-            children: <StorageBrowserTab fileSystem={fs} />
+          defaultActiveKey={urlFileSystem ?? data?.[0]?.name}
+          items={data?.map(fileSystem => ({
+            label: fileSystem.name.toUpperCase(),
+            key: fileSystem.name,
+            children: <StorageBrowserTab fileSystem={fileSystem} />
           }))}
         />
       </LoadingErrorWrapper>