Skip to content

Commit e8b1c8b

Browse files
tomerm-iguaziotomerm-iguazio
authored
DBFS - list fixes: "details" in error and empty lists (#1834)
1 parent 353ba73 commit e8b1c8b

File tree

4 files changed

+231
-118
lines changed

4 files changed

+231
-118
lines changed

‎fsspec/implementations/dbfs.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,13 @@ class DatabricksException(Exception):
1414
Helper class for exceptions raised in this module.
1515
"""
1616

17-
def __init__(self, error_code, message):
17+
def __init__(self, error_code, message, details=None):
1818
"""Create a new DatabricksException"""
1919
super().__init__(message)
2020

2121
self.error_code = error_code
2222
self.message = message
23+
self.details = details
2324

2425

2526
class DatabricksFileSystem(AbstractFileSystem):
@@ -80,7 +81,7 @@ def ls(self, path, detail=True, **kwargs):
8081
raise FileNotFoundError(e.message) from e
8182

8283
raise
83-
files = r["files"]
84+
files = r.get("files", [])
8485
out = [
8586
{
8687
"name": o["path"],

‎fsspec/implementations/tests/cassettes/test_dbfs/test_dbfs_file_listing.yaml

Lines changed: 0 additions & 112 deletions
This file was deleted.

‎fsspec/implementations/tests/cassettes/test_dbfs/test_dbfs_listing.yaml

Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
interactions:
2+
- request:
3+
body: '{"path": "/FileStore/fsspec_dbfs/project_root/empty_dir"}'
4+
headers:
5+
Accept:
6+
- '*/*'
7+
Accept-Encoding:
8+
- gzip, deflate
9+
Connection:
10+
- keep-alive
11+
Content-Length:
12+
- '57'
13+
Content-Type:
14+
- application/json
15+
User-Agent:
16+
- python-requests/2.32.3
17+
authorization:
18+
- DUMMY
19+
method: POST
20+
uri: https://my_instance.com/api/2.0/dbfs/mkdirs
21+
response:
22+
body:
23+
string: '{}'
24+
headers:
25+
access-control-allow-headers:
26+
- Authorization, X-Databricks-Azure-Workspace-Resource-Id, X-Databricks-Org-Id,
27+
Content-Type
28+
access-control-allow-origin:
29+
- '*'
30+
cache-control:
31+
- no-cache, no-store, must-revalidate
32+
content-length:
33+
- '2'
34+
content-type:
35+
- application/json
36+
expires:
37+
- '0'
38+
pragma:
39+
- no-cache
40+
server:
41+
- databricks
42+
strict-transport-security:
43+
- max-age=31536000; includeSubDomains; preload
44+
x-content-type-options:
45+
- nosniff
46+
x-request-id:
47+
- e71bb75f-b05a-4523-b8bc-ce331812fbf0
48+
status:
49+
code: 200
50+
message: OK
51+
- request:
52+
body: '{"path": "/FileStore/fsspec_dbfs"}'
53+
headers:
54+
Accept:
55+
- '*/*'
56+
Accept-Encoding:
57+
- gzip, deflate
58+
Connection:
59+
- keep-alive
60+
Content-Length:
61+
- '34'
62+
Content-Type:
63+
- application/json
64+
User-Agent:
65+
- python-requests/2.32.3
66+
authorization:
67+
- DUMMY
68+
method: GET
69+
uri: https://my_instance.com/api/2.0/dbfs/list
70+
response:
71+
body:
72+
string: !!binary |
73+
H4sIAAAAAAAEAxzMMQ7CMAwF0Lv8OVKYcwAuwIiQ1SaOMKI4ss3SqncvZX3D29DlzY5y3zCmeKIg
74+
X39yCzXO3X1wpTZ3z8P0xTXIVAMJ4tTEUMK+nP4JuayMcklYtEmXOoXoh0KWU/fHfgAAAP//AwCB
75+
uORhbAAAAA==
76+
headers:
77+
access-control-allow-headers:
78+
- Authorization, X-Databricks-Azure-Workspace-Resource-Id, X-Databricks-Org-Id,
79+
Content-Type
80+
access-control-allow-origin:
81+
- '*'
82+
cache-control:
83+
- no-cache, no-store, must-revalidate
84+
content-encoding:
85+
- gzip
86+
content-type:
87+
- application/json
88+
expires:
89+
- '0'
90+
pragma:
91+
- no-cache
92+
server:
93+
- databricks
94+
strict-transport-security:
95+
- max-age=31536000; includeSubDomains; preload
96+
transfer-encoding:
97+
- chunked
98+
vary:
99+
- Accept-Encoding
100+
x-content-type-options:
101+
- nosniff
102+
x-request-id:
103+
- 6e66514c-c883-49c7-b215-2fecc5ef8222
104+
status:
105+
code: 200
106+
message: OK
107+
- request:
108+
body: '{"path": "/FileStore/fsspec_dbfs/project_root/empty_dir"}'
109+
headers:
110+
Accept:
111+
- '*/*'
112+
Accept-Encoding:
113+
- gzip, deflate
114+
Connection:
115+
- keep-alive
116+
Content-Length:
117+
- '57'
118+
Content-Type:
119+
- application/json
120+
User-Agent:
121+
- python-requests/2.32.3
122+
authorization:
123+
- DUMMY
124+
method: GET
125+
uri: https://my_instance.com/api/2.0/dbfs/list
126+
response:
127+
body:
128+
string: '{}'
129+
headers:
130+
access-control-allow-headers:
131+
- Authorization, X-Databricks-Azure-Workspace-Resource-Id, X-Databricks-Org-Id,
132+
Content-Type
133+
access-control-allow-origin:
134+
- '*'
135+
cache-control:
136+
- no-cache, no-store, must-revalidate
137+
content-length:
138+
- '2'
139+
content-type:
140+
- application/json
141+
expires:
142+
- '0'
143+
pragma:
144+
- no-cache
145+
server:
146+
- databricks
147+
strict-transport-security:
148+
- max-age=31536000; includeSubDomains; preload
149+
x-content-type-options:
150+
- nosniff
151+
x-request-id:
152+
- 59b8c763-be19-4402-8d37-135a7d6c7aed
153+
status:
154+
code: 200
155+
message: OK
156+
- request:
157+
body: '{"path": "/FileStore/fsspec_dbfs/project_root/missing_directory"}'
158+
headers:
159+
Accept:
160+
- '*/*'
161+
Accept-Encoding:
162+
- gzip, deflate
163+
Connection:
164+
- keep-alive
165+
Content-Length:
166+
- '65'
167+
Content-Type:
168+
- application/json
169+
User-Agent:
170+
- python-requests/2.32.3
171+
authorization:
172+
- DUMMY
173+
method: GET
174+
uri: https://my_instance.com/api/2.0/dbfs/list
175+
response:
176+
body:
177+
string: !!binary |
178+
H4sIAAAAAAAEAzyOwWrDMBBEf0XoXNtx4hqSU6B1IZcY4hQKpQhFWrkqtlfdVUtDyL9XbqCnZdh5
179+
M3ORQISkDFqQG3louvb58NCox7bp1L49quZl1x3lnRyBWfezZ4/C+QEEkrCewESks4Afz5EFTiLo
180+
+C6Kp+To0gcKxxzAKHtyXATCjwQoQozF6Jn91Kv/kDzVWIjaDyw3rxe5jecwF84n7xH7AXTwnBsc
181+
i5vMKZj8AJ9fwHE3OUwBdFPK20TWTjuoa8jK2q6yamGq7ATrZbYuYelW9aLU91ViGOj7b4mOOlHy
182+
+nb9BQAA//8DADr+FHAYAQAA
183+
headers:
184+
access-control-allow-headers:
185+
- Authorization, X-Databricks-Azure-Workspace-Resource-Id, X-Databricks-Org-Id,
186+
Content-Type
187+
access-control-allow-origin:
188+
- '*'
189+
cache-control:
190+
- no-cache, no-store, must-revalidate
191+
content-encoding:
192+
- gzip
193+
content-type:
194+
- application/json
195+
expires:
196+
- '0'
197+
pragma:
198+
- no-cache
199+
server:
200+
- databricks
201+
strict-transport-security:
202+
- max-age=31536000; includeSubDomains; preload
203+
transfer-encoding:
204+
- chunked
205+
vary:
206+
- Accept-Encoding
207+
x-content-type-options:
208+
- nosniff
209+
x-request-id:
210+
- 6fafe66e-16d3-40c4-be92-91e2f3601a54
211+
status:
212+
code: 404
213+
message: Not Found
214+
version: 1

‎fsspec/implementations/tests/test_dbfs.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -129,11 +129,21 @@ def make_mock_diabetes_ds():
129129

130130

131131
@pytest.mark.vcr()
132-
def test_dbfs_file_listing(dbfsFS):
133-
assert "/FileStore" in dbfsFS.ls("/", detail=False)
134-
assert {"name": "/FileStore", "size": 0, "type": "directory"} in dbfsFS.ls(
135-
"/", detail=True
132+
def test_dbfs_listing(dbfsFS):
133+
dbfsFS.mkdir(
134+
"/FileStore/fsspec_dbfs/project_root/empty_dir",
135+
create_parents=True,
136+
exist_ok=True,
136137
)
138+
assert {
139+
"name": "/FileStore/fsspec_dbfs/project_root",
140+
"size": 0,
141+
"type": "directory",
142+
} in dbfsFS.ls("/FileStore/fsspec_dbfs", detail=True)
143+
assert dbfsFS.ls("/FileStore/fsspec_dbfs/project_root/empty_dir", detail=True) == []
144+
# Test that FileNotFoundError is raised when listing a missing directory
145+
with pytest.raises(FileNotFoundError):
146+
dbfsFS.ls("/FileStore/fsspec_dbfs/project_root/missing_directory", detail=True)
137147

138148

139149
@pytest.mark.vcr()

0 commit comments

Comments
 (0)