Skip to content

Add views to navigation tree #838

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 19, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 21 additions & 8 deletions crates/api-ui/src/navigation_trees/handlers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,17 +72,20 @@ pub async fn get_navigation_trees(
.await
.map_err(|e| NavigationTreesAPIError::Execution { source: e })?;

let mut catalogs_tree: BTreeMap<String, BTreeMap<String, Vec<String>>> = BTreeMap::new();
let mut catalogs_tree: BTreeMap<String, BTreeMap<String, Vec<(String, String)>>> =
BTreeMap::new();

for batch in tree_batches {
let databases = downcast_string_column(&batch, "database")?;
let schemas = downcast_string_column(&batch, "schema")?;
let tables = downcast_string_column(&batch, "table")?;
let table_types = downcast_string_column(&batch, "table_type")?;

for j in 0..batch.num_rows() {
let database = databases.value(j).to_string();
let schema = schemas.value(j).to_string();
let table = tables.value(j).to_string();
let table_type = table_types.value(j).to_string();

let db_entry = catalogs_tree.entry(database).or_default();

Expand All @@ -91,7 +94,7 @@ pub async fn get_navigation_trees(
}
let schema_entry = db_entry.entry(schema).or_default();
if !table.is_empty() {
schema_entry.push(table);
schema_entry.push((table, table_type));
}
}
}
Expand All @@ -107,12 +110,22 @@ pub async fn get_navigation_trees(
name: catalog_name,
schemas: schemas_map
.into_iter()
.map(|(schema_name, table_names)| NavigationTreeSchema {
name: schema_name,
tables: table_names
.into_iter()
.map(|name| NavigationTreeTable { name })
.collect(),
.map(|(schema_name, table_names)| {
let tables = table_names
.iter()
.filter(|(_, table_type)| table_type != "VIEW")
.map(|(name, _)| NavigationTreeTable { name: name.clone() })
.collect();
let views = table_names
.iter()
.filter(|(_, table_type)| table_type == "VIEW")
.map(|(name, _)| NavigationTreeTable { name: name.clone() })
.collect();
NavigationTreeSchema {
name: schema_name,
tables,
views,
}
})
.collect(),
})
Expand Down
1 change: 1 addition & 0 deletions crates/api-ui/src/navigation_trees/models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ pub struct NavigationTreeDatabase {
pub struct NavigationTreeSchema {
pub name: String,
pub tables: Vec<NavigationTreeTable>,
pub views: Vec<NavigationTreeTable>,
}

#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Validate, ToSchema)]
Expand Down
17 changes: 3 additions & 14 deletions crates/api-ui/src/tests/navigation_trees.rs
Original file line number Diff line number Diff line change
Expand Up @@ -133,18 +133,7 @@ async fn test_ui_databases_navigation() {
let query_payload = QueryCreatePayload {
worksheet_id: Some(worksheet.id),
query: format!(
"create or replace Iceberg TABLE {}.{}.{}
external_volume = ''
catalog = ''
base_location = ''
(
APP_ID TEXT,
PLATFORM TEXT,
ETL_TSTAMP TEXT,
COLLECTOR_TSTAMP TEXT NOT NULL,
DVCE_CREATED_TSTAMP TEXT,
EVENT TEXT,
EVENT_ID TEXT);",
"CREATE TABLE {}.{}.{} (APP_ID TEXT)",
expected1.data.name.clone(),
schema_name.clone(),
"tested1"
Expand Down Expand Up @@ -179,7 +168,7 @@ async fn test_ui_databases_navigation() {
.tables
.len()
);
// Information schema tables
// Information schema views
assert_eq!(
8,
databases_navigation
Expand All @@ -189,7 +178,7 @@ async fn test_ui_databases_navigation() {
.schemas
.first()
.unwrap()
.tables
.views
.len()
);

Expand Down
1 change: 0 additions & 1 deletion crates/core-executor/src/tests/query.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ use core_metastore::{
Database as MetastoreDatabase, Schema as MetastoreSchema, SchemaIdent as MetastoreSchemaIdent,
TableIdent as MetastoreTableIdent, Volume as MetastoreVolume,
};
use datafusion::arrow::compute::{SortColumn, SortOptions, take_record_batch};
use datafusion::assert_batches_eq;
use datafusion::sql::parser::{DFParser, Statement as DFStatement};
use datafusion::sql::sqlparser::ast::Statement as SQLStatement;
Expand Down
6 changes: 4 additions & 2 deletions crates/df-catalog/src/information_schema/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -81,17 +81,18 @@ impl InformationSchemaConfig {
"Catalog '{catalog_name}' not found in catalog list"
))
})?;
builder.add_navigation_tree(&catalog_name, None, None);
builder.add_navigation_tree(&catalog_name, None, None, None);

for schema_name in catalog.schema_names() {
builder.add_navigation_tree(&catalog_name, Some(schema_name.clone()), None);
builder.add_navigation_tree(&catalog_name, Some(schema_name.clone()), None, None);

if let Some(schema) = catalog.schema(&schema_name) {
for table_name in schema.table_names() {
builder.add_navigation_tree(
&catalog_name,
Some(schema_name.clone()),
Some(table_name),
Some(TableType::Base),
);
}
}
Expand All @@ -103,6 +104,7 @@ impl InformationSchemaConfig {
&catalog_name,
Some(INFORMATION_SCHEMA.to_string()),
Some(table.key().to_string()),
Some(TableType::View),
);
}
}
Expand Down
12 changes: 12 additions & 0 deletions crates/df-catalog/src/information_schema/navigation_tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use datafusion::arrow::{
};
use datafusion::execution::TaskContext;
use datafusion_common::DataFusionError;
use datafusion_expr::TableType;
use datafusion_physical_plan::SendableRecordBatchStream;
use datafusion_physical_plan::stream::RecordBatchStreamAdapter;
use datafusion_physical_plan::streaming::PartitionStream;
Expand All @@ -25,6 +26,7 @@ impl InformationSchemaNavigationTree {
Field::new("database", DataType::Utf8, false),
Field::new("schema", DataType::Utf8, true),
Field::new("table", DataType::Utf8, true),
Field::new("table_type", DataType::Utf8, true),
]))
}
pub(crate) fn new(config: InformationSchemaConfig) -> Self {
Expand All @@ -37,6 +39,7 @@ impl InformationSchemaNavigationTree {
databases: StringBuilder::new(),
schemas: StringBuilder::new(),
tables: StringBuilder::new(),
table_types: StringBuilder::new(),
schema: Arc::clone(&self.schema),
}
}
Expand Down Expand Up @@ -70,6 +73,7 @@ pub struct InformationSchemaNavigationTreeBuilder {
databases: StringBuilder,
schemas: StringBuilder,
tables: StringBuilder,
table_types: StringBuilder,
}

impl InformationSchemaNavigationTreeBuilder {
Expand All @@ -78,11 +82,18 @@ impl InformationSchemaNavigationTreeBuilder {
database: impl AsRef<str>,
schema: Option<String>,
table: Option<String>,
table_type: Option<TableType>,
) {
// Note: append_value is actually infallible.
self.databases.append_value(database.as_ref());
self.schemas.append_option(schema);
self.tables.append_option(table);
self.table_types
.append_option(table_type.map(|ttype| match ttype {
TableType::Base => "TABLE",
TableType::View => "VIEW",
TableType::Temporary => "TEMPORARY",
}));
}

fn finish(&mut self) -> Result<RecordBatch, ArrowError> {
Expand All @@ -92,6 +103,7 @@ impl InformationSchemaNavigationTreeBuilder {
Arc::new(self.databases.finish()),
Arc::new(self.schemas.finish()),
Arc::new(self.tables.finish()),
Arc::new(self.table_types.finish()),
],
)
}
Expand Down
1 change: 0 additions & 1 deletion crates/df-catalog/src/tests/information_schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ use crate::information_schema::information_schema::{
};
use crate::test_utils::sort_record_batch_by_sortable_columns;
use core_metastore::SlateDBMetastore;
use datafusion::arrow::compute::{SortColumn, SortOptions, take_record_batch};
use datafusion::execution::SessionStateBuilder;
use datafusion::execution::context::SessionContext;
use datafusion::prelude::SessionConfig;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,21 @@ source: crates/df-catalog/src/tests/information_schema.rs
description: "SELECT * FROM embucket.information_schema.navigation_tree ORDER BY database, schema, table"
snapshot_kind: text
---
+----------+--------------------+-----------------+
| database | schema | table |
+----------+--------------------+-----------------+
| embucket | | |
| embucket | information_schema | |
| embucket | information_schema | columns |
| embucket | information_schema | databases |
| embucket | information_schema | df_settings |
| embucket | information_schema | navigation_tree |
| embucket | information_schema | parameters |
| embucket | information_schema | routines |
| embucket | information_schema | schemata |
| embucket | information_schema | tables |
| embucket | information_schema | views |
| embucket | public | |
| embucket | public | first |
| embucket | public | second |
+----------+--------------------+-----------------+
+----------+--------------------+-----------------+------------+
| database | schema | table | table_type |
+----------+--------------------+-----------------+------------+
| embucket | | | |
| embucket | information_schema | | |
| embucket | information_schema | columns | VIEW |
| embucket | information_schema | databases | VIEW |
| embucket | information_schema | df_settings | VIEW |
| embucket | information_schema | navigation_tree | TABLE |
| embucket | information_schema | parameters | VIEW |
| embucket | information_schema | routines | VIEW |
| embucket | information_schema | schemata | VIEW |
| embucket | information_schema | tables | VIEW |
| embucket | information_schema | views | VIEW |
| embucket | public | | |
| embucket | public | first | TABLE |
| embucket | public | second | TABLE |
+----------+--------------------+-----------------+------------+