diff --git a/CHANGELOG.md b/CHANGELOG.md index a8582f42..496a6719 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to Data Hopper EDW (formerly hop-datavault) are documented i ## Unreleased +### Import tables shows progress (issue #134) + +- Catalog **Import database tables**, source-model **Import schema**, and dimensional **Import database tables** run JDBC listing and per-table import under Hop's cancelable progress dialog (wait cursor on Hop Web) +- Cancel stops before the next table; already-written catalog records and imported canvas tables are kept +- CSV / Parquet / Iceberg catalog import shows a wait cursor while discovering the file or table schema + ### Partition large BV SCD2 loads (issue #141) - SCD2 table option **Hash-key partitions** (None / 4 / 8 / 16) splits a Full rebuild so each satellite `ORDER BY` covers a first-byte slice of the parent hash key diff --git a/docs/help/import-database-tables-catalog-dialog.adoc b/docs/help/import-database-tables-catalog-dialog.adoc index da452941..9cf49a27 100644 --- a/docs/help/import-database-tables-catalog-dialog.adoc +++ b/docs/help/import-database-tables-catalog-dialog.adoc @@ -9,3 +9,5 @@ Related guide: link:../datavault-source-database.adoc[datavault-source-database] Select a Hop database connection and tables to import as catalog `DV_SOURCE` record definitions (or onto a source model). Filter by schema, multi-select tables, and optionally import primary/foreign keys. Existing catalog names are not overwritten without confirmation. + +Listing tables and importing selected tables each show a progress dialog (table name as the current step). Cancel stops before the next table; records already written to the catalog are kept. On Hop Web, a wait cursor is shown instead of a progress bar. diff --git a/src/main/java/org/hopper/edw/catalog/hopgui/perspective/importmenu/DataCatalogImportMenu.java b/src/main/java/org/hopper/edw/catalog/hopgui/perspective/importmenu/DataCatalogImportMenu.java index c2fa2195..e8f7f4c8 100644 --- a/src/main/java/org/hopper/edw/catalog/hopgui/perspective/importmenu/DataCatalogImportMenu.java +++ b/src/main/java/org/hopper/edw/catalog/hopgui/perspective/importmenu/DataCatalogImportMenu.java @@ -25,7 +25,6 @@ import org.apache.hop.ui.hopgui.context.GuiContextUtil; import org.apache.hop.ui.hopgui.context.IGuiContextHandler; import org.hopper.edw.catalog.hopgui.perspective.DataCatalogPerspective; -import org.hopper.edw.datavault.hopgui.GuiBusySupport; import org.hopper.edw.datavault.metadata.DataVaultModel; /** Shows the import menu for the Data Catalog perspective. */ @@ -88,15 +87,12 @@ private static List buildActions(DataCatalogImportContext context) { label, tooltip, image, - (shiftClicked, controlClicked, parameters) -> - GuiBusySupport.showWhile( - context.getShell(), - () -> { - importer.execute(context); - if (context.getOnComplete() != null) { - context.getOnComplete().run(); - } - }))); + (shiftClicked, controlClicked, parameters) -> { + importer.execute(context); + if (context.getOnComplete() != null) { + context.getOnComplete().run(); + } + })); } return actions; } diff --git a/src/main/java/org/hopper/edw/datavault/hopgui/GuiBusySupport.java b/src/main/java/org/hopper/edw/datavault/hopgui/GuiBusySupport.java index 50314f72..9a6113eb 100644 --- a/src/main/java/org/hopper/edw/datavault/hopgui/GuiBusySupport.java +++ b/src/main/java/org/hopper/edw/datavault/hopgui/GuiBusySupport.java @@ -15,6 +15,8 @@ */ package org.hopper.edw.datavault.hopgui; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicReference; import org.eclipse.swt.SWT; import org.eclipse.swt.graphics.Cursor; import org.eclipse.swt.widgets.Control; @@ -26,6 +28,31 @@ public final class GuiBusySupport { private GuiBusySupport() {} + /** + * Runs {@code callable} under the wait cursor and returns its value. Checked exceptions from + * {@code callable} are rethrown to the caller after the cursor is restored. + */ + public static T callWhile(Control control, Callable callable) throws Exception { + if (callable == null) { + return null; + } + AtomicReference value = new AtomicReference<>(); + AtomicReference error = new AtomicReference<>(); + showWhile( + control, + () -> { + try { + value.set(callable.call()); + } catch (Exception e) { + error.set(e); + } + }); + if (error.get() != null) { + throw error.get(); + } + return value.get(); + } + public static void showWhile(Control control, Runnable runnable) { if (runnable == null) { return; diff --git a/src/main/java/org/hopper/edw/datavault/hopgui/GuiProgressSupport.java b/src/main/java/org/hopper/edw/datavault/hopgui/GuiProgressSupport.java new file mode 100644 index 00000000..7f2f46ad --- /dev/null +++ b/src/main/java/org/hopper/edw/datavault/hopgui/GuiProgressSupport.java @@ -0,0 +1,188 @@ +/* + * Copyright 2026 i-Bridge bv + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.hopper.edw.datavault.hopgui; + +import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hop.core.IProgressMonitor; +import org.apache.hop.core.ProgressNullMonitorListener; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.dialog.ProgressMonitorDialog; +import org.apache.hop.ui.util.EnvironmentUtils; +import org.eclipse.swt.widgets.Shell; + +/** + * Runs long work under Hop's {@link ProgressMonitorDialog} (desktop) or a wait cursor (Hop Web / + * missing shell). + * + *

{@link ProgressMonitorDialog}'s monitor {@code done()} disposes the dialog and unblocks the UI + * thread. Callers must not invoke {@code done()} themselves. This helper defers {@code done()} + * until the worker return value is stored, matching {@code ModelDialogValidationSupport}. + */ +public final class GuiProgressSupport { + + private static final Class PKG = GuiProgressSupport.class; + + private GuiProgressSupport() {} + + /** Work that reports progress and may throw checked exceptions. */ + @FunctionalInterface + public interface ProgressWork { + T run(IProgressMonitor monitor) throws Exception; + } + + /** + * Result of {@link #run(Shell, boolean, ProgressWork)}, including whether the user cancelled. + * + *

{@code value} is {@code null} when work threw (an error dialog is already shown) or when + * there was no work. + */ + public record ProgressResult(T value, boolean cancelled) {} + + /** + * Runs {@code work} with a cancelable progress dialog on desktop Hop. On Hop Web, or when {@code + * shell} is missing, falls back to {@link GuiBusySupport} and a null monitor. + */ + public static ProgressResult run(Shell shell, boolean cancelable, ProgressWork work) { + if (work == null) { + return new ProgressResult<>(null, false); + } + if (shell == null || shell.isDisposed() || EnvironmentUtils.getInstance().isWeb()) { + return runWithWaitCursor(shell, work); + } + + AtomicReference value = new AtomicReference<>(); + ProgressMonitorDialog monitorDialog = new ProgressMonitorDialog(shell); + try { + monitorDialog.run( + cancelable, + monitor -> { + DeferredDoneMonitor deferred = new DeferredDoneMonitor(monitor); + try { + value.set(work.run(deferred)); + } catch (Throwable e) { + throw new InvocationTargetException( + e, + BaseMessages.getString( + PKG, "GuiProgressSupport.Error.Exception", e.getMessage())); + } finally { + deferred.finish(); + } + }); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "GuiProgressSupport.Error.Title"), + BaseMessages.getString(PKG, "GuiProgressSupport.Error.Message"), + e); + return new ProgressResult<>(null, false); + } catch (Exception e) { + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "GuiProgressSupport.Error.Title"), + BaseMessages.getString(PKG, "GuiProgressSupport.Error.Message"), + e); + return new ProgressResult<>(null, false); + } + + boolean cancelled = + monitorDialog.getProgressMonitor() != null + && monitorDialog.getProgressMonitor().isCanceled(); + return new ProgressResult<>(value.get(), cancelled); + } + + private static ProgressResult runWithWaitCursor(Shell shell, ProgressWork work) { + AtomicReference value = new AtomicReference<>(); + AtomicReference error = new AtomicReference<>(); + Runnable body = + () -> { + try { + value.set(work.run(new ProgressNullMonitorListener())); + } catch (Exception e) { + error.set(e); + } + }; + if (shell != null && !shell.isDisposed()) { + GuiBusySupport.showWhile(shell, body); + } else { + body.run(); + } + if (error.get() != null) { + if (shell != null && !shell.isDisposed()) { + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "GuiProgressSupport.Error.Title"), + BaseMessages.getString(PKG, "GuiProgressSupport.Error.Message"), + error.get()); + } + return new ProgressResult<>(null, false); + } + return new ProgressResult<>(value.get(), false); + } + + /** + * Forwards progress updates but defers {@link #done()} so the progress shell is not disposed + * until the caller has stored the worker result. + */ + private static final class DeferredDoneMonitor implements IProgressMonitor { + private final IProgressMonitor delegate; + private boolean finished; + + private DeferredDoneMonitor(IProgressMonitor delegate) { + this.delegate = delegate != null ? delegate : new ProgressNullMonitorListener(); + } + + @Override + public void beginTask(String message, int nrWorks) { + delegate.beginTask(message, nrWorks); + } + + @Override + public void subTask(String message) { + delegate.subTask(message); + } + + @Override + public boolean isCanceled() { + return delegate.isCanceled(); + } + + @Override + public void worked(int nrWorks) { + delegate.worked(nrWorks); + } + + @Override + public void done() { + // Deferred — see finish(). + } + + @Override + public void setTaskName(String taskName) { + delegate.setTaskName(taskName); + } + + private void finish() { + if (finished) { + return; + } + finished = true; + delegate.done(); + } + } +} diff --git a/src/main/java/org/hopper/edw/datavault/hopgui/file/dimensional/HopGuiDmDatabaseImportSupport.java b/src/main/java/org/hopper/edw/datavault/hopgui/file/dimensional/HopGuiDmDatabaseImportSupport.java index 67c89b85..04db0ef8 100644 --- a/src/main/java/org/hopper/edw/datavault/hopgui/file/dimensional/HopGuiDmDatabaseImportSupport.java +++ b/src/main/java/org/hopper/edw/datavault/hopgui/file/dimensional/HopGuiDmDatabaseImportSupport.java @@ -18,12 +18,8 @@ import java.util.ArrayList; import java.util.List; import org.apache.hop.core.Const; -import org.apache.hop.core.database.Database; import org.apache.hop.core.database.DatabaseMeta; import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.logging.ILoggingObject; -import org.apache.hop.core.logging.LoggingObjectType; -import org.apache.hop.core.logging.SimpleLoggingObject; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.IValueMeta; import org.apache.hop.core.row.RowMeta; @@ -38,6 +34,8 @@ import org.apache.hop.ui.hopgui.HopGui; import org.eclipse.swt.SWT; import org.eclipse.swt.widgets.Shell; +import org.hopper.edw.datavault.hopgui.GuiProgressSupport; +import org.hopper.edw.datavault.metadata.database.DvDatabaseSourceImportSupport; import org.hopper.edw.datavault.metadata.dimensional.DimensionalModel; import org.hopper.edw.datavault.metadata.dimensional.IDmTable; import org.hopper.edw.datavault.metadata.dimensional.dbimport.DmDatabaseImportOptions; @@ -102,10 +100,24 @@ public static void importDatabaseTables( return; } + GuiProgressSupport.ProgressResult progress = + GuiProgressSupport.run( + shell, + true, + monitor -> + DmDatabaseTableImportSupport.importTables( + model, + databaseMeta, + options, + selectedTables, + variables, + metadataProvider, + monitor)); + if (progress.value() == null) { + return; + } + DmDatabaseImportResult result = progress.value(); try { - DmDatabaseImportResult result = - DmDatabaseTableImportSupport.importTables( - model, databaseMeta, options, selectedTables, variables, metadataProvider); for (IDmTable table : result.getImportedTablesOrEmpty()) { model.getTables().add(table); } @@ -113,12 +125,20 @@ public static void importDatabaseTables( onChanged.run(); } - StringBuilder message = - new StringBuilder( - BaseMessages.getString( - PKG, - "HopGuiDmDatabaseImportSupport.Success.Message", - result.getImportedTablesOrEmpty().size())); + StringBuilder message = new StringBuilder(); + if (progress.cancelled()) { + message.append( + BaseMessages.getString( + PKG, + "HopGuiDmDatabaseImportSupport.Cancelled.Message", + result.getImportedTablesOrEmpty().size())); + message.append(Const.CR).append(Const.CR); + } + message.append( + BaseMessages.getString( + PKG, + "HopGuiDmDatabaseImportSupport.Success.Message", + result.getImportedTablesOrEmpty().size())); if (!result.getWarningsOrEmpty().isEmpty()) { message.append(Const.CR).append(Const.CR); @@ -159,25 +179,19 @@ public static void importDatabaseTables( private static List promptForTableSelection( Shell shell, IVariables variables, DatabaseMeta databaseMeta, DmDatabaseImportOptions options) throws HopException { - String schemaName = variables.resolve(options.getSchemaName()); - String[] tableNames; - ILoggingObject loggingObject = - new SimpleLoggingObject("DmDatabaseTableImport", LoggingObjectType.GENERAL, null); - try (Database database = new Database(loggingObject, variables, databaseMeta)) { - database.connect(); - tableNames = database.getTablenames(schemaName, false); - } catch (Exception e) { - new ErrorDialog( - shell, - BaseMessages.getString( - PKG, "ImportDmDatabaseTablesOptionsDialog.ErrorListingTables.Title"), - BaseMessages.getString( - PKG, "ImportDmDatabaseTablesOptionsDialog.ErrorListingTables.Message"), - e); + GuiProgressSupport.ProgressResult listed = + GuiProgressSupport.run( + shell, + true, + monitor -> + DvDatabaseSourceImportSupport.listTableNames( + databaseMeta, variables, options.getSchemaName(), monitor)); + if (listed.cancelled() || listed.value() == null) { return null; } + String[] tableNames = listed.value(); - if (tableNames == null || tableNames.length == 0) { + if (tableNames.length == 0) { MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); mb.setText( BaseMessages.getString(PKG, "ImportDmDatabaseTablesOptionsDialog.NoTablesFound.Title")); diff --git a/src/main/java/org/hopper/edw/datavault/hopgui/file/sourcemodel/HopGuiSourceModelImportSupport.java b/src/main/java/org/hopper/edw/datavault/hopgui/file/sourcemodel/HopGuiSourceModelImportSupport.java index a79a5f91..34fd2cca 100644 --- a/src/main/java/org/hopper/edw/datavault/hopgui/file/sourcemodel/HopGuiSourceModelImportSupport.java +++ b/src/main/java/org/hopper/edw/datavault/hopgui/file/sourcemodel/HopGuiSourceModelImportSupport.java @@ -19,12 +19,7 @@ import java.util.List; import java.util.Set; import org.apache.hop.core.Const; -import org.apache.hop.core.database.Database; import org.apache.hop.core.database.DatabaseMeta; -import org.apache.hop.core.exception.HopException; -import org.apache.hop.core.logging.ILoggingObject; -import org.apache.hop.core.logging.LoggingObjectType; -import org.apache.hop.core.logging.SimpleLoggingObject; import org.apache.hop.core.util.Utils; import org.apache.hop.core.variables.IVariables; import org.apache.hop.i18n.BaseMessages; @@ -35,6 +30,7 @@ import org.apache.hop.ui.hopgui.HopGui; import org.eclipse.swt.SWT; import org.eclipse.swt.widgets.Shell; +import org.hopper.edw.datavault.hopgui.GuiProgressSupport; import org.hopper.edw.datavault.metadata.database.DvDatabaseSourceImportSupport; import org.hopper.edw.datavault.metadata.sourcemodel.SourceModel; import org.hopper.edw.datavault.metadata.sourcemodel.importing.DatabaseSchemaImportSupport; @@ -108,62 +104,56 @@ public static void importSchema(HopGui hopGui, SourceModel model, Runnable onCha return; } - List selectedTables; - try { - selectedTables = promptForTables(shell, variables, databaseMeta, options); - } catch (HopException e) { - new ErrorDialog( - shell, - BaseMessages.getString(PKG, "HopGuiSourceModelImportSupport.Error.Title"), - BaseMessages.getString(PKG, "HopGuiSourceModelImportSupport.Error.Message"), - e); - return; - } + List selectedTables = promptForTables(shell, variables, databaseMeta, options); if (selectedTables == null || selectedTables.isEmpty()) { return; } - try { - SourceSchemaImportResult result = - DatabaseSchemaImportSupport.importTables( - model, databaseMeta, options, selectedTables, variables, metadataProvider); - DatabaseSchemaImportSupport.applyImportResult(model, result); - if (options.isPublishToCatalog()) { - DvDatabaseSourceImportSupport.refreshCatalogPerspective(); - } - if (onChanged != null) { - onChanged.run(); - } - showResultDialog(shell, result); - } catch (Exception e) { - new ErrorDialog( - shell, - BaseMessages.getString(PKG, "HopGuiSourceModelImportSupport.Error.Title"), - BaseMessages.getString(PKG, "HopGuiSourceModelImportSupport.Error.Message"), - e); + GuiProgressSupport.ProgressResult progress = + GuiProgressSupport.run( + shell, + true, + monitor -> + DatabaseSchemaImportSupport.importTables( + model, + databaseMeta, + options, + selectedTables, + variables, + metadataProvider, + monitor)); + if (progress.value() == null) { + return; + } + SourceSchemaImportResult result = progress.value(); + DatabaseSchemaImportSupport.applyImportResult(model, result); + if (options.isPublishToCatalog()) { + DvDatabaseSourceImportSupport.refreshCatalogPerspective(); } + if (onChanged != null) { + onChanged.run(); + } + showResultDialog(shell, result, progress.cancelled()); } private static List promptForTables( Shell shell, IVariables variables, DatabaseMeta databaseMeta, - SourceSchemaImportOptions options) - throws HopException { - String schemaName = variables != null ? variables.resolve(options.getSchemaName()) : ""; - String[] tableNames; - ILoggingObject loggingObject = - new SimpleLoggingObject("SourceSchemaImport", LoggingObjectType.GENERAL, null); - try (Database database = new Database(loggingObject, variables, databaseMeta)) { - database.connect(); - tableNames = database.getTablenames(schemaName, false); - } catch (Exception e) { - throw new HopException( - BaseMessages.getString(PKG, "HopGuiSourceModelImportSupport.ErrorListingTables.Message"), - e); + SourceSchemaImportOptions options) { + GuiProgressSupport.ProgressResult listed = + GuiProgressSupport.run( + shell, + true, + monitor -> + DvDatabaseSourceImportSupport.listTableNames( + databaseMeta, variables, options.getSchemaName(), monitor)); + if (listed.cancelled() || listed.value() == null) { + return null; } + String[] tableNames = listed.value(); - if (tableNames == null || tableNames.length == 0) { + if (tableNames.length == 0) { MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); mb.setText(BaseMessages.getString(PKG, "HopGuiSourceModelImportSupport.NoTablesFound.Title")); mb.setMessage( @@ -203,8 +193,17 @@ private static List promptForTables( return new ArrayList<>(picked); } - private static void showResultDialog(Shell shell, SourceSchemaImportResult result) { + private static void showResultDialog( + Shell shell, SourceSchemaImportResult result, boolean cancelled) { StringBuilder message = new StringBuilder(); + if (cancelled) { + message.append( + BaseMessages.getString( + PKG, + "HopGuiSourceModelImportSupport.Cancelled.Message", + result.getImportedTablesOrEmpty().size())); + message.append(Const.CR).append(Const.CR); + } message.append( BaseMessages.getString( PKG, diff --git a/src/main/java/org/hopper/edw/datavault/metadata/database/DvDatabaseSourceImportSupport.java b/src/main/java/org/hopper/edw/datavault/metadata/database/DvDatabaseSourceImportSupport.java index d5f478c8..76d32c98 100644 --- a/src/main/java/org/hopper/edw/datavault/metadata/database/DvDatabaseSourceImportSupport.java +++ b/src/main/java/org/hopper/edw/datavault/metadata/database/DvDatabaseSourceImportSupport.java @@ -22,6 +22,8 @@ import java.util.List; import java.util.Set; import org.apache.hop.core.Const; +import org.apache.hop.core.IProgressMonitor; +import org.apache.hop.core.ProgressNullMonitorListener; import org.apache.hop.core.database.Database; import org.apache.hop.core.database.DatabaseMeta; import org.apache.hop.core.exception.HopDatabaseException; @@ -49,6 +51,7 @@ import org.hopper.edw.datavault.catalog.DvSourceCatalogService; import org.hopper.edw.datavault.catalog.RecordSourceIndicatorOptions; import org.hopper.edw.datavault.catalog.RecordSourceIndicatorSupport; +import org.hopper.edw.datavault.hopgui.GuiProgressSupport; import org.hopper.edw.datavault.metadata.DataVaultModel; import org.hopper.edw.datavault.metadata.DataVaultSource; import org.hopper.edw.datavault.metadata.SourceField; @@ -62,6 +65,23 @@ public final class DvDatabaseSourceImportSupport { /** Above this count, the table pick dialog starts with no tables pre-selected. */ static final int LARGE_SCHEMA_TABLE_THRESHOLD = 25; + /** Table selected for catalog import, with the record-definition name to write. */ + public record TableImportRequest(String tableName, String recordDefinitionName) {} + + /** Outcome of a bulk catalog table import. */ + public record CatalogTableImportResult( + int importedCount, List errors, boolean cancelled) { + public CatalogTableImportResult { + errors = errors != null ? List.copyOf(errors) : List.of(); + } + } + + @FunctionalInterface + interface TableImportWork { + /** Imports one table. Per-table failures should be handled by the implementation. */ + void importOne(TableImportRequest request) throws Exception; + } + private DvDatabaseSourceImportSupport() {} /** Bulk-import database tables as catalog-backed record definitions. */ @@ -125,22 +145,15 @@ public static void importDatabaseTables( } String schemaName = Const.NVL(options.getSchemaName(), ""); - String[] tableNames; - ILoggingObject loggingObject = - new SimpleLoggingObject("DvDatabaseSourceImport", LoggingObjectType.GENERAL, null); - try (Database db = new Database(loggingObject, variables, databaseMeta)) { - db.connect(); - tableNames = db.getTablenames(schemaName, false); - } catch (Exception e) { - new ErrorDialog( - shell, - BaseMessages.getString(PKG, "DvDatabaseSourceEditor.ErrorListingTables.DialogTitle"), - BaseMessages.getString(PKG, "DvDatabaseSourceEditor.ErrorListingTables.DialogMessage"), - e); + GuiProgressSupport.ProgressResult listed = + GuiProgressSupport.run( + shell, true, monitor -> listTableNames(databaseMeta, variables, schemaName, monitor)); + if (listed.cancelled() || listed.value() == null) { return; } + String[] tableNames = listed.value(); - if (tableNames == null || tableNames.length == 0) { + if (tableNames.length == 0) { MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); mb.setMessage( BaseMessages.getString(PKG, "DvDatabaseSourceEditor.NoTablesFound.DialogMessage")); @@ -211,74 +224,246 @@ public static void importDatabaseTables( return; } - int importedCount = 0; - List errors = new ArrayList<>(); - try (Database db = new Database(loggingObject, variables, databaseMeta)) { - db.connect(); + List requests = tableImportRequestsFromRows(selectedRows); + if (requests.isEmpty()) { + return; + } - for (Object[] row : selectedRows) { - if (row == null || row.length < 2) { - continue; - } - String tableName = stripTableNameQuotes(row[0] != null ? row[0].toString() : null); - String dataVaultSourceName = row[1] != null ? row[1].toString() : null; - if (Utils.isEmpty(tableName) || Utils.isEmpty(dataVaultSourceName)) { - continue; - } + GuiProgressSupport.ProgressResult progress = + GuiProgressSupport.run( + shell, + true, + monitor -> + importSelectedTables( + databaseMeta, + connectionName, + schemaName, + catalogConnectionName, + model, + variables, + metadataProvider, + requests, + options.getRecordSourceOptions(), + monitor)); + if (progress.value() == null) { + return; + } - try { - if (DvSourceCatalogService.exists( - dataVaultSourceName, catalogConnectionName, variables, metadataProvider)) { - errors.add( - BaseMessages.getString( - PKG, - "DvDatabaseSourceEditor.ImportTables.Exists.Message", - dataVaultSourceName, - tableName)); - continue; - } - - List fields = importFieldsFromTable(db, variables, schemaName, tableName); - RecordSourceIndicatorOptions tableRecordSource = - RecordSourceIndicatorSupport.resolveForTable( - options.getRecordSourceOptions(), fields, dataVaultSourceName); - DataVaultSource imported = - createDataVaultSource( - dataVaultSourceName, - connectionName, - schemaName, - tableName, - fields, - tableRecordSource); - RecordDefinitionCatalogWriter.upsertDataVaultSource( - imported, - catalogConnectionName, - model, - variables, - metadataProvider, - null, - null, - null); - importedCount++; - } catch (Exception e) { - errors.add( - BaseMessages.getString( - PKG, - "DvDatabaseSourceEditor.ImportTables.TableError.Message", - tableName, - e.getMessage())); - } + refreshCatalogPerspective(); + showCatalogImportResult(shell, progress.value(), progress.cancelled()); + } + + /** + * Lists physical table names for {@code schemaName} (empty schema = connection default). Returns + * an empty array when none are found. Returns {@code null} when the user cancelled. + */ + public static String[] listTableNames( + DatabaseMeta databaseMeta, IVariables variables, String schemaName, IProgressMonitor monitor) + throws HopException { + if (databaseMeta == null) { + throw new HopException( + BaseMessages.getString(PKG, "DvDatabaseSourceEditor.ErrorListingTables.DialogMessage")); + } + IProgressMonitor progress = monitor != null ? monitor : new ProgressNullMonitorListener(); + if (progress.isCanceled()) { + return null; + } + String resolvedSchema = + variables != null + ? Const.NVL(variables.resolve(schemaName), "") + : Const.NVL(schemaName, ""); + String connectionName = Const.NVL(databaseMeta.getName(), ""); + progress.beginTask( + BaseMessages.getString(PKG, "DvDatabaseSourceEditor.ListTables.Progress.Task"), 1); + if (Utils.isEmpty(resolvedSchema)) { + progress.subTask( + BaseMessages.getString( + PKG, "DvDatabaseSourceEditor.ListTables.Progress.SubTask", connectionName)); + } else { + progress.subTask( + BaseMessages.getString( + PKG, + "DvDatabaseSourceEditor.ListTables.Progress.SubTaskSchema", + connectionName, + resolvedSchema)); + } + ILoggingObject loggingObject = + new SimpleLoggingObject("DatabaseTableList", LoggingObjectType.GENERAL, null); + try (Database db = new Database(loggingObject, variables, databaseMeta)) { + db.connect(); + if (progress.isCanceled()) { + return null; } + String[] tableNames = db.getTablenames(resolvedSchema, false); + progress.worked(1); + if (progress.isCanceled()) { + return null; + } + return tableNames != null ? tableNames : new String[0]; } catch (Exception e) { - new ErrorDialog( - shell, - BaseMessages.getString(PKG, "DvDatabaseSourceEditor.ErrorImportingTables.DialogTitle"), + throw new HopException( + BaseMessages.getString(PKG, "DvDatabaseSourceEditor.ErrorListingTables.DialogMessage"), + e); + } + } + + /** + * Imports selected tables into the data catalog. SWT-free so unit tests and {@link + * GuiProgressSupport} can drive it with an {@link IProgressMonitor}. Does not call {@code + * monitor.done()}. + */ + public static CatalogTableImportResult importSelectedTables( + DatabaseMeta databaseMeta, + String connectionName, + String schemaName, + String catalogConnectionName, + DataVaultModel model, + IVariables variables, + IHopMetadataProvider metadataProvider, + List requests, + RecordSourceIndicatorOptions recordSourceOptions, + IProgressMonitor monitor) + throws HopException { + List workItems = requests != null ? requests : List.of(); + String taskName = + BaseMessages.getString( + PKG, "DvDatabaseSourceEditor.ImportTables.Progress.Task", workItems.size()); + List errors = new ArrayList<>(); + int[] importedCount = {0}; + ILoggingObject loggingObject = + new SimpleLoggingObject("DvDatabaseSourceImport", LoggingObjectType.GENERAL, null); + try (Database db = new Database(loggingObject, variables, databaseMeta)) { + db.connect(); + boolean cancelled = + forEachTableImport( + workItems, + taskName, + monitor, + request -> { + String tableName = stripTableNameQuotes(request.tableName()); + String dataVaultSourceName = Const.NVL(request.recordDefinitionName(), "").trim(); + if (Utils.isEmpty(tableName) || Utils.isEmpty(dataVaultSourceName)) { + return; + } + try { + if (DvSourceCatalogService.exists( + dataVaultSourceName, catalogConnectionName, variables, metadataProvider)) { + errors.add( + BaseMessages.getString( + PKG, + "DvDatabaseSourceEditor.ImportTables.Exists.Message", + dataVaultSourceName, + tableName)); + return; + } + List fields = + importFieldsFromTable(db, variables, schemaName, tableName); + RecordSourceIndicatorOptions tableRecordSource = + RecordSourceIndicatorSupport.resolveForTable( + recordSourceOptions, fields, dataVaultSourceName); + DataVaultSource imported = + createDataVaultSource( + dataVaultSourceName, + connectionName, + schemaName, + tableName, + fields, + tableRecordSource); + RecordDefinitionCatalogWriter.upsertDataVaultSource( + imported, + catalogConnectionName, + model, + variables, + metadataProvider, + null, + null, + null); + importedCount[0]++; + } catch (Exception e) { + errors.add( + BaseMessages.getString( + PKG, + "DvDatabaseSourceEditor.ImportTables.TableError.Message", + tableName, + e.getMessage())); + } + }); + return new CatalogTableImportResult(importedCount[0], errors, cancelled); + } catch (HopException e) { + throw e; + } catch (Exception e) { + throw new HopException( BaseMessages.getString(PKG, "DvDatabaseSourceEditor.ErrorImportingTables.DialogMessage"), e); - return; } + } - refreshCatalogPerspective(); + static boolean forEachTableImport( + List requests, + String taskName, + IProgressMonitor monitor, + TableImportWork work) + throws Exception { + IProgressMonitor progress = monitor != null ? monitor : new ProgressNullMonitorListener(); + List items = requests != null ? requests : List.of(); + progress.beginTask(Const.NVL(taskName, ""), items.size()); + boolean cancelled = false; + for (TableImportRequest request : items) { + if (progress.isCanceled()) { + cancelled = true; + break; + } + String tableName = request != null ? Const.NVL(request.tableName(), "") : ""; + progress.subTask(tableName); + if (request != null && work != null) { + work.importOne(request); + } + progress.worked(1); + } + return cancelled || progress.isCanceled(); + } + + static List tableImportRequestsFromRows(List selectedRows) { + List requests = new ArrayList<>(); + if (selectedRows == null) { + return requests; + } + for (Object[] row : selectedRows) { + if (row == null || row.length < 2) { + continue; + } + String tableName = stripTableNameQuotes(row[0] != null ? row[0].toString() : null); + String dataVaultSourceName = row[1] != null ? row[1].toString() : null; + if (Utils.isEmpty(tableName) || Utils.isEmpty(dataVaultSourceName)) { + continue; + } + requests.add(new TableImportRequest(tableName, dataVaultSourceName)); + } + return requests; + } + + private static void showCatalogImportResult( + Shell shell, CatalogTableImportResult result, boolean cancelled) { + int importedCount = result != null ? result.importedCount() : 0; + List errors = result != null ? result.errors() : List.of(); + boolean wasCancelled = cancelled || (result != null && result.cancelled()); + + if (wasCancelled) { + MessageBox mb = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); + mb.setText( + BaseMessages.getString(PKG, "DvDatabaseSourceEditor.ImportTables.Cancelled.Title")); + StringBuilder message = new StringBuilder(); + message.append( + BaseMessages.getString( + PKG, "DvDatabaseSourceEditor.ImportTables.Cancelled.Message", importedCount)); + if (!errors.isEmpty()) { + message.append(Const.CR).append(Const.CR); + message.append(String.join(Const.CR, errors)); + } + mb.setMessage(message.toString()); + mb.open(); + return; + } if (!errors.isEmpty()) { new ErrorDialog( diff --git a/src/main/java/org/hopper/edw/datavault/metadata/dimensional/dbimport/DmDatabaseTableImportSupport.java b/src/main/java/org/hopper/edw/datavault/metadata/dimensional/dbimport/DmDatabaseTableImportSupport.java index 4b5b42f3..afe0df93 100644 --- a/src/main/java/org/hopper/edw/datavault/metadata/dimensional/dbimport/DmDatabaseTableImportSupport.java +++ b/src/main/java/org/hopper/edw/datavault/metadata/dimensional/dbimport/DmDatabaseTableImportSupport.java @@ -20,6 +20,8 @@ import java.util.List; import java.util.Locale; import java.util.Set; +import org.apache.hop.core.IProgressMonitor; +import org.apache.hop.core.ProgressNullMonitorListener; import org.apache.hop.core.database.Database; import org.apache.hop.core.database.DatabaseMeta; import org.apache.hop.core.exception.HopDatabaseException; @@ -71,6 +73,19 @@ public static DmDatabaseImportResult importTables( IVariables variables, IHopMetadataProvider metadataProvider) throws HopException { + return importTables( + model, databaseMeta, options, tableNames, variables, metadataProvider, null); + } + + public static DmDatabaseImportResult importTables( + DimensionalModel model, + DatabaseMeta databaseMeta, + DmDatabaseImportOptions options, + List tableNames, + IVariables variables, + IHopMetadataProvider metadataProvider, + IProgressMonitor monitor) + throws HopException { if (model == null) { throw new HopException( BaseMessages.getString(PKG, "DmDatabaseTableImportSupport.Error.NoModel")); @@ -94,14 +109,25 @@ public static DmDatabaseImportResult importTables( new SimpleLoggingObject("DmDatabaseTableImport", LoggingObjectType.GENERAL, null); String schemaName = variables != null ? variables.resolve(resolvedOptions.getSchemaName()) : ""; + IProgressMonitor progress = monitor != null ? monitor : new ProgressNullMonitorListener(); + progress.beginTask( + BaseMessages.getString( + PKG, "DmDatabaseTableImportSupport.Progress.Task", tableNames.size()), + tableNames.size()); + try (Database database = new Database(loggingObject, variables, databaseMeta)) { database.connect(); int index = 0; for (String rawTableName : tableNames) { + if (progress.isCanceled()) { + break; + } String tableName = stripTableNameQuotes(rawTableName); if (Utils.isEmpty(tableName)) { + progress.worked(1); continue; } + progress.subTask(tableName); try { IRowMeta rowMeta = database.getTableFieldsMeta(schemaName, tableName); if (rowMeta == null || rowMeta.isEmpty()) { @@ -133,6 +159,7 @@ public static DmDatabaseImportResult importTables( tableName, e.getMessage())); } + progress.worked(1); } } catch (HopDatabaseException e) { throw new HopException( diff --git a/src/main/java/org/hopper/edw/datavault/metadata/file/DvCsvSourceImportSupport.java b/src/main/java/org/hopper/edw/datavault/metadata/file/DvCsvSourceImportSupport.java index 92c6d37a..e7f67d8f 100644 --- a/src/main/java/org/hopper/edw/datavault/metadata/file/DvCsvSourceImportSupport.java +++ b/src/main/java/org/hopper/edw/datavault/metadata/file/DvCsvSourceImportSupport.java @@ -38,6 +38,7 @@ import org.hopper.edw.datavault.catalog.DvSourceCatalogService; import org.hopper.edw.datavault.catalog.RecordSourceIndicatorOptions; import org.hopper.edw.datavault.catalog.RecordSourceIndicatorSupport; +import org.hopper.edw.datavault.hopgui.GuiBusySupport; import org.hopper.edw.datavault.metadata.DataVaultModel; import org.hopper.edw.datavault.metadata.DataVaultSource; import org.hopper.edw.datavault.metadata.DvSourceType; @@ -81,11 +82,14 @@ public static void importCsvFile( RecordDefinitionDiscoveryService.DiscoveryResult discovery; try { discovery = - RecordDefinitionDiscoveryService.discover( - DvSourceType.CSV, - PhysicalSourceRef.builder().filePath(resolvedFile).build(), - variables, - metadataProvider); + GuiBusySupport.callWhile( + shell, + () -> + RecordDefinitionDiscoveryService.discover( + DvSourceType.CSV, + PhysicalSourceRef.builder().filePath(resolvedFile).build(), + variables, + metadataProvider)); } catch (Exception e) { new ErrorDialog( shell, diff --git a/src/main/java/org/hopper/edw/datavault/metadata/file/DvParquetSourceImportSupport.java b/src/main/java/org/hopper/edw/datavault/metadata/file/DvParquetSourceImportSupport.java index 3c9b5622..044b849c 100644 --- a/src/main/java/org/hopper/edw/datavault/metadata/file/DvParquetSourceImportSupport.java +++ b/src/main/java/org/hopper/edw/datavault/metadata/file/DvParquetSourceImportSupport.java @@ -37,6 +37,7 @@ import org.hopper.edw.datavault.catalog.DvSourceCatalogService; import org.hopper.edw.datavault.catalog.RecordSourceIndicatorOptions; import org.hopper.edw.datavault.catalog.RecordSourceIndicatorSupport; +import org.hopper.edw.datavault.hopgui.GuiBusySupport; import org.hopper.edw.datavault.metadata.DataVaultModel; import org.hopper.edw.datavault.metadata.DataVaultSource; import org.hopper.edw.datavault.metadata.DvSourceType; @@ -77,11 +78,14 @@ public static void importParquetFile( RecordDefinitionDiscoveryService.DiscoveryResult discovery; try { discovery = - RecordDefinitionDiscoveryService.discover( - DvSourceType.PARQUET, - PhysicalSourceRef.builder().filePath(resolvedFile).build(), - variables, - metadataProvider); + GuiBusySupport.callWhile( + shell, + () -> + RecordDefinitionDiscoveryService.discover( + DvSourceType.PARQUET, + PhysicalSourceRef.builder().filePath(resolvedFile).build(), + variables, + metadataProvider)); } catch (Exception e) { new ErrorDialog( shell, diff --git a/src/main/java/org/hopper/edw/datavault/metadata/iceberg/DvIcebergSourceImportSupport.java b/src/main/java/org/hopper/edw/datavault/metadata/iceberg/DvIcebergSourceImportSupport.java index 67612ade..ee8d7d65 100644 --- a/src/main/java/org/hopper/edw/datavault/metadata/iceberg/DvIcebergSourceImportSupport.java +++ b/src/main/java/org/hopper/edw/datavault/metadata/iceberg/DvIcebergSourceImportSupport.java @@ -33,6 +33,7 @@ import org.hopper.edw.datavault.catalog.DvSourceCatalogService; import org.hopper.edw.datavault.catalog.RecordSourceIndicatorOptions; import org.hopper.edw.datavault.catalog.RecordSourceIndicatorSupport; +import org.hopper.edw.datavault.hopgui.GuiBusySupport; import org.hopper.edw.datavault.metadata.DataVaultModel; import org.hopper.edw.datavault.metadata.DataVaultSource; import org.hopper.edw.datavault.metadata.DvSourceType; @@ -81,8 +82,11 @@ public static void importIcebergTable( RecordDefinitionDiscoveryService.DiscoveryResult discovery; try { discovery = - RecordDefinitionDiscoveryService.discover( - DvSourceType.ICEBERG, physicalRef, variables, metadataProvider); + GuiBusySupport.callWhile( + shell, + () -> + RecordDefinitionDiscoveryService.discover( + DvSourceType.ICEBERG, physicalRef, variables, metadataProvider)); } catch (Exception e) { new ErrorDialog( shell, diff --git a/src/main/java/org/hopper/edw/datavault/metadata/sourcemodel/importing/DatabaseSchemaImportSupport.java b/src/main/java/org/hopper/edw/datavault/metadata/sourcemodel/importing/DatabaseSchemaImportSupport.java index d338f312..a6f096dc 100644 --- a/src/main/java/org/hopper/edw/datavault/metadata/sourcemodel/importing/DatabaseSchemaImportSupport.java +++ b/src/main/java/org/hopper/edw/datavault/metadata/sourcemodel/importing/DatabaseSchemaImportSupport.java @@ -23,6 +23,8 @@ import java.util.Map; import java.util.Set; import org.apache.hop.core.Const; +import org.apache.hop.core.IProgressMonitor; +import org.apache.hop.core.ProgressNullMonitorListener; import org.apache.hop.core.database.Database; import org.apache.hop.core.database.DatabaseMeta; import org.apache.hop.core.exception.HopException; @@ -75,6 +77,19 @@ public static SourceSchemaImportResult importTables( IVariables variables, IHopMetadataProvider metadataProvider) throws HopException { + return importTables( + model, databaseMeta, options, tableNames, variables, metadataProvider, null); + } + + public static SourceSchemaImportResult importTables( + SourceModel model, + DatabaseMeta databaseMeta, + SourceSchemaImportOptions options, + List tableNames, + IVariables variables, + IHopMetadataProvider metadataProvider, + IProgressMonitor monitor) + throws HopException { if (model == null) { throw new HopException( BaseMessages.getString(PKG, "DatabaseSchemaImportSupport.Error.NoModel")); @@ -108,6 +123,11 @@ public static SourceSchemaImportResult importTables( Map physicalToLogical = new HashMap<>(); List cleanedTableNames = new ArrayList<>(); + IProgressMonitor progress = monitor != null ? monitor : new ProgressNullMonitorListener(); + progress.beginTask( + BaseMessages.getString(PKG, "DatabaseSchemaImportSupport.Progress.Task", tableNames.size()), + tableNames.size() + 1); + ILoggingObject loggingObject = new SimpleLoggingObject("SourceSchemaImport", LoggingObjectType.GENERAL, null); try (Database database = new Database(loggingObject, variables, databaseMeta)) { @@ -115,10 +135,15 @@ public static SourceSchemaImportResult importTables( int layoutIndex = 0; for (String rawTableName : tableNames) { + if (progress.isCanceled()) { + break; + } String tableName = DvDatabaseSourceImportSupport.stripTableNameQuotes(rawTableName); if (Utils.isEmpty(tableName)) { + progress.worked(1); continue; } + progress.subTask(tableName); cleanedTableNames.add(tableName); try { List fields = @@ -184,36 +209,42 @@ public static SourceSchemaImportResult importTables( tableName, e.getMessage())); } + progress.worked(1); } - // Include existing model tables in the name map so FKs to already-modeled parents resolve. - for (SourceTable existing : model.getTables()) { - if (existing == null || Utils.isEmpty(existing.getTableName())) { - continue; + if (!progress.isCanceled()) { + // Include existing model tables in the name map so FKs to already-modeled parents resolve. + for (SourceTable existing : model.getTables()) { + if (existing == null || Utils.isEmpty(existing.getTableName())) { + continue; + } + String key = normalizePhysicalKey(existing.getSchemaName(), existing.getTableName()); + physicalToLogical.putIfAbsent(key, existing.getName()); + // Also map bare table name for drivers that omit schema on parent side. + physicalToLogical.putIfAbsent( + normalizePhysicalKey(null, existing.getTableName()), existing.getName()); + } + for (SourceTable imported : importedTables) { + physicalToLogical.putIfAbsent( + normalizePhysicalKey(null, imported.getTableName()), imported.getName()); } - String key = normalizePhysicalKey(existing.getSchemaName(), existing.getTableName()); - physicalToLogical.putIfAbsent(key, existing.getName()); - // Also map bare table name for drivers that omit schema on parent side. - physicalToLogical.putIfAbsent( - normalizePhysicalKey(null, existing.getTableName()), existing.getName()); - } - for (SourceTable imported : importedTables) { - physicalToLogical.putIfAbsent( - normalizePhysicalKey(null, imported.getTableName()), imported.getName()); - } - try { - List foreignKeys = - DatabaseForeignKeyDiscoverySupport.discoverImportedForeignKeysForTables( - database, databaseMeta, schemaName, cleanedTableNames); - List relationships = - buildRelationshipsFromForeignKeys( - foreignKeys, physicalToLogical, model, importedRelationships, warnings); - importedRelationships.addAll(relationships); - } catch (Exception e) { - warnings.add( - BaseMessages.getString( - PKG, "DatabaseSchemaImportSupport.Warning.FkDiscoveryFailed", e.getMessage())); + progress.subTask( + BaseMessages.getString(PKG, "DatabaseSchemaImportSupport.Progress.ForeignKeys")); + try { + List foreignKeys = + DatabaseForeignKeyDiscoverySupport.discoverImportedForeignKeysForTables( + database, databaseMeta, schemaName, cleanedTableNames); + List relationships = + buildRelationshipsFromForeignKeys( + foreignKeys, physicalToLogical, model, importedRelationships, warnings); + importedRelationships.addAll(relationships); + } catch (Exception e) { + warnings.add( + BaseMessages.getString( + PKG, "DatabaseSchemaImportSupport.Warning.FkDiscoveryFailed", e.getMessage())); + } + progress.worked(1); } } catch (Exception e) { throw new HopException( diff --git a/src/main/resources/org/hopper/edw/datavault/hopgui/file/sourcemodel/messages/messages_en_US.properties b/src/main/resources/org/hopper/edw/datavault/hopgui/file/sourcemodel/messages/messages_en_US.properties index 6f681ab6..e3d1c677 100644 --- a/src/main/resources/org/hopper/edw/datavault/hopgui/file/sourcemodel/messages/messages_en_US.properties +++ b/src/main/resources/org/hopper/edw/datavault/hopgui/file/sourcemodel/messages/messages_en_US.properties @@ -210,6 +210,7 @@ HopGuiSourceModelImportSupport.NoneSelected.Title=Nothing selected HopGuiSourceModelImportSupport.NoneSelected.Message=No tables were selected for import. HopGuiSourceModelImportSupport.Success.Title=Schema import complete HopGuiSourceModelImportSupport.Success.Message=Imported {0} table(s) and {1} relationship(s). Published {2} catalog feed(s). +HopGuiSourceModelImportSupport.Cancelled.Message=Import was cancelled after {0} table(s). Remaining tables were skipped. HopGuiSourceModelImportSupport.Success.WarningsHeader=Warnings\: HopGuiSourceModelImportSupport.Success.ErrorsHeader=Errors\: diff --git a/src/main/resources/org/hopper/edw/datavault/hopgui/messages/messages_en_US.properties b/src/main/resources/org/hopper/edw/datavault/hopgui/messages/messages_en_US.properties index c03c47d2..5e81fcd1 100644 --- a/src/main/resources/org/hopper/edw/datavault/hopgui/messages/messages_en_US.properties +++ b/src/main/resources/org/hopper/edw/datavault/hopgui/messages/messages_en_US.properties @@ -52,4 +52,8 @@ StandardProjectElementsOffer.Error.Message=Unable to create the selected standar StandardProjectElementsOffer.SourceModel.Description=Shared source model defaults for this project StandardProjectElementsOffer.DataVault.Description=Shared Data Vault hashing, naming, and load settings for this project StandardProjectElementsOffer.BusinessVault.Description=Shared Business Vault target and load settings for this project -StandardProjectElementsOffer.Dimensional.Description=Shared dimensional warehouse and load settings for this project \ No newline at end of file +StandardProjectElementsOffer.Dimensional.Description=Shared dimensional warehouse and load settings for this project + +GuiProgressSupport.Error.Title=Operation failed +GuiProgressSupport.Error.Message=An unexpected error occurred while running the operation. +GuiProgressSupport.Error.Exception=Error\: {0} \ No newline at end of file diff --git a/src/main/resources/org/hopper/edw/datavault/metadata/database/messages/messages_en_US.properties b/src/main/resources/org/hopper/edw/datavault/metadata/database/messages/messages_en_US.properties index ceebd4cb..c1f49698 100644 --- a/src/main/resources/org/hopper/edw/datavault/metadata/database/messages/messages_en_US.properties +++ b/src/main/resources/org/hopper/edw/datavault/metadata/database/messages/messages_en_US.properties @@ -99,6 +99,12 @@ DvDatabaseSourceEditor.ImportTables.PartialFailure.Title=Import completed with e DvDatabaseSourceEditor.ImportTables.PartialFailure.Message=Imported {0} table(s).{1}{1}{2} DvDatabaseSourceEditor.ImportTables.Success.Title=Import complete DvDatabaseSourceEditor.ImportTables.Success.Message=Successfully imported {0} table(s). +DvDatabaseSourceEditor.ImportTables.Cancelled.Title=Import cancelled +DvDatabaseSourceEditor.ImportTables.Cancelled.Message=Imported {0} table(s). The remaining tables were skipped. +DvDatabaseSourceEditor.ImportTables.Progress.Task=Importing {0} table(s) into the data catalog +DvDatabaseSourceEditor.ListTables.Progress.Task=Listing tables +DvDatabaseSourceEditor.ListTables.Progress.SubTask=Reading table names from ''{0}'' +DvDatabaseSourceEditor.ListTables.Progress.SubTaskSchema=Reading table names from ''{0}'' (schema ''{1}'') DvDatabaseSourceEditor.NoConnection.DialogTitle=Error DvDatabaseSourceEditor.NoConnection.DialogMessage=Please select or enter a database connection name before importing fields from a table. diff --git a/src/main/resources/org/hopper/edw/datavault/metadata/dimensional/dbimport/messages/messages_en_US.properties b/src/main/resources/org/hopper/edw/datavault/metadata/dimensional/dbimport/messages/messages_en_US.properties index f9f8cde6..498230e4 100644 --- a/src/main/resources/org/hopper/edw/datavault/metadata/dimensional/dbimport/messages/messages_en_US.properties +++ b/src/main/resources/org/hopper/edw/datavault/metadata/dimensional/dbimport/messages/messages_en_US.properties @@ -18,6 +18,7 @@ DmDatabaseTableImportSupport.Error.NoDatabase=No database connection was selecte DmDatabaseTableImportSupport.Error.DatabaseConnection=Failed to connect to the selected database. DmDatabaseTableImportSupport.Error.NoColumns=Table ''{0}'' has no importable columns. DmDatabaseTableImportSupport.Error.TableImportFailed=Failed to import table ''{0}'': {1} +DmDatabaseTableImportSupport.Progress.Task=Importing {0} table(s) into the dimensional model DmDatabaseTableImportSupport.Dimension.Description=Dimension imported from database table ''{0}''. DmDatabaseTableImportSupport.Fact.Description=Fact imported from database table ''{0}''. @@ -47,6 +48,7 @@ ImportDmDatabaseTablesOptionsDialog.Selection.Column.TableName=Table name HopGuiDmDatabaseImportSupport.Success.Title=Database import complete HopGuiDmDatabaseImportSupport.Success.Message=Imported {0} table(s) from the database. Review source SQL, grains, and SCD policies before running Dimensional Update. +HopGuiDmDatabaseImportSupport.Cancelled.Message=Import was cancelled after {0} table(s). Remaining tables were skipped. HopGuiDmDatabaseImportSupport.Success.WarningsHeader=Review these import warnings: HopGuiDmDatabaseImportSupport.Success.ErrorsHeader=Some tables could not be imported: HopGuiDmDatabaseImportSupport.Error.Title=Database import failed diff --git a/src/main/resources/org/hopper/edw/datavault/metadata/sourcemodel/importing/messages/messages_en_US.properties b/src/main/resources/org/hopper/edw/datavault/metadata/sourcemodel/importing/messages/messages_en_US.properties index 3949e80a..3b1a3f87 100644 --- a/src/main/resources/org/hopper/edw/datavault/metadata/sourcemodel/importing/messages/messages_en_US.properties +++ b/src/main/resources/org/hopper/edw/datavault/metadata/sourcemodel/importing/messages/messages_en_US.properties @@ -23,6 +23,8 @@ DatabaseSchemaImportSupport.Warning.CatalogPublishFailed=Could not publish catal DatabaseSchemaImportSupport.Warning.FkDiscoveryFailed=Foreign key discovery failed\: {0} DatabaseSchemaImportSupport.Warning.FkEndpointMissing=Foreign key ''{0}'' skipped (endpoint tables not in model)\: {1} \u2192 {2} DatabaseSchemaImportSupport.Warning.FkDuplicateSkipped=Duplicate foreign key relationship skipped\: {0} +DatabaseSchemaImportSupport.Progress.Task=Importing {0} table(s) into the source model +DatabaseSchemaImportSupport.Progress.ForeignKeys=Discovering foreign keys ImportSourceSchemaOptionsDialog.Shell.Title=Import source schema ImportSourceSchemaOptionsDialog.DatabaseName.Label=Database connection diff --git a/src/test/java/org/hopper/edw/datavault/metadata/database/DvDatabaseSourceImportSupportTest.java b/src/test/java/org/hopper/edw/datavault/metadata/database/DvDatabaseSourceImportSupportTest.java index fe9380d0..fd39e038 100644 --- a/src/test/java/org/hopper/edw/datavault/metadata/database/DvDatabaseSourceImportSupportTest.java +++ b/src/test/java/org/hopper/edw/datavault/metadata/database/DvDatabaseSourceImportSupportTest.java @@ -20,8 +20,12 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.ArrayList; import java.util.List; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import org.apache.hop.core.IProgressMonitor; +import org.hopper.edw.datavault.metadata.database.DvDatabaseSourceImportSupport.TableImportRequest; import org.junit.jupiter.api.Test; class DvDatabaseSourceImportSupportTest { @@ -69,4 +73,108 @@ void tableNamesForSelectionIndexesPreservesDialogOrder() { assertEquals(List.of("gamma", "alpha"), List.copyOf(picked)); } + + @Test + void tableImportRequestsFromRowsSkipsIncompleteRows() { + List requests = + DvDatabaseSourceImportSupport.tableImportRequestsFromRows( + java.util.Arrays.asList( + new Object[] {"\"orders\"", "crm-orders"}, + new Object[] {null, "missing-table"}, + new Object[] {"customers"}, + null, + new Object[] {"customers", "crm-customers"})); + + assertEquals(2, requests.size()); + assertEquals("orders", requests.get(0).tableName()); + assertEquals("crm-orders", requests.get(0).recordDefinitionName()); + assertEquals("customers", requests.get(1).tableName()); + } + + @Test + void forEachTableImportReportsProgressPerTable() throws Exception { + List requests = + List.of( + new TableImportRequest("alpha", "src-alpha"), + new TableImportRequest("beta", "src-beta"), + new TableImportRequest("gamma", "src-gamma")); + RecordingMonitor monitor = new RecordingMonitor(); + List imported = new ArrayList<>(); + + boolean cancelled = + DvDatabaseSourceImportSupport.forEachTableImport( + requests, + "Importing 3 table(s)", + monitor, + request -> imported.add(request.tableName())); + + assertFalse(cancelled); + assertEquals(3, monitor.beginTaskWork); + assertEquals("Importing 3 table(s)", monitor.beginTaskName); + assertEquals(List.of("alpha", "beta", "gamma"), imported); + assertEquals(List.of("alpha", "beta", "gamma"), monitor.subTasks); + assertEquals(3, monitor.workedTotal); + } + + @Test + void forEachTableImportStopsWhenCancelled() throws Exception { + List requests = + List.of( + new TableImportRequest("alpha", "src-alpha"), + new TableImportRequest("beta", "src-beta"), + new TableImportRequest("gamma", "src-gamma"), + new TableImportRequest("delta", "src-delta")); + RecordingMonitor monitor = new RecordingMonitor(); + monitor.cancelAfterWorked = 2; + List imported = new ArrayList<>(); + + boolean cancelled = + DvDatabaseSourceImportSupport.forEachTableImport( + requests, "Importing", monitor, request -> imported.add(request.tableName())); + + assertTrue(cancelled); + assertEquals(List.of("alpha", "beta"), imported); + assertEquals(2, monitor.workedTotal); + } + + private static final class RecordingMonitor implements IProgressMonitor { + int beginTaskWork; + String beginTaskName; + int workedTotal; + int cancelAfterWorked = -1; + final List subTasks = new ArrayList<>(); + private final AtomicInteger worked = new AtomicInteger(); + + @Override + public void beginTask(String message, int nrWorks) { + beginTaskName = message; + beginTaskWork = nrWorks; + } + + @Override + public void subTask(String message) { + subTasks.add(message); + } + + @Override + public boolean isCanceled() { + return cancelAfterWorked >= 0 && worked.get() >= cancelAfterWorked; + } + + @Override + public void worked(int nrWorks) { + worked.addAndGet(nrWorks); + workedTotal += nrWorks; + } + + @Override + public void done() { + // unused + } + + @Override + public void setTaskName(String taskName) { + // unused + } + } } diff --git a/src/test/java/org/hopper/edw/datavault/metadata/sourcemodel/importing/DatabaseSchemaImportSupportTest.java b/src/test/java/org/hopper/edw/datavault/metadata/sourcemodel/importing/DatabaseSchemaImportSupportTest.java index ddcf5f90..10fb14b4 100644 --- a/src/test/java/org/hopper/edw/datavault/metadata/sourcemodel/importing/DatabaseSchemaImportSupportTest.java +++ b/src/test/java/org/hopper/edw/datavault/metadata/sourcemodel/importing/DatabaseSchemaImportSupportTest.java @@ -26,8 +26,12 @@ import java.util.Map; import java.util.Set; import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.IProgressMonitor; +import org.apache.hop.core.database.DatabaseMeta; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.gui.Point; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider; import org.hopper.edw.datavault.metadata.SourceField; import org.hopper.edw.datavault.metadata.database.DiscoveredForeignKey; import org.hopper.edw.datavault.metadata.sourcemodel.SourceColumn; @@ -203,4 +207,51 @@ void sanitizeNameRemovesOddCharacters() { assertEquals("order_header", DatabaseSchemaImportSupport.sanitizeName("order header")); assertFalse(DatabaseSchemaImportSupport.sanitizeName("!!!").isEmpty()); } + + @Test + void importTablesEmptyListReturnsEmptyWithoutProgress() throws Exception { + SourceModel model = new SourceModel(); + DatabaseMeta databaseMeta = new DatabaseMeta(); + databaseMeta.setName("crm"); + CountingMonitor monitor = new CountingMonitor(); + + SourceSchemaImportResult result = + DatabaseSchemaImportSupport.importTables( + model, + databaseMeta, + null, + List.of(), + new Variables(), + new MemoryMetadataProvider(), + monitor); + + assertTrue(result.getImportedTablesOrEmpty().isEmpty()); + assertEquals(0, monitor.beginTaskWork); + } + + private static final class CountingMonitor implements IProgressMonitor { + int beginTaskWork; + + @Override + public void beginTask(String message, int nrWorks) { + beginTaskWork = nrWorks; + } + + @Override + public void subTask(String message) {} + + @Override + public boolean isCanceled() { + return false; + } + + @Override + public void worked(int nrWorks) {} + + @Override + public void done() {} + + @Override + public void setTaskName(String taskName) {} + } }