diff --git a/assemblies/debug/pom.xml b/assemblies/debug/pom.xml index dd528af8b82..d87ba26f14e 100644 --- a/assemblies/debug/pom.xml +++ b/assemblies/debug/pom.xml @@ -553,6 +553,12 @@ ${project.version} provided + + org.apache.hop + hop-transform-transformsmetrics + ${project.version} + provided + org.apache.hop hop-transform-validator diff --git a/assemblies/plugins/pom.xml b/assemblies/plugins/pom.xml index 3d3fe213e91..69000d8f745 100644 --- a/assemblies/plugins/pom.xml +++ b/assemblies/plugins/pom.xml @@ -1530,6 +1530,12 @@ ${project.version} zip + + org.apache.hop + hop-transform-transformsmetrics + ${project.version} + zip + org.apache.hop hop-transform-uniquerows diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/transformsmetrics.svg b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/transformsmetrics.svg new file mode 100644 index 00000000000..6829085d350 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/transformsmetrics.svg @@ -0,0 +1,23 @@ + + + + + + + diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/output-transform-metrics-dialog-fields-tab.png b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/output-transform-metrics-dialog-fields-tab.png new file mode 100644 index 00000000000..c54e155179e Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/output-transform-metrics-dialog-fields-tab.png differ diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/output-transform-metrics-dialog-transforms-tab.png b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/output-transform-metrics-dialog-transforms-tab.png new file mode 100644 index 00000000000..5d46bf31b78 Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/output-transform-metrics-dialog-transforms-tab.png differ diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc index 7e3762a5806..5c7aeeca614 100644 --- a/docs/hop-user-manual/modules/ROOT/nav.adoc +++ b/docs/hop-user-manual/modules/ROOT/nav.adoc @@ -227,6 +227,7 @@ under the License. *** xref:pipeline/transforms/numberrange.adoc[Number range] *** xref:pipeline/transforms/odata-input.adoc[OData input] *** xref:pipeline/transforms/orabulkloader.adoc[Oracle Bulk Loader] +*** xref:pipeline/transforms/outputtransformmetrics.adoc[Output transform metrics] *** xref:pipeline/transforms/parquet-file-input.adoc[Parquet file input] *** xref:pipeline/transforms/parquet-file-output.adoc[Parquet file output] *** xref:pipeline/transforms/pipeline-data-probe.adoc[Pipeline Data Probe] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc index 15f47959a23..b4e71ff8023 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc @@ -177,6 +177,7 @@ Those are explained once in xref:pipeline/formatting-values.adoc[Formatting numb * xref:pipeline/transforms/numberrange.adoc[Number range] * xref:pipeline/transforms/odata-input.adoc[OData input] * xref:pipeline/transforms/orabulkloader.adoc[Oracle Bulk Loader] +* xref:pipeline/transforms/outputtransformmetrics.adoc[Output transform metrics] * xref:pipeline/transforms/parquet-file-input.adoc[Parquet file input] * xref:pipeline/transforms/parquet-file-output.adoc[Parquet file output] * xref:pipeline/transforms/pipeline-data-probe.adoc[Pipeline Data Probe] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/outputtransformmetrics.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/outputtransformmetrics.adoc new file mode 100644 index 00000000000..db5fd7ac81a --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/outputtransformmetrics.adoc @@ -0,0 +1,91 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +:documentationPath: /pipeline/transforms/ +:language: en_US +:description: Wait for selected transforms to finish and output their metrics as rows. + += image:transforms/icons/transformsmetrics.svg[Output transform metrics Icon, role="image-doc-icon"] Output transform metrics + +[%noheader,cols="3a,1a", role="table-no-borders" ] +|=== +| +== Description + +Output transform metrics waits until the listed transform copies in the same pipeline have finished, then emits *one output row per watched copy* with the requested counters. + +This transform is a *row generator*. It does not consume incoming rows and must not have incoming hops. Connecting an upstream hop can deadlock the pipeline: the watched transform cannot finish while its output buffer is full, and this transform never drains that buffer. + +Use this transform when a pipeline itself needs metrics (for example after a Kettle/PDI import, or to feed Write to log). For project-wide monitoring, prefer xref:metadata-types/pipeline-log.adoc[Pipeline logging] or xref:pipeline/transforms/execinfo.adoc[Execution information]. + +| +== Supported Engines +[%noheader,cols="2,1a",frame=none, role="table-supported-engines"] +!=== +!Hop Engine! image:check_mark.svg[Supported, 24] +!Single Threaded! image:cross.svg[Not Supported, 24] +!Native Spark! image:cross.svg[Not Supported, 24] +!Beam Spark! image:cross.svg[Not Supported, 24] +!Beam Flink! image:cross.svg[Not Supported, 24] +!Beam Dataflow! image:cross.svg[Not Supported, 24] +!=== +|=== + +== Options + +=== Transforms + +image:transforms/output-transform-metrics-dialog-transforms-tab.png[Output transform metrics Transforms tab, width="90%"] + +[options="header"] +|=== +|Option|Description +|Transform name|Name of this transform. +|Metrics for the following transforms|The transform copies to wait for. You cannot watch this transform itself or a downstream target of it. +|Get Fields|Fill the grid with every other transform that is not a downstream target. +|=== + +[options="header"] +|=== +|Column|Description +|Transform name|Name of the transform to watch. +|Copy Nr|0-based copy number. Use one grid row per copy if the transform runs multiple copies. Variables are allowed. +|Required|When Yes, the pipeline fails if that copy cannot be found. When No, a missing copy is skipped. +|=== + +=== Fields + +image:transforms/output-transform-metrics-dialog-fields-tab.png[Output transform metrics Fields tab, width="90%"] + +Leave a field name empty to omit that column from the output. + +[options="header"] +|=== +|Option|Description +|Transform name|Name of the watched transform. +|Transform id|Plugin id of the watched transform. +|Lines input|Rows read from an external source. +|Lines output|Rows written to an external destination. +|Lines read|Rows read from previous transforms. +|Lines updated|Rows updated. +|Lines written|Rows written to the next transforms. +|Lines rejected|Rows sent to an error hop (`getLinesRejected()`). This is *not* the error count. +|Duration (ms)|Execution duration in milliseconds. +|=== + +== Kettle / PDI import + +Kettle called this transform *Output steps metrics* (`StepsMetrics`). Hop import rewrites the plugin type to `TransformsMetrics` and the XML tags from `step*` to `transform*`. Stored *values* of output field names (for example `Step name`) are left unchanged so existing pipelines keep their column names. diff --git a/integration-tests/transforms/0110-output-transform-metrics.hpl b/integration-tests/transforms/0110-output-transform-metrics.hpl new file mode 100644 index 00000000000..0f3a5808bcb --- /dev/null +++ b/integration-tests/transforms/0110-output-transform-metrics.hpl @@ -0,0 +1,254 @@ + + + + + 0110-output-transform-metrics + Y + + + + Normal + + + N + 1000 + 100 + - + 2026/08/29 12:00:00.000 + - + 2026/08/29 12:00:00.000 + + N + + + + + + 10 rows + Dummy A + Y + + + 5 rows + Dummy B + Y + + + Output transform metrics + Sort + Y + + + Sort + Verify + Y + + + + 10 rows + RowGenerator + + Y + + 1 + + none + + + + + value + Integer + # + + + + 1 + -1 + -1 + N + + + 10 + N + 5000 + now + FiveSecondsAgo + + + 80 + 80 + + + + 5 rows + RowGenerator + + Y + + 1 + + none + + + + + value + Integer + # + + + + 1 + -1 + -1 + N + + + 5 + N + 5000 + now + FiveSecondsAgo + + + 80 + 224 + + + + Dummy A + Dummy + + Y + + 1 + + none + + + + + 256 + 80 + + + + Dummy B + Dummy + + Y + + 1 + + none + + + + + 256 + 224 + + + + Output transform metrics + TransformsMetrics + + Y + + 1 + + none + + + + + Dummy A + 0 + Y + + + Dummy B + 0 + Y + + + Transform name + Transform id + Lines written + + + 448 + 152 + + + + Sort + SortRows + + Y + + 1 + + none + + + N + ${java.io.tmpdir} + + + Y + Y + N + 0 + Transform name + N + + + out + 1000000 + N + + + 656 + 152 + + + + Verify + Dummy + + Y + + 1 + + none + + + + + 832 + 152 + + + + + + diff --git a/integration-tests/transforms/datasets/golden-output-transform-metrics.csv b/integration-tests/transforms/datasets/golden-output-transform-metrics.csv new file mode 100644 index 00000000000..333ac5e5823 --- /dev/null +++ b/integration-tests/transforms/datasets/golden-output-transform-metrics.csv @@ -0,0 +1,3 @@ +Transform name,Transform id,Lines written +Dummy A,Dummy,10 +Dummy B,Dummy,5 diff --git a/integration-tests/transforms/main-0110-output-transform-metrics.hwf b/integration-tests/transforms/main-0110-output-transform-metrics.hwf new file mode 100644 index 00000000000..ea2a6011d3f --- /dev/null +++ b/integration-tests/transforms/main-0110-output-transform-metrics.hwf @@ -0,0 +1,80 @@ + + + + main-0110-output-transform-metrics + Y + + + + 0 + - + 2026/08/29 12:00:00.000 + - + 2026/08/29 12:00:00.000 + + + + + Start + + SPECIAL + + N + 0 + 0 + 60 + 12 + 0 + 1 + 1 + N + 128 + 112 + + + + Run Pipeline Unit Tests + + RunPipelineTests + + + + 0110-output-transform-metrics UNIT + + + N + 320 + 112 + + + + + + Start + Run Pipeline Unit Tests + Y + Y + Y + + + + + + diff --git a/integration-tests/transforms/metadata/dataset/golden-output-transform-metrics.json b/integration-tests/transforms/metadata/dataset/golden-output-transform-metrics.json new file mode 100644 index 00000000000..14d7ea83d02 --- /dev/null +++ b/integration-tests/transforms/metadata/dataset/golden-output-transform-metrics.json @@ -0,0 +1,32 @@ +{ + "base_filename": "golden-output-transform-metrics.csv", + "name": "golden-output-transform-metrics", + "description": "", + "dataset_fields": [ + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_format": "", + "field_name": "Transform name" + }, + { + "field_comment": "", + "field_length": -1, + "field_type": 2, + "field_precision": -1, + "field_format": "", + "field_name": "Transform id" + }, + { + "field_comment": "", + "field_length": 10, + "field_type": 5, + "field_precision": 0, + "field_format": "#", + "field_name": "Lines written" + } + ], + "folder_name": "" +} diff --git a/integration-tests/transforms/metadata/unit-test/0110-output-transform-metrics UNIT.json b/integration-tests/transforms/metadata/unit-test/0110-output-transform-metrics UNIT.json new file mode 100644 index 00000000000..4943dc65fc5 --- /dev/null +++ b/integration-tests/transforms/metadata/unit-test/0110-output-transform-metrics UNIT.json @@ -0,0 +1,36 @@ +{ + "variableValues": [], + "database_replacements": [], + "autoOpening": true, + "basePath": "", + "golden_data_sets": [ + { + "field_mappings": [ + { + "transform_field": "Transform name", + "data_set_field": "Transform name" + }, + { + "transform_field": "Transform id", + "data_set_field": "Transform id" + }, + { + "transform_field": "Lines written", + "data_set_field": "Lines written" + } + ], + "field_order": [ + "Transform name" + ], + "transform_name": "Verify", + "data_set_name": "golden-output-transform-metrics" + } + ], + "input_data_sets": [], + "name": "0110-output-transform-metrics UNIT", + "description": "", + "trans_test_tweaks": [], + "persist_filename": "", + "pipeline_filename": "./0110-output-transform-metrics.hpl", + "test_type": "UNIT_TEST" +} diff --git a/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleConst.java b/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleConst.java index 1c80f671697..92f5ce4461e 100644 --- a/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleConst.java +++ b/plugins/misc/import/src/main/java/org/apache/hop/imports/kettle/KettleConst.java @@ -87,6 +87,17 @@ public class KettleConst { {"step7", "transform7"}, {"step8", "transform8"}, {"step9", "transform9"}, + // Output transform metrics (Kettle: Output steps metrics) + {"stepRequired", "transformRequired"}, + {"stepnamefield", "transformnamefield"}, + {"stepidfield", "transformidfield"}, + {"steplinesinputfield", "transformlinesinputfield"}, + {"steplinesoutputfield", "transformlinesoutputfield"}, + {"steplinesreadfield", "transformlinesreadfield"}, + {"steplinesupdatedfield", "transformlinesupdatedfield"}, + {"steplineswrittentfield", "transformlineswrittenfield"}, + {"steplineserrorsfield", "transformlineserrorsfield"}, + {"stepsecondsfield", "transformsecondsfield"}, }) .collect(Collectors.toMap(data -> (String) data[0], data -> (String) data[1])); @@ -160,6 +171,7 @@ public class KettleConst { {"ROOT_JOB", "ROOT_WORKFLOW"}, {"BlockingStep", "BlockingTransform"}, {"BlockUntilStepsFinish", "BlockUntilTransformsFinish"}, + {"StepsMetrics", "TransformsMetrics"}, {"TypeExitExcelWriterStep", "TypeExitExcelWriterTransform"}, {"StepMetastructure", "TransformMetaStructure"}, {"JobExecutor", "WorkflowExecutor"}, diff --git a/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportTransformsMetricsTest.java b/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportTransformsMetricsTest.java new file mode 100644 index 00000000000..21594944b04 --- /dev/null +++ b/plugins/misc/import/src/test/java/org/apache/hop/imports/kettle/KettleImportTransformsMetricsTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.imports.kettle; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.io.ByteArrayInputStream; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import org.apache.hop.core.HopClientEnvironment; +import org.apache.hop.core.xml.XmlHandler; +import org.apache.hop.core.xml.XmlParserFactoryProducer; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Node; + +class KettleImportTransformsMetricsTest { + + private KettleImport kettleImport; + + @BeforeAll + static void setUpBeforeClass() throws Exception { + HopClientEnvironment.init(); + } + + @BeforeEach + void setUp() { + kettleImport = new KettleImport(); + } + + @Test + void testStepsMetricsBecomesTransformsMetrics() throws Exception { + Document doc = parse(kettleSnippet()); + processNode(doc); + + Node pipeline = XmlHandler.getSubNode(doc, "pipeline"); + assertNotNull(pipeline); + Node transform = XmlHandler.getSubNode(pipeline, "transform"); + assertNotNull(transform); + assertEquals("TransformsMetrics", XmlHandler.getTagValue(transform, "type")); + + Node list = XmlHandler.getSubNode(transform, "transforms"); + assertNotNull(list, "nested steps list should become transforms"); + Node watched = XmlHandler.getSubNode(list, "transform"); + assertNotNull(watched); + assertEquals("A", XmlHandler.getTagValue(watched, "name")); + assertEquals("0", XmlHandler.getTagValue(watched, "copyNr")); + assertEquals("Y", XmlHandler.getTagValue(watched, "transformRequired")); + assertNull(XmlHandler.getTagValue(watched, "stepRequired")); + + assertEquals("Step name", XmlHandler.getTagValue(transform, "transformnamefield")); + assertNull(XmlHandler.getTagValue(transform, "stepnamefield")); + assertEquals("Lines written", XmlHandler.getTagValue(transform, "transformlineswrittenfield")); + assertNull(XmlHandler.getTagValue(transform, "steplineswrittentfield")); + assertEquals("Lines rejected", XmlHandler.getTagValue(transform, "transformlineserrorsfield")); + assertNull(XmlHandler.getTagValue(transform, "steplineserrorsfield")); + } + + private String kettleSnippet() { + return "" + + "" + + "metrics" + + "StepsMetrics" + + "" + + "" + + "A" + + "0" + + "Y" + + "" + + "" + + "Step name" + + "Lines written" + + "Lines rejected" + + "" + + ""; + } + + private void processNode(Document doc) throws Exception { + Method method = + KettleImport.class.getDeclaredMethod( + "processNode", + Document.class, + Node.class, + Class.forName(entryTypeClassName()), + int.class); + method.setAccessible(true); + method.invoke(kettleImport, doc, doc, otherEntryType(), 0); + } + + private static String entryTypeClassName() { + return "org.apache.hop.imports.kettle.KettleImport$EntryType"; + } + + private Object otherEntryType() throws Exception { + Class entryTypeClass = Class.forName(entryTypeClassName()); + for (Object constant : entryTypeClass.getEnumConstants()) { + if ("OTHER".equals(constant.toString())) { + return constant; + } + } + throw new IllegalStateException("No OTHER entry type"); + } + + private static Document parse(String xml) throws Exception { + return XmlParserFactoryProducer.createSecureDocBuilderFactory() + .newDocumentBuilder() + .parse(new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8))); + } +} diff --git a/plugins/transforms/pom.xml b/plugins/transforms/pom.xml index c8ad5a2c5f6..29ecd883875 100644 --- a/plugins/transforms/pom.xml +++ b/plugins/transforms/pom.xml @@ -163,6 +163,7 @@ textfile tika tokenreplacement + transformsmetrics uniquerows uniquerowsbyhashset update diff --git a/plugins/transforms/transformsmetrics/pom.xml b/plugins/transforms/transformsmetrics/pom.xml new file mode 100644 index 00000000000..e46e5045e16 --- /dev/null +++ b/plugins/transforms/transformsmetrics/pom.xml @@ -0,0 +1,32 @@ + + + + 4.0.0 + + + org.apache.hop + hop-plugins-transforms + 2.20.0-SNAPSHOT + + + hop-transform-transformsmetrics + jar + Hop Plugins Transforms Output Transform Metrics + + diff --git a/plugins/transforms/transformsmetrics/src/assembly/assembly.xml b/plugins/transforms/transformsmetrics/src/assembly/assembly.xml new file mode 100644 index 00000000000..098027e8bb8 --- /dev/null +++ b/plugins/transforms/transformsmetrics/src/assembly/assembly.xml @@ -0,0 +1,46 @@ + + + + hop-transform-transformsmetrics + + zip + + . + + + ${project.basedir}/src/main/resources/version.xml + ${hop.plugin.libdir} + true + + + + + + ${project.basedir}/src/main/samples + config/projects/samples/ + + + + + + ${maven.multiModuleProjectDirectory}/assemblies/shared/hop-plugin-libs.xml + + diff --git a/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/MetricTransform.java b/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/MetricTransform.java new file mode 100644 index 00000000000..5be783f370d --- /dev/null +++ b/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/MetricTransform.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.transformsmetrics; + +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.metadata.api.HopMetadataProperty; + +@Getter +@Setter +public class MetricTransform { + + @HopMetadataProperty( + injectionKey = "TRANSFORM_NAME", + injectionKeyDescription = "TransformsMetricsMeta.Injection.TRANSFORM_NAME") + private String name; + + @HopMetadataProperty( + key = "copyNr", + injectionKey = "TRANSFORM_COPY_NR", + injectionKeyDescription = "TransformsMetricsMeta.Injection.TRANSFORM_COPY_NR") + private String copyNr; + + @HopMetadataProperty( + key = "transformRequired", + injectionKey = "TRANSFORM_REQUIRED", + injectionKeyDescription = "TransformsMetricsMeta.Injection.TRANSFORM_REQUIRED") + private boolean required; + + public MetricTransform() {} + + public MetricTransform(String name, String copyNr, boolean required) { + this.name = name; + this.copyNr = copyNr; + this.required = required; + } + + public MetricTransform(MetricTransform other) { + this.name = other.name; + this.copyNr = other.copyNr; + this.required = other.required; + } +} diff --git a/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetrics.java b/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetrics.java new file mode 100644 index 00000000000..efc5ab56277 --- /dev/null +++ b/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetrics.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.transformsmetrics; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.Const; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.RowDataUtil; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.engine.EngineComponent.ComponentExecutionStatus; +import org.apache.hop.pipeline.engine.IEngineComponent; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; + +/** + * Wait for selected sibling transforms to finish, then output one metrics row per watched copy. + * This transform does not consume incoming rows. + */ +public class TransformsMetrics extends BaseTransform { + + private static final Class PKG = TransformsMetricsMeta.class; + private static final long POLL_SLEEP_MS = 20L; + + public TransformsMetrics( + TransformMeta transformMeta, + TransformsMetricsMeta meta, + TransformsMetricsData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + @Override + public boolean processRow() throws HopException { + if (first) { + first = false; + initializeWatchedTransforms(); + } + + List remaining = data.getRemaining(); + while (!remaining.isEmpty() && !isStopped()) { + boolean waiting = false; + Iterator iterator = remaining.iterator(); + while (iterator.hasNext()) { + IEngineComponent component = iterator.next(); + if (isStillRunning(component)) { + waiting = true; + } else { + iterator.remove(); + if (isDetailed()) { + logDetailed( + BaseMessages.getString( + PKG, + "TransformsMetrics.Log.Finished", + component.getName(), + String.valueOf(component.getCopyNr()))); + } + emitMetricsRow(component); + } + } + if (waiting && !remaining.isEmpty() && !isStopped()) { + try { + TimeUnit.MILLISECONDS.sleep(POLL_SLEEP_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + setStopped(true); + } + } + } + + setOutputDone(); + return false; + } + + private void initializeWatchedTransforms() throws HopException { + if (meta.getMetricTransforms() == null || meta.getMetricTransforms().isEmpty()) { + throw new HopException(BaseMessages.getString(PKG, "TransformsMetrics.Error.NotTransforms")); + } + + data.setTransformNameField(resolve(meta.getTransformNameField())); + data.setTransformIdField(resolve(meta.getTransformIdField())); + data.setLinesInputField(resolve(meta.getLinesInputField())); + data.setLinesOutputField(resolve(meta.getLinesOutputField())); + data.setLinesReadField(resolve(meta.getLinesReadField())); + data.setLinesUpdatedField(resolve(meta.getLinesUpdatedField())); + data.setLinesWrittenField(resolve(meta.getLinesWrittenField())); + data.setLinesRejectedField(resolve(meta.getLinesRejectedField())); + data.setDurationField(resolve(meta.getDurationField())); + + String[] nextTransforms = getPipelineMeta().getNextTransformNames(getTransformMeta()); + List remaining = new ArrayList<>(); + + for (MetricTransform metricTransform : meta.getMetricTransforms()) { + String name = metricTransform.getName(); + if (StringUtils.isEmpty(name)) { + continue; + } + if (name.equals(getTransformName())) { + throw new HopException( + BaseMessages.getString(PKG, "TransformsMetrics.Error.CannotWatchSelf", name)); + } + if (nextTransforms != null) { + for (String nextTransform : nextTransforms) { + if (name.equals(nextTransform)) { + throw new HopException( + BaseMessages.getString( + PKG, "TransformsMetrics.Error.CannotWatchTarget", nextTransform)); + } + } + } + + int copyNr = Const.toInt(resolve(metricTransform.getCopyNr()), 0); + IEngineComponent component = getDispatcher().findComponent(name, copyNr); + if (component == null) { + if (metricTransform.isRequired()) { + throw new HopException( + BaseMessages.getString( + PKG, "TransformsMetrics.Error.TransformNotFound", name, String.valueOf(copyNr))); + } + continue; + } + remaining.add(component); + } + + data.setRemaining(remaining); + + data.setOutputRowMeta(new RowMeta()); + meta.getFields( + data.getOutputRowMeta(), getTransformName(), null, null, this, getMetadataProvider()); + } + + private boolean isStillRunning(IEngineComponent component) { + ComponentExecutionStatus status = component.getStatus(); + return status == ComponentExecutionStatus.STATUS_RUNNING + || status == ComponentExecutionStatus.STATUS_IDLE + || status == ComponentExecutionStatus.STATUS_INIT + || status == ComponentExecutionStatus.STATUS_PAUSED; + } + + private void emitMetricsRow(IEngineComponent component) throws HopException { + Object[] rowData = RowDataUtil.allocateRowData(data.getOutputRowMeta().size()); + incrementLinesRead(); + + int index = 0; + if (StringUtils.isNotBlank(data.getTransformNameField())) { + rowData[index++] = component.getName(); + } + if (StringUtils.isNotBlank(data.getTransformIdField())) { + rowData[index++] = pluginIdOf(component); + } + if (StringUtils.isNotBlank(data.getLinesInputField())) { + rowData[index++] = component.getLinesInput(); + } + if (StringUtils.isNotBlank(data.getLinesOutputField())) { + rowData[index++] = component.getLinesOutput(); + } + if (StringUtils.isNotBlank(data.getLinesReadField())) { + rowData[index++] = component.getLinesRead(); + } + if (StringUtils.isNotBlank(data.getLinesUpdatedField())) { + rowData[index++] = component.getLinesUpdated(); + } + if (StringUtils.isNotBlank(data.getLinesWrittenField())) { + rowData[index++] = component.getLinesWritten(); + } + if (StringUtils.isNotBlank(data.getLinesRejectedField())) { + rowData[index++] = component.getLinesRejected(); + } + if (StringUtils.isNotBlank(data.getDurationField())) { + rowData[index] = component.getExecutionDuration(); + } + + putRow(data.getOutputRowMeta(), rowData); + } + + private String pluginIdOf(IEngineComponent component) { + TransformMeta watched = getPipelineMeta().findTransform(component.getName()); + return watched == null ? null : watched.getTransformPluginId(); + } +} diff --git a/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetricsData.java b/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetricsData.java new file mode 100644 index 00000000000..4019a0422bb --- /dev/null +++ b/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetricsData.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.transformsmetrics; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.pipeline.engine.IEngineComponent; +import org.apache.hop.pipeline.transform.BaseTransformData; +import org.apache.hop.pipeline.transform.ITransformData; + +@Getter +@Setter +public class TransformsMetricsData extends BaseTransformData implements ITransformData { + + private List remaining = new ArrayList<>(); + private IRowMeta outputRowMeta; + + private String transformNameField; + private String transformIdField; + private String linesInputField; + private String linesOutputField; + private String linesReadField; + private String linesUpdatedField; + private String linesWrittenField; + private String linesRejectedField; + private String durationField; +} diff --git a/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetricsDialog.java b/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetricsDialog.java new file mode 100644 index 00000000000..ad0914119da --- /dev/null +++ b/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetricsDialog.java @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.transformsmetrics; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.Const; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.gui.GuiCompositeWidgets; +import org.apache.hop.ui.core.gui.GuiCompositeWidgetsAdapter; +import org.apache.hop.ui.core.widget.ColumnInfo; +import org.apache.hop.ui.core.widget.TableView; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.Table; +import org.eclipse.swt.widgets.TableItem; + +public class TransformsMetricsDialog extends BaseTransformDialog { + private static final Class PKG = TransformsMetricsMeta.class; + + private final TransformsMetricsMeta input; + private GuiCompositeWidgets widgets; + private TableView wFields; + private String[] previousTransforms; + + public TransformsMetricsDialog( + Shell parent, + IVariables variables, + TransformsMetricsMeta transformMeta, + PipelineMeta pipelineMeta) { + super(parent, variables, transformMeta, pipelineMeta); + input = transformMeta; + } + + @Override + public String open() { + createShell(BaseMessages.getString(PKG, "TransformsMetricsDialog.Shell.Title")); + + changed = input.hasChanged(); + + buildButtonBar().ok(e -> ok()).get(e -> get()).cancel(e -> cancel()).build(); + + setTransformNames(); + + Composite area = new Composite(shell, SWT.NONE); + PropsUi.setLook(area); + area.setLayout(new FormLayout()); + FormData fdArea = new FormData(); + fdArea.left = new FormAttachment(0, 0); + fdArea.top = new FormAttachment(wSpacer, margin); + fdArea.right = new FormAttachment(100, 0); + fdArea.bottom = new FormAttachment(wOk, -2 * margin); + area.setLayoutData(fdArea); + + widgets = new GuiCompositeWidgets(variables); + widgets.registerExtraGroup( + BaseMessages.getString(PKG, "TransformsMetricsMeta.Group.Transforms"), + "10", + null, + this::addTransformsTable); + widgets.setWidgetsListener( + new GuiCompositeWidgetsAdapter() { + @Override + public void widgetModified( + GuiCompositeWidgets compositeWidgets, Control changedWidget, String widgetId) { + input.setChanged(); + } + }); + widgets.createCompositeWidgets( + input, null, area, TransformsMetricsMeta.GUI_PLUGIN_ELEMENT_PARENT_ID, null); + + getData(); + input.setChanged(changed); + focusTransformName(); + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + return transformName; + } + + private void addTransformsTable(Composite parent) { + Label wlFields = new Label(parent, SWT.NONE); + wlFields.setText(BaseMessages.getString(PKG, "TransformsMetricsDialog.Fields.Label")); + PropsUi.setLook(wlFields); + FormData fdlFields = new FormData(); + fdlFields.left = new FormAttachment(0, 0); + fdlFields.top = new FormAttachment(0, 0); + wlFields.setLayoutData(fdlFields); + + String[] yesNo = + new String[] { + BaseMessages.getString(PKG, "System.Combo.No"), + BaseMessages.getString(PKG, "System.Combo.Yes") + }; + + ColumnInfo[] colinf = new ColumnInfo[3]; + colinf[0] = + new ColumnInfo( + BaseMessages.getString(PKG, "TransformsMetricsDialog.Fieldname.Transform"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + previousTransforms, + false); + colinf[1] = + new ColumnInfo( + BaseMessages.getString(PKG, "TransformsMetricsDialog.Fieldname.CopyNr"), + ColumnInfo.COLUMN_TYPE_TEXT, + false); + colinf[1].setUsingVariables(true); + colinf[2] = + new ColumnInfo( + BaseMessages.getString(PKG, "TransformsMetricsDialog.Fieldname.Required"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + yesNo); + + wFields = + new TableView( + variables, + parent, + SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI, + colinf, + input.getMetricTransforms() == null ? 0 : input.getMetricTransforms().size(), + e -> input.setChanged(), + props); + + FormData fdFields = new FormData(); + fdFields.left = new FormAttachment(0, 0); + fdFields.top = new FormAttachment(wlFields, margin); + fdFields.right = new FormAttachment(100, 0); + fdFields.bottom = new FormAttachment(100, 0); + wFields.setLayoutData(fdFields); + } + + private void setTransformNames() { + previousTransforms = pipelineMeta.getTransformNames(); + List nextTransforms = getNextTransforms(new ArrayList<>(), transformMeta); + + List entries = new ArrayList<>(); + for (String previousTransform : previousTransforms) { + if (!previousTransform.equals(transformName) && !nextTransforms.contains(previousTransform)) { + entries.add(previousTransform); + } + } + previousTransforms = entries.toArray(new String[0]); + } + + private List getNextTransforms(List transformNames, TransformMeta transformMeta) { + List nextTransformMeta = pipelineMeta.findNextTransforms(transformMeta); + for (TransformMeta nextTransform : nextTransformMeta) { + transformNames.add(nextTransform.getName()); + getNextTransforms(transformNames, nextTransform); + } + return transformNames.stream().distinct().toList(); + } + + private void get() { + wFields.removeAll(); + Table table = wFields.table; + for (int i = 0; i < previousTransforms.length; i++) { + TableItem ti = new TableItem(table, SWT.NONE); + ti.setText(0, "" + (i + 1)); + ti.setText(1, previousTransforms[i]); + ti.setText(2, "0"); + ti.setText(3, BaseMessages.getString(PKG, "System.Combo.No")); + } + wFields.removeEmptyRows(); + wFields.setRowNums(); + wFields.optWidth(true); + input.setChanged(); + } + + private void getData() { + widgets.setWidgetsContents(input, shell, TransformsMetricsMeta.GUI_PLUGIN_ELEMENT_PARENT_ID); + + Table table = wFields.table; + if (input.getMetricTransforms() != null && !input.getMetricTransforms().isEmpty()) { + table.removeAll(); + } + if (input.getMetricTransforms() != null) { + for (MetricTransform metricTransform : input.getMetricTransforms()) { + TableItem ti = new TableItem(table, SWT.NONE); + ti.setText(1, Const.NVL(metricTransform.getName(), "")); + ti.setText(2, Const.NVL(metricTransform.getCopyNr(), "")); + ti.setText( + 3, + metricTransform.isRequired() + ? BaseMessages.getString(PKG, "System.Combo.Yes") + : BaseMessages.getString(PKG, "System.Combo.No")); + } + } + wFields.removeEmptyRows(); + wFields.setRowNums(); + wFields.optWidth(true); + } + + private void cancel() { + transformName = null; + input.setChanged(changed); + dispose(); + } + + private void ok() { + if (Utils.isEmpty(wTransformName.getText())) { + return; + } + + widgets.getWidgetsContents(input, TransformsMetricsMeta.GUI_PLUGIN_ELEMENT_PARENT_ID); + transformName = wTransformName.getText(); + + if (input.getMetricTransforms() == null) { + input.setMetricTransforms(new ArrayList<>()); + } else { + input.getMetricTransforms().clear(); + } + String yes = BaseMessages.getString(PKG, "System.Combo.Yes"); + for (TableItem item : wFields.getNonEmptyItems()) { + String name = item.getText(1); + if (Utils.isEmpty(name)) { + continue; + } + input + .getMetricTransforms() + .add(new MetricTransform(name, item.getText(2), yes.equalsIgnoreCase(item.getText(3)))); + } + input.setChanged(); + dispose(); + } +} diff --git a/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetricsMeta.java b/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetricsMeta.java new file mode 100644 index 00000000000..2a1177628e0 --- /dev/null +++ b/plugins/transforms/transformsmetrics/src/main/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetricsMeta.java @@ -0,0 +1,358 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.transformsmetrics; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import lombok.Setter; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.CheckResult; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.gui.plugin.GuiElementType; +import org.apache.hop.core.gui.plugin.GuiPlugin; +import org.apache.hop.core.gui.plugin.GuiWidgetElement; +import org.apache.hop.core.gui.plugin.GuiWidgetGroupType; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.PipelineMeta.PipelineType; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; + +@Getter +@Setter +@Transform( + id = "TransformsMetrics,StepsMetrics", + image = "transformsmetrics.svg", + name = "i18n::TransformsMetrics.Name", + description = "i18n::TransformsMetrics.Description", + categoryDescription = + "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Statistics", + keywords = "i18n::TransformsMetricsMeta.keyword", + documentationUrl = "/pipeline/transforms/outputtransformmetrics.html") +@GuiPlugin +public class TransformsMetricsMeta + extends BaseTransformMeta { + private static final Class PKG = TransformsMetricsMeta.class; + + public static final String GUI_PLUGIN_ELEMENT_PARENT_ID = "TRANSFORMS_METRICS_DIALOG_OPTIONS"; + public static final String GROUP_FIELDS = "i18n::TransformsMetricsMeta.Group.Fields"; + public static final String WIDGET_TRANSFORM_NAME_FIELD = "TRANSFORM_NAME_FIELD"; + public static final String WIDGET_TRANSFORM_ID_FIELD = "TRANSFORM_ID_FIELD"; + public static final String WIDGET_LINES_INPUT_FIELD = "LINES_INPUT_FIELD"; + public static final String WIDGET_LINES_OUTPUT_FIELD = "LINES_OUTPUT_FIELD"; + public static final String WIDGET_LINES_READ_FIELD = "LINES_READ_FIELD"; + public static final String WIDGET_LINES_UPDATED_FIELD = "LINES_UPDATED_FIELD"; + public static final String WIDGET_LINES_WRITTEN_FIELD = "LINES_WRITTEN_FIELD"; + public static final String WIDGET_LINES_REJECTED_FIELD = "LINES_REJECTED_FIELD"; + public static final String WIDGET_DURATION_FIELD = "DURATION_FIELD"; + + public static final String DEFAULT_TRANSFORM_NAME_FIELD = "Transform name"; + public static final String DEFAULT_TRANSFORM_ID_FIELD = "Transform id"; + public static final String DEFAULT_LINES_INPUT_FIELD = "Lines input"; + public static final String DEFAULT_LINES_OUTPUT_FIELD = "Lines output"; + public static final String DEFAULT_LINES_READ_FIELD = "Lines read"; + public static final String DEFAULT_LINES_UPDATED_FIELD = "Lines updated"; + public static final String DEFAULT_LINES_WRITTEN_FIELD = "Lines written"; + public static final String DEFAULT_LINES_REJECTED_FIELD = "Lines rejected"; + public static final String DEFAULT_DURATION_FIELD = "Duration"; + + @HopMetadataProperty( + groupKey = "transforms", + key = "transform", + injectionGroupKey = "TRANSFORMS", + injectionGroupDescription = "TransformsMetricsMeta.Injection.TRANSFORMS") + private List metricTransforms; + + @GuiWidgetElement( + id = WIDGET_TRANSFORM_NAME_FIELD, + order = "0100", + type = GuiElementType.TEXT, + label = "i18n::TransformsMetricsMeta.TransformNameField.Label", + toolTip = "i18n::TransformsMetricsMeta.TransformNameField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + group = GROUP_FIELDS, + groupOrder = "20", + groupType = GuiWidgetGroupType.TABS) + @HopMetadataProperty( + key = "transformnamefield", + injectionKey = "TRANSFORM_NAME_FIELD", + injectionKeyDescription = "TransformsMetricsMeta.Injection.TRANSFORM_NAME_FIELD") + private String transformNameField; + + @GuiWidgetElement( + id = WIDGET_TRANSFORM_ID_FIELD, + order = "0200", + type = GuiElementType.TEXT, + label = "i18n::TransformsMetricsMeta.TransformIdField.Label", + toolTip = "i18n::TransformsMetricsMeta.TransformIdField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + group = GROUP_FIELDS, + groupOrder = "20", + groupType = GuiWidgetGroupType.TABS) + @HopMetadataProperty( + key = "transformidfield", + injectionKey = "TRANSFORM_ID_FIELD", + injectionKeyDescription = "TransformsMetricsMeta.Injection.TRANSFORM_ID_FIELD") + private String transformIdField; + + @GuiWidgetElement( + id = WIDGET_LINES_INPUT_FIELD, + order = "0300", + type = GuiElementType.TEXT, + label = "i18n::TransformsMetricsMeta.LinesInputField.Label", + toolTip = "i18n::TransformsMetricsMeta.LinesInputField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + group = GROUP_FIELDS, + groupOrder = "20", + groupType = GuiWidgetGroupType.TABS) + @HopMetadataProperty( + key = "transformlinesinputfield", + injectionKey = "LINES_INPUT_FIELD", + injectionKeyDescription = "TransformsMetricsMeta.Injection.LINES_INPUT_FIELD") + private String linesInputField; + + @GuiWidgetElement( + id = WIDGET_LINES_OUTPUT_FIELD, + order = "0400", + type = GuiElementType.TEXT, + label = "i18n::TransformsMetricsMeta.LinesOutputField.Label", + toolTip = "i18n::TransformsMetricsMeta.LinesOutputField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + group = GROUP_FIELDS, + groupOrder = "20", + groupType = GuiWidgetGroupType.TABS) + @HopMetadataProperty( + key = "transformlinesoutputfield", + injectionKey = "LINES_OUTPUT_FIELD", + injectionKeyDescription = "TransformsMetricsMeta.Injection.LINES_OUTPUT_FIELD") + private String linesOutputField; + + @GuiWidgetElement( + id = WIDGET_LINES_READ_FIELD, + order = "0500", + type = GuiElementType.TEXT, + label = "i18n::TransformsMetricsMeta.LinesReadField.Label", + toolTip = "i18n::TransformsMetricsMeta.LinesReadField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + group = GROUP_FIELDS, + groupOrder = "20", + groupType = GuiWidgetGroupType.TABS) + @HopMetadataProperty( + key = "transformlinesreadfield", + injectionKey = "LINES_READ_FIELD", + injectionKeyDescription = "TransformsMetricsMeta.Injection.LINES_READ_FIELD") + private String linesReadField; + + @GuiWidgetElement( + id = WIDGET_LINES_UPDATED_FIELD, + order = "0600", + type = GuiElementType.TEXT, + label = "i18n::TransformsMetricsMeta.LinesUpdatedField.Label", + toolTip = "i18n::TransformsMetricsMeta.LinesUpdatedField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + group = GROUP_FIELDS, + groupOrder = "20", + groupType = GuiWidgetGroupType.TABS) + @HopMetadataProperty( + key = "transformlinesupdatedfield", + injectionKey = "LINES_UPDATED_FIELD", + injectionKeyDescription = "TransformsMetricsMeta.Injection.LINES_UPDATED_FIELD") + private String linesUpdatedField; + + @GuiWidgetElement( + id = WIDGET_LINES_WRITTEN_FIELD, + order = "0700", + type = GuiElementType.TEXT, + label = "i18n::TransformsMetricsMeta.LinesWrittenField.Label", + toolTip = "i18n::TransformsMetricsMeta.LinesWrittenField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + group = GROUP_FIELDS, + groupOrder = "20", + groupType = GuiWidgetGroupType.TABS) + @HopMetadataProperty( + key = "transformlineswrittenfield", + injectionKey = "LINES_WRITTEN_FIELD", + injectionKeyDescription = "TransformsMetricsMeta.Injection.LINES_WRITTEN_FIELD") + private String linesWrittenField; + + @GuiWidgetElement( + id = WIDGET_LINES_REJECTED_FIELD, + order = "0800", + type = GuiElementType.TEXT, + label = "i18n::TransformsMetricsMeta.LinesRejectedField.Label", + toolTip = "i18n::TransformsMetricsMeta.LinesRejectedField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + group = GROUP_FIELDS, + groupOrder = "20", + groupType = GuiWidgetGroupType.TABS) + @HopMetadataProperty( + key = "transformlineserrorsfield", + injectionKey = "LINES_REJECTED_FIELD", + injectionKeyDescription = "TransformsMetricsMeta.Injection.LINES_REJECTED_FIELD") + private String linesRejectedField; + + @GuiWidgetElement( + id = WIDGET_DURATION_FIELD, + order = "0900", + type = GuiElementType.TEXT, + label = "i18n::TransformsMetricsMeta.DurationField.Label", + toolTip = "i18n::TransformsMetricsMeta.DurationField.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + group = GROUP_FIELDS, + groupOrder = "20", + groupType = GuiWidgetGroupType.TABS) + @HopMetadataProperty( + key = "transformsecondsfield", + injectionKey = "DURATION_FIELD", + injectionKeyDescription = "TransformsMetricsMeta.Injection.DURATION_FIELD") + private String durationField; + + public TransformsMetricsMeta() { + metricTransforms = new ArrayList<>(); + } + + @Override + public void setDefault() { + metricTransforms = new ArrayList<>(); + transformNameField = DEFAULT_TRANSFORM_NAME_FIELD; + transformIdField = DEFAULT_TRANSFORM_ID_FIELD; + linesInputField = DEFAULT_LINES_INPUT_FIELD; + linesOutputField = DEFAULT_LINES_OUTPUT_FIELD; + linesReadField = DEFAULT_LINES_READ_FIELD; + linesUpdatedField = DEFAULT_LINES_UPDATED_FIELD; + linesWrittenField = DEFAULT_LINES_WRITTEN_FIELD; + linesRejectedField = DEFAULT_LINES_REJECTED_FIELD; + durationField = DEFAULT_DURATION_FIELD; + } + + @Override + public void getFields( + IRowMeta inputRowMeta, + String name, + IRowMeta[] info, + TransformMeta nextTransform, + IVariables variables, + IHopMetadataProvider metadataProvider) + throws HopTransformException { + inputRowMeta.clear(); + addStringField(inputRowMeta, variables.resolve(transformNameField), name); + addStringField(inputRowMeta, variables.resolve(transformIdField), name); + addIntegerField(inputRowMeta, variables.resolve(linesInputField), name); + addIntegerField(inputRowMeta, variables.resolve(linesOutputField), name); + addIntegerField(inputRowMeta, variables.resolve(linesReadField), name); + addIntegerField(inputRowMeta, variables.resolve(linesUpdatedField), name); + addIntegerField(inputRowMeta, variables.resolve(linesWrittenField), name); + addIntegerField(inputRowMeta, variables.resolve(linesRejectedField), name); + addIntegerField(inputRowMeta, variables.resolve(durationField), name); + } + + private void addStringField(IRowMeta rowMeta, String fieldName, String origin) { + if (StringUtils.isBlank(fieldName)) { + return; + } + ValueMetaString valueMeta = new ValueMetaString(fieldName); + valueMeta.setOrigin(origin); + rowMeta.addValueMeta(valueMeta); + } + + private void addIntegerField(IRowMeta rowMeta, String fieldName, String origin) { + if (StringUtils.isBlank(fieldName)) { + return; + } + ValueMetaInteger valueMeta = + new ValueMetaInteger(fieldName, IValueMeta.DEFAULT_INTEGER_LENGTH, 0); + valueMeta.setOrigin(origin); + rowMeta.addValueMeta(valueMeta); + } + + @Override + public void check( + List remarks, + PipelineMeta pipelineMeta, + TransformMeta transformMeta, + IRowMeta prev, + String[] input, + String[] output, + IRowMeta info, + IVariables variables, + IHopMetadataProvider metadataProvider) { + if (input.length > 0) { + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString( + PKG, "TransformsMetricsMeta.CheckResult.IncomingHopsNotSupported"), + transformMeta)); + } else { + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString(PKG, "TransformsMetricsMeta.CheckResult.NoIncomingHops"), + transformMeta)); + } + + if (metricTransforms == null || metricTransforms.isEmpty()) { + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_ERROR, + BaseMessages.getString(PKG, "TransformsMetricsMeta.CheckResult.NoTransformsEntered"), + transformMeta)); + return; + } + + boolean allFound = true; + for (MetricTransform metricTransform : metricTransforms) { + if (StringUtils.isEmpty(metricTransform.getName())) { + continue; + } + if (pipelineMeta.findTransform(metricTransform.getName()) == null) { + allFound = false; + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_WARNING, + BaseMessages.getString( + PKG, + "TransformsMetricsMeta.CheckResult.TransformNotFound", + metricTransform.getName()), + transformMeta)); + } + } + if (allFound) { + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_OK, + BaseMessages.getString(PKG, "TransformsMetricsMeta.CheckResult.AllTransformsFound"), + transformMeta)); + } + } + + @Override + public PipelineType[] getSupportedPipelineTypes() { + return new PipelineType[] {PipelineType.Normal}; + } +} diff --git a/plugins/transforms/transformsmetrics/src/main/resources/org/apache/hop/pipeline/transforms/transformsmetrics/messages/messages_en_US.properties b/plugins/transforms/transformsmetrics/src/main/resources/org/apache/hop/pipeline/transforms/transformsmetrics/messages/messages_en_US.properties new file mode 100644 index 00000000000..3e4f3ff9b39 --- /dev/null +++ b/plugins/transforms/transformsmetrics/src/main/resources/org/apache/hop/pipeline/transforms/transformsmetrics/messages/messages_en_US.properties @@ -0,0 +1,68 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +TransformsMetrics.Name=Output transform metrics +TransformsMetrics.Description=Wait for selected transforms to finish and output their metrics +TransformsMetricsMeta.keyword=metrics,statistics,wait,transform,output +TransformsMetrics.Error.NotTransforms=You must define at least one transform for which metrics will be returned. +TransformsMetrics.Error.CannotWatchSelf=You can not get metrics for the current transform [{0}]. +TransformsMetrics.Error.CannotWatchTarget=You can not get metrics for the target transform [{0}]. +TransformsMetrics.Error.TransformNotFound=We cannot get transform [{0}] copyNr={1}. +TransformsMetrics.Log.Finished=Finished running transform [{0} ({1})]. +TransformsMetricsDialog.Shell.Title=Output transform metrics +TransformsMetricsDialog.Fields.Label=Metrics for the following transforms +TransformsMetricsDialog.Fieldname.Transform=Transform name +TransformsMetricsDialog.Fieldname.CopyNr=Copy Nr +TransformsMetricsDialog.Fieldname.Required=Required +TransformsMetricsMeta.Group.Transforms=Transforms +TransformsMetricsMeta.Group.Fields=Fields +TransformsMetricsMeta.TransformNameField.Label=Transform name +TransformsMetricsMeta.TransformNameField.Tooltip=Output field for the watched transform name. Leave empty to omit this field. +TransformsMetricsMeta.TransformIdField.Label=Transform id +TransformsMetricsMeta.TransformIdField.Tooltip=Output field for the watched transform plugin id. Leave empty to omit this field. +TransformsMetricsMeta.LinesInputField.Label=Lines input +TransformsMetricsMeta.LinesInputField.Tooltip=Output field for lines input. Leave empty to omit this field. +TransformsMetricsMeta.LinesOutputField.Label=Lines output +TransformsMetricsMeta.LinesOutputField.Tooltip=Output field for lines output. Leave empty to omit this field. +TransformsMetricsMeta.LinesReadField.Label=Lines read +TransformsMetricsMeta.LinesReadField.Tooltip=Output field for lines read. Leave empty to omit this field. +TransformsMetricsMeta.LinesUpdatedField.Label=Lines updated +TransformsMetricsMeta.LinesUpdatedField.Tooltip=Output field for lines updated. Leave empty to omit this field. +TransformsMetricsMeta.LinesWrittenField.Label=Lines written +TransformsMetricsMeta.LinesWrittenField.Tooltip=Output field for lines written. Leave empty to omit this field. +TransformsMetricsMeta.LinesRejectedField.Label=Lines rejected +TransformsMetricsMeta.LinesRejectedField.Tooltip=Output field for lines rejected (error hop). Leave empty to omit this field. +TransformsMetricsMeta.DurationField.Label=Duration (ms) +TransformsMetricsMeta.DurationField.Tooltip=Output field for execution duration in milliseconds. Leave empty to omit this field. +TransformsMetricsMeta.CheckResult.IncomingHopsNotSupported=Incoming hops are not supported and would deadlock this transform. Remove them. +TransformsMetricsMeta.CheckResult.NoIncomingHops=Not receiving input from other transforms. +TransformsMetricsMeta.CheckResult.NoTransformsEntered=No transforms are entered. +TransformsMetricsMeta.CheckResult.TransformNotFound=Watched transform [{0}] was not found in the pipeline. +TransformsMetricsMeta.CheckResult.AllTransformsFound=All watched transforms were found. +TransformsMetricsMeta.Injection.TRANSFORMS=Transforms to watch +TransformsMetricsMeta.Injection.TRANSFORM_NAME=Transform name +TransformsMetricsMeta.Injection.TRANSFORM_COPY_NR=Copy number +TransformsMetricsMeta.Injection.TRANSFORM_REQUIRED=Required +TransformsMetricsMeta.Injection.TRANSFORM_NAME_FIELD=Transform name field +TransformsMetricsMeta.Injection.TRANSFORM_ID_FIELD=Transform id field +TransformsMetricsMeta.Injection.LINES_INPUT_FIELD=Lines input field +TransformsMetricsMeta.Injection.LINES_OUTPUT_FIELD=Lines output field +TransformsMetricsMeta.Injection.LINES_READ_FIELD=Lines read field +TransformsMetricsMeta.Injection.LINES_UPDATED_FIELD=Lines updated field +TransformsMetricsMeta.Injection.LINES_WRITTEN_FIELD=Lines written field +TransformsMetricsMeta.Injection.LINES_REJECTED_FIELD=Lines rejected field +TransformsMetricsMeta.Injection.DURATION_FIELD=Duration field diff --git a/plugins/transforms/transformsmetrics/src/main/resources/transformsmetrics.svg b/plugins/transforms/transformsmetrics/src/main/resources/transformsmetrics.svg new file mode 100644 index 00000000000..6829085d350 --- /dev/null +++ b/plugins/transforms/transformsmetrics/src/main/resources/transformsmetrics.svg @@ -0,0 +1,23 @@ + + + + + + + diff --git a/plugins/transforms/transformsmetrics/src/main/resources/version.xml b/plugins/transforms/transformsmetrics/src/main/resources/version.xml new file mode 100644 index 00000000000..36ab20e22eb --- /dev/null +++ b/plugins/transforms/transformsmetrics/src/main/resources/version.xml @@ -0,0 +1,20 @@ + + + +${project.version} diff --git a/plugins/transforms/transformsmetrics/src/main/samples/transforms/output-transform-metrics-basic.hpl b/plugins/transforms/transformsmetrics/src/main/samples/transforms/output-transform-metrics-basic.hpl new file mode 100644 index 00000000000..1fa7eb8363d --- /dev/null +++ b/plugins/transforms/transformsmetrics/src/main/samples/transforms/output-transform-metrics-basic.hpl @@ -0,0 +1,201 @@ + + + + + output-transform-metrics-basic + Y + Wait for sibling transforms to finish and output their metrics + + + Normal + + + N + 1000 + 100 + - + 2026/08/29 12:00:00.000 + - + 2026/08/29 12:00:00.000 + + N + + + + + + 10 rows + Dummy A + Y + + + 5 rows + Dummy B + Y + + + + 10 rows + RowGenerator + + Y + + 1 + + none + + + + + value + Integer + # + + + + 1 + -1 + -1 + N + + + 10 + N + 5000 + now + FiveSecondsAgo + + + 80 + 80 + + + + 5 rows + RowGenerator + + Y + + 1 + + none + + + + + value + Integer + # + + + + 1 + -1 + -1 + N + + + 5 + N + 5000 + now + FiveSecondsAgo + + + 80 + 224 + + + + Dummy A + Dummy + + Y + + 1 + + none + + + + + 256 + 80 + + + + Dummy B + Dummy + + Y + + 1 + + none + + + + + 256 + 224 + + + + Output transform metrics + TransformsMetrics + + Y + + 1 + + none + + + + + Dummy A + 0 + Y + + + Dummy B + 0 + Y + + + Transform name + Transform id + Lines input + Lines output + Lines read + Lines updated + Lines written + Lines rejected + Duration + + + 448 + 152 + + + + + + diff --git a/plugins/transforms/transformsmetrics/src/test/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetricsMetaTest.java b/plugins/transforms/transformsmetrics/src/test/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetricsMetaTest.java new file mode 100644 index 00000000000..7231bbb5a44 --- /dev/null +++ b/plugins/transforms/transformsmetrics/src/test/java/org/apache/hop/pipeline/transforms/transformsmetrics/TransformsMetricsMetaTest.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.transformsmetrics; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.HopClientEnvironment; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.xml.XmlHandler; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +class TransformsMetricsMetaTest { + + @BeforeAll + static void beforeAll() throws Exception { + HopClientEnvironment.init(); + } + + @Test + void testXmlRoundTrip() throws Exception { + TransformsMetricsMeta meta = new TransformsMetricsMeta(); + meta.setDefault(); + meta.getMetricTransforms().add(new MetricTransform("A", "0", true)); + meta.getMetricTransforms().add(new MetricTransform("B", "1", false)); + meta.setTransformNameField("Transform name"); + meta.setLinesWrittenField("Lines written"); + meta.setDurationField(""); + + String transformXml = + XmlHandler.openTag(TransformMeta.XML_TAG) + + meta.getXml() + + XmlHandler.closeTag(TransformMeta.XML_TAG); + TransformsMetricsMeta loaded = new TransformsMetricsMeta(); + loaded.loadXml(XmlHandler.loadXmlString(transformXml, TransformMeta.XML_TAG), null); + + assertEquals(2, loaded.getMetricTransforms().size()); + assertEquals("A", loaded.getMetricTransforms().get(0).getName()); + assertEquals("0", loaded.getMetricTransforms().get(0).getCopyNr()); + assertTrue(loaded.getMetricTransforms().get(0).isRequired()); + assertEquals("B", loaded.getMetricTransforms().get(1).getName()); + assertFalse(loaded.getMetricTransforms().get(1).isRequired()); + assertEquals("Transform name", loaded.getTransformNameField()); + assertEquals("Lines written", loaded.getLinesWrittenField()); + assertTrue(StringUtils.isBlank(loaded.getDurationField())); + assertTrue(transformXml.contains("")); + assertTrue(transformXml.contains("")); + assertTrue(transformXml.contains("")); + } + + @Test + void testGetFieldsClearsIncomingAndOmitsEmptyNames() throws Exception { + TransformsMetricsMeta meta = new TransformsMetricsMeta(); + meta.setDefault(); + meta.setTransformIdField(""); + meta.setLinesInputField(null); + meta.setDurationField(" "); + + IRowMeta rowMeta = new RowMeta(); + rowMeta.addValueMeta(new ValueMetaString("incoming")); + meta.getFields(rowMeta, "metrics", null, null, new Variables(), null); + + List names = new ArrayList<>(); + for (int i = 0; i < rowMeta.size(); i++) { + names.add(rowMeta.getValueMeta(i).getName()); + } + assertFalse(names.contains("incoming")); + assertTrue(names.contains(TransformsMetricsMeta.DEFAULT_TRANSFORM_NAME_FIELD)); + assertFalse(names.contains(TransformsMetricsMeta.DEFAULT_TRANSFORM_ID_FIELD)); + assertFalse(names.contains(TransformsMetricsMeta.DEFAULT_LINES_INPUT_FIELD)); + assertTrue(names.contains(TransformsMetricsMeta.DEFAULT_LINES_WRITTEN_FIELD)); + assertEquals(IValueMeta.TYPE_INTEGER, rowMeta.searchValueMeta("Lines written").getType()); + } + + @Test + void testCheckEmptyListAndIncomingHops() { + TransformsMetricsMeta meta = new TransformsMetricsMeta(); + meta.setDefault(); + TransformMeta transformMeta = new TransformMeta(); + transformMeta.setName("metrics"); + PipelineMeta pipelineMeta = new PipelineMeta(); + + List remarks = new ArrayList<>(); + meta.check( + remarks, + pipelineMeta, + transformMeta, + new RowMeta(), + new String[] {"upstream"}, + new String[0], + new RowMeta(), + new Variables(), + null); + assertTrue( + remarks.stream().anyMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_ERROR), + "incoming hops should be an error"); + assertTrue( + remarks.stream().anyMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_ERROR), + "empty watch list should be an error"); + } + + @Test + void testCheckMissingWatchedTransformIsWarning() { + TransformsMetricsMeta meta = new TransformsMetricsMeta(); + meta.setDefault(); + meta.getMetricTransforms().add(new MetricTransform("missing", "0", false)); + TransformMeta transformMeta = new TransformMeta(); + transformMeta.setName("metrics"); + PipelineMeta pipelineMeta = new PipelineMeta(); + + List remarks = new ArrayList<>(); + meta.check( + remarks, + pipelineMeta, + transformMeta, + new RowMeta(), + new String[0], + new String[0], + new RowMeta(), + new Variables(), + null); + assertTrue(remarks.stream().anyMatch(r -> r.getType() == ICheckResult.TYPE_RESULT_WARNING)); + } +}