== Physical Plan ==
AdaptiveSparkPlan (11)
+- == Final Plan ==
ResultQueryStage (7), Statistics(sizeInBytes=8.0 EiB)
+- ObjectHashAggregate (6)
+- ShuffleQueryStage (5), Statistics(sizeInBytes=10.7 KiB, rowCount=50)
+- Exchange (4)
+- ObjectHashAggregate (3)
+- * Project (2)
+- * Scan ExistingRDD Delta Table State #1 - hdlfs://ced12e6d-bca7-47af-9525-449cba9023d7.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-product-productplant/_delta_log (1)
+- == Initial Plan ==
ObjectHashAggregate (10)
+- Exchange (9)
+- ObjectHashAggregate (8)
+- Project (2)
+- Scan ExistingRDD Delta Table State #1 - hdlfs://ced12e6d-bca7-47af-9525-449cba9023d7.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-product-productplant/_delta_log (1)
(1) Scan ExistingRDD Delta Table State #1 - hdlfs://ced12e6d-bca7-47af-9525-449cba9023d7.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-product-productplant/_delta_log [codegen id : 1]
Output [10]: [txn#1226890, add#1226891, remove#1226892, metaData#1226893, protocol#1226894, cdc#1226895, checkpointMetadata#1226896, sidecar#1226897, domainMetadata#1226898, commitInfo#1226899]
Arguments: [txn#1226890, add#1226891, remove#1226892, metaData#1226893, protocol#1226894, cdc#1226895, checkpointMetadata#1226896, sidecar#1226897, domainMetadata#1226898, commitInfo#1226899], Delta Table State #1 - hdlfs://ced12e6d-bca7-47af-9525-449cba9023d7.files.hdl.prod-eu20.hanacloud.ondemand.com:443/crp-dl-stream-service/cornerstone/sap-cic-product-productplant/_delta_log MapPartitionsRDD[70536] at $anonfun$recordDeltaOperationInternal$1 at DatabricksLogging.scala:128, ExistingRDD, UnknownPartitioning(0)
(2) Project [codegen id : 1]
Output [6]: [txn#1226890, add#1226891, remove#1226892, metaData#1226893, protocol#1226894, domainMetadata#1226898]
Input [10]: [txn#1226890, add#1226891, remove#1226892, metaData#1226893, protocol#1226894, cdc#1226895, checkpointMetadata#1226896, sidecar#1226897, domainMetadata#1226898, commitInfo#1226899]
(3) ObjectHashAggregate
Input [6]: [txn#1226890, add#1226891, remove#1226892, metaData#1226893, protocol#1226894, domainMetadata#1226898]
Keys: []
Functions [10]: [partial_collect_set(domainMetadata#1226898, 0, 0), partial_collect_set(txn#1226890, 0, 0), partial_count(protocol#1226894), partial_sum(add#1226891.size), partial_last(metaData#1226893, true), partial_count(metaData#1226893), partial_last(protocol#1226894, true), partial_count(remove#1226892), partial_count(add#1226891), partial_count(txn#1226890)]
Aggregate Attributes [12]: [buf#1227000, buf#1227001, count#1227002L, sum#1227003L, last#1227004, valueSet#1227005, count#1227006L, last#1227007, valueSet#1227008, count#1227009L, count#1227010L, count#1227011L]
Results [12]: [buf#1227012, buf#1227013, count#1227014L, sum#1227015L, last#1227016, valueSet#1227017, count#1227018L, last#1227019, valueSet#1227020, count#1227021L, count#1227022L, count#1227023L]
(4) Exchange
Input [12]: [buf#1227012, buf#1227013, count#1227014L, sum#1227015L, last#1227016, valueSet#1227017, count#1227018L, last#1227019, valueSet#1227020, count#1227021L, count#1227022L, count#1227023L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1921931]
(5) ShuffleQueryStage
Output [12]: [buf#1227012, buf#1227013, count#1227014L, sum#1227015L, last#1227016, valueSet#1227017, count#1227018L, last#1227019, valueSet#1227020, count#1227021L, count#1227022L, count#1227023L]
Arguments: 0
(6) ObjectHashAggregate
Input [12]: [buf#1227012, buf#1227013, count#1227014L, sum#1227015L, last#1227016, valueSet#1227017, count#1227018L, last#1227019, valueSet#1227020, count#1227021L, count#1227022L, count#1227023L]
Keys: []
Functions [10]: [collect_set(domainMetadata#1226898, 0, 0), collect_set(txn#1226890, 0, 0), count(protocol#1226894), sum(add#1226891.size), last(metaData#1226893, true), count(metaData#1226893), last(protocol#1226894, true), count(remove#1226892), count(add#1226891), count(txn#1226890)]
Aggregate Attributes [10]: [collect_set(domainMetadata#1226898, 0, 0)#1226946, collect_set(txn#1226890, 0, 0)#1226947, count(protocol#1226894)#1226948L, sum(add#1226891.size)#1226949L, last(metaData#1226893) ignore nulls#1226950, count(metaData#1226893)#1226951L, last(protocol#1226894) ignore nulls#1226952, count(remove#1226892)#1226953L, count(add#1226891)#1226954L, count(txn#1226890)#1226955L]
Results [14]: [collect_set(domainMetadata#1226898, 0, 0)#1226946 AS domainMetadata#1226931, null AS fileSizeHistogram#1226932, collect_set(txn#1226890, 0, 0)#1226947 AS setTransactions#1226933, null AS numDeletedRecordsOpt#1226934, count(protocol#1226894)#1226948L AS numOfProtocol#1226935L, coalesce(sum(add#1226891.size)#1226949L, 0) AS sizeInBytes#1226936L, last(metaData#1226893) ignore nulls#1226950 AS metadata#1226937, count(metaData#1226893)#1226951L AS numOfMetadata#1226938L, null AS deletedRecordCountsHistogramOpt#1226939, null AS numDeletionVectorsOpt#1226940, last(protocol#1226894) ignore nulls#1226952 AS protocol#1226941, count(remove#1226892)#1226953L AS numOfRemoves#1226942L, count(add#1226891)#1226954L AS numOfFiles#1226943L, count(txn#1226890)#1226955L AS numOfSetTransactions#1226944L]
(7) ResultQueryStage
Output [14]: [domainMetadata#1226931, fileSizeHistogram#1226932, setTransactions#1226933, numDeletedRecordsOpt#1226934, numOfProtocol#1226935L, sizeInBytes#1226936L, metadata#1226937, numOfMetadata#1226938L, deletedRecordCountsHistogramOpt#1226939, numDeletionVectorsOpt#1226940, protocol#1226941, numOfRemoves#1226942L, numOfFiles#1226943L, numOfSetTransactions#1226944L]
Arguments: 1
(8) ObjectHashAggregate
Input [6]: [txn#1226890, add#1226891, remove#1226892, metaData#1226893, protocol#1226894, domainMetadata#1226898]
Keys: []
Functions [10]: [partial_collect_set(domainMetadata#1226898, 0, 0), partial_collect_set(txn#1226890, 0, 0), partial_count(protocol#1226894), partial_sum(add#1226891.size), partial_last(metaData#1226893, true), partial_count(metaData#1226893), partial_last(protocol#1226894, true), partial_count(remove#1226892), partial_count(add#1226891), partial_count(txn#1226890)]
Aggregate Attributes [12]: [buf#1227000, buf#1227001, count#1227002L, sum#1227003L, last#1227004, valueSet#1227005, count#1227006L, last#1227007, valueSet#1227008, count#1227009L, count#1227010L, count#1227011L]
Results [12]: [buf#1227012, buf#1227013, count#1227014L, sum#1227015L, last#1227016, valueSet#1227017, count#1227018L, last#1227019, valueSet#1227020, count#1227021L, count#1227022L, count#1227023L]
(9) Exchange
Input [12]: [buf#1227012, buf#1227013, count#1227014L, sum#1227015L, last#1227016, valueSet#1227017, count#1227018L, last#1227019, valueSet#1227020, count#1227021L, count#1227022L, count#1227023L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=1921920]
(10) ObjectHashAggregate
Input [12]: [buf#1227012, buf#1227013, count#1227014L, sum#1227015L, last#1227016, valueSet#1227017, count#1227018L, last#1227019, valueSet#1227020, count#1227021L, count#1227022L, count#1227023L]
Keys: []
Functions [10]: [collect_set(domainMetadata#1226898, 0, 0), collect_set(txn#1226890, 0, 0), count(protocol#1226894), sum(add#1226891.size), last(metaData#1226893, true), count(metaData#1226893), last(protocol#1226894, true), count(remove#1226892), count(add#1226891), count(txn#1226890)]
Aggregate Attributes [10]: [collect_set(domainMetadata#1226898, 0, 0)#1226946, collect_set(txn#1226890, 0, 0)#1226947, count(protocol#1226894)#1226948L, sum(add#1226891.size)#1226949L, last(metaData#1226893) ignore nulls#1226950, count(metaData#1226893)#1226951L, last(protocol#1226894) ignore nulls#1226952, count(remove#1226892)#1226953L, count(add#1226891)#1226954L, count(txn#1226890)#1226955L]
Results [14]: [collect_set(domainMetadata#1226898, 0, 0)#1226946 AS domainMetadata#1226931, null AS fileSizeHistogram#1226932, collect_set(txn#1226890, 0, 0)#1226947 AS setTransactions#1226933, null AS numDeletedRecordsOpt#1226934, count(protocol#1226894)#1226948L AS numOfProtocol#1226935L, coalesce(sum(add#1226891.size)#1226949L, 0) AS sizeInBytes#1226936L, last(metaData#1226893) ignore nulls#1226950 AS metadata#1226937, count(metaData#1226893)#1226951L AS numOfMetadata#1226938L, null AS deletedRecordCountsHistogramOpt#1226939, null AS numDeletionVectorsOpt#1226940, last(protocol#1226894) ignore nulls#1226952 AS protocol#1226941, count(remove#1226892)#1226953L AS numOfRemoves#1226942L, count(add#1226891)#1226954L AS numOfFiles#1226943L, count(txn#1226890)#1226955L AS numOfSetTransactions#1226944L]
(11) AdaptiveSparkPlan
Output [14]: [domainMetadata#1226931, fileSizeHistogram#1226932, setTransactions#1226933, numDeletedRecordsOpt#1226934, numOfProtocol#1226935L, sizeInBytes#1226936L, metadata#1226937, numOfMetadata#1226938L, deletedRecordCountsHistogramOpt#1226939, numDeletionVectorsOpt#1226940, protocol#1226941, numOfRemoves#1226942L, numOfFiles#1226943L, numOfSetTransactions#1226944L]
Arguments: isFinalPlan=true