Uploading Profile Data
This topic describes how to upload job profile data files in Dremio as a dataset, so you can use SQL for analyzing the profile data. You can either upload your profile_attempt_0.json file or use the sample profile_attempt_0.json file below. For more information, see Uploading files.
To upload the profile data:
- Download your profile_attempt_0.json file or the sample profile_attempt_0.json file.
- For your profile_attempt_0.json file, go to Download Profile.
- For the sample profile_attempt_0.json file, use the profile_attempt_0.json file below.
- Click the Datasets
(grid) icon in the side navigation bar. - Click the Add
(plus) icon in the top-right corner and select Upload File. - Drop a local file into the popup or browse to add the file.
- Click Next and then click Save.
The downloaded profile_attempt_0.json file is now uploaded and in Dremio as a dataset.
Sample: profile_attempt_0
This section provides sample data used for the profile_attempt_0.json file, which can be used in the profile examples to generate a screenshot of the query results.
Sample data for profile examples{"id":
{"part1":2498103721662745869,"part2":-3366896587773423360},"start":1565848550550,"end":1565848550770,"query":"SELECT line_id, count(*)\nFROM dim_order\n--where promise_date <> orig_promise_date\ngroup by line_id\nhaving count(*) > 1\norder by line_id","plan":"00-00 Screen : rowType = RecordType(VARCHAR(65536) Fragment, BIGINT Records, VARCHAR(65536) Path, VARBINARY(65536) Metadata, INTEGER Partition, BIGINT FileSize): rowcount = 1.0, cumulative cost = {31.1 rows, 1066.1000999999999 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15270\n00-01 Project(Fragment=[$0], Records=[$1], Path=[$2], Metadata=[$3], Partition=[$4], FileSize=[$5]) : rowType = RecordType(VARCHAR(65536) Fragment, BIGINT Records, VARCHAR(65536) Path, VARBINARY(65536) Metadata, INTEGER Partition, BIGINT FileSize): rowcount = 1.0, cumulative cost = {31.0 rows, 1066.0001 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15269\n00-02 WriterCommitter(final=[/Users/balajiramaswamy/Downloads/dremio-enterprise-3.3.1-201907291852280797-df23756/data/pdfs/results/22ab0c19-6dca-0d0d-d146-61645a4a5100]) : rowType = RecordType(VARCHAR(65536) Fragment, BIGINT Records, VARCHAR(65536) Path, VARBINARY(65536) Metadata, INTEGER Partition, BIGINT FileSize): rowcount = 1.0, cumulative cost = {30.0 rows, 1066.00004 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15268\n00-03 Writer : rowType = RecordType(VARCHAR(65536) Fragment, BIGINT Records, VARCHAR(65536) Path, VARBINARY(65536) Metadata, INTEGER Partition, BIGINT FileSize): rowcount = 1.0, cumulative cost = {29.0 rows, 1065.00004 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15267\n00-04 Project(line_id=[$0], EXPR$1=[$1]) : rowType = RecordType(INTEGER line_id, BIGINT EXPR$1): rowcount = 1.0, cumulative cost = {28.0 rows, 1064.00004 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15266\n00-05 Project(line_id=[$0], EXPR$1=[$1]) : rowType = RecordType(INTEGER line_id, BIGINT EXPR$1): rowcount = 1.0, cumulative cost = {27.0 rows, 1064.00002 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15265\n00-06 Sort(sort0=[$0], dir0=[ASC]) : rowType = RecordType(INTEGER line_id, BIGINT EXPR$1): rowcount = 1.0, cumulative cost = {26.0 rows, 1064.0 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15264\n00-07 Filter(condition=[>($1, 1)]) : rowType = RecordType(INTEGER line_id, BIGINT EXPR$1): rowcount = 1.0, cumulative cost = {25.0 rows, 1064.0 cpu, 832.0 io, 832.0 network, 64.0 memory}, id = 15263\n00-08 StreamAgg(group=[{0}], EXPR$1=[COUNT()]) : rowType = RecordType(INTEGER line_id, BIGINT EXPR$1): rowcount = 1.0, cumulative cost = {24.0 rows, 1056.0 cpu, 832.0 io, 832.0 network, 64.0 memory}, id = 15262\n00-09 Sort(sort0=[$0], dir0=[ASC]) : rowType = RecordType(INTEGER line_id): rowcount = 8.0, cumulative cost = {16.0 rows, 928.0 cpu, 832.0 io, 832.0 network, 64.0 memory}, id = 15261\n00-10 HiveScan(table=[localhive.\"default\".dim_order], columns=[`line_id`], splits=[4], mode=[BASIC]) : rowType = RecordType(INTEGER line_id): rowcount = 8.0, cumulative cost = {8.0 rows, 832.0 cpu, 832.0 io, 832.0 network, 0.0 memory}, id = 15260\n","foreman":{"address":"localhost","userPort":31010,"fabricPort":45678,"roles":{"sqlQuery":true,"javaExecutor":true,"master":true},"startTime":1565843427321,"maxDirectMemory":8589934592,"availableCores":8},"state":2,"totalFragments":1,"finishedFragments":0,"fragmentProfile":[{"majorFragmentId":0,"minorFragmentProfile":[{"state":3,"minorFragmentId":0,"operatorProfile":[{"inputProfile":[{"records":1,"batches":1,"size":394}],"operatorId":0,"operatorType":13,"setupNanos":3316,"processNanos":286806,"peakLocalMemoryAllocated":229376,"metric":[{"metricId":0,"longValue":394}],"waitNanos":237649},{"inputProfile":[{"records":1,"batches":1,"size":394}],"operatorId":1,"operatorType":10,"setupNanos":401663,"processNanos":61857,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":3,"longValue":0},{"metricId":4,"longValue":0},{"metricId":2,"longValue":0},{"metricId":5,"longValue":0},{"metricId":7,"longValue":0},{"metricId":1,"longValue":0},{"metricId":6,"longValue":0}],"waitNanos":0,"details":{"splitInfos":[]}},{"inputProfile":[{"records":1,"batches":1,"size":394}],"operatorId":2,"operatorType":49,"setupNanos":1261852,"processNanos":99500,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":2,"longValue":0},{"metricId":1,"longValue":0},{"metricId":5,"longValue":0},{"metricId":4,"longValue":0},{"metricId":7,"longValue":0},{"metricId":6,"longValue":0},{"metricId":3,"longValue":0}],"waitNanos":0,"details":{"splitInfos":[]}},{"inputProfile":[{"records":2,"batches":1,"size":26}],"operatorId":3,"operatorType":43,"setupNanos":337587,"processNanos":504465,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":169},{"metricId":1,"longValue":0}],"waitNanos":8005890},{"inputProfile":[{"records":2,"batches":1,"size":26}],"operatorId":4,"operatorType":10,"setupNanos":442082,"processNanos":16306,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":6,"longValue":0},{"metricId":4,"longValue":0},{"metricId":5,"longValue":0},{"metricId":1,"longValue":0},{"metricId":2,"longValue":0},{"metricId":3,"longValue":0},{"metricId":7,"longValue":0}],"waitNanos":0,"details":{"splitInfos":[]}},{"inputProfile":[{"records":2,"batches":1,"size":26}],"operatorId":5,"operatorType":10,"setupNanos":1002489,"processNanos":34216,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":0},{"metricId":5,"longValue":0},{"metricId":2,"longValue":0},{"metricId":7,"longValue":0},{"metricId":6,"longValue":0},{"metricId":4,"longValue":0},{"metricId":3,"longValue":0}],"waitNanos":0,"details":{"splitInfos":[]}},{"inputProfile":[{"records":2,"batches":1,"size":26}],"operatorId":6,"operatorType":17,"setupNanos":29853952,"processNanos":2784887,"peakLocalMemoryAllocated":262152,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":0},{"metricId":8,"longValue":0},{"metricId":5,"longValue":0},{"metricId":6,"longValue":0},{"metricId":3,"longValue":0},{"metricId":7,"longValue":0},{"metricId":13,"longValue":0},{"metricId":12,"longValue":0},{"metricId":11,"longValue":0},{"metricId":9,"longValue":0},{"metricId":4,"longValue":0},{"metricId":10,"longValue":0},{"metricId":2,"longValue":1}],"waitNanos":0},{"inputProfile":[{"records":2,"batches":1,"size":26}],"operatorId":7,"operatorType":2,"setupNanos":1988867,"processNanos":132731,"peakLocalMemoryAllocated":0,"metric":[{"metricId":0,"longValue":0},{"metricId":1,"longValue":0},{"metricId":4,"longValue":0},{"metricId":6,"longValue":0},{"metricId":5,"longValue":1},{"metricId":2,"longValue":0},{"metricId":3,"longValue":0}],"waitNanos":0,"details":{"splitInfos":[{"namedExpression":"NamedExpression [expr=FunctionHolderExpression [args=[ValueVectorReadExpression [fieldId=TypedFieldId [fieldIds=[1], remainder=null]], FunctionHolderExpression [args=[ValueExpression[int=1]], name=castBIGINT, returnType=int64, isRandom=false]], name=greater_than, returnType=boolean, isRandom=false], ref=`_filter_`]","inGandiva":true,"outputName":"_filter_","dependsOn":[]}]}},{"inputProfile":[{"records":4,"batches":1,"size":17}],"operatorId":8,"operatorType":15,"setupNanos":2549848,"processNanos":66286,"peakLocalMemoryAllocated":49184,"metric":[],"waitNanos":0},{"inputProfile":[{"records":4,"batches":4,"size":20}],"operatorId":9,"operatorType":17,"setupNanos":50367597,"processNanos":2526946,"peakLocalMemoryAllocated":229376,"metric":[{"metricId":0,"longValue":0},{"metricId":8,"longValue":0},{"metricId":9,"longValue":0},{"metricId":12,"longValue":0},{"metricId":5,"longValue":0},{"metricId":7,"longValue":0},{"metricId":1,"longValue":0},{"metricId":2,"longValue":4},{"metricId":10,"longValue":0},{"metricId":11,"longValue":0},{"metricId":6,"longValue":0},{"metricId":13,"longValue":0},{"metricId":3,"longValue":0},{"metricId":4,"longValue":0}],"waitNanos":0},{"inputProfile":[{"records":4,"batches":5,"size":20}],"operatorId":10,"operatorType":22,"setupNanos":1807353,"processNanos":5719210,"peakLocalMemoryAllocated":16384,"metric":[{"metricId":1,"longValue":4}],"waitNanos":3706118}],"startTime":1565848550643,"endTime":1565848550767,"memoryUsed":0,"maxMemoryUsed":49000000,"endpoint":{"address":"localhost","userPort":31010,"fabricPort":45678},"lastUpdate":1565848550769,"lastProgress":1565848550769,"sleepingDuration":0,"blockedDuration":2,"firstRun":1565848550645,"runDuration":120,"numRuns":29,"setupDuration":103,"finishDuration":2,"blockedOnUpstreamDuration":0,"blockedOnDownstreamDuration":1,"blockedOnSharedResourceDuration":1,"perResourceBlockedDuration":[{"resource":"FRAGMENT_ACTIVATE_SIGNAL","category":2,"duration":1}]}],"nodePhaseProfile":[{"endpoint":{"address":"localhost","userPort":31010,"fabricPort":45678},"maxMemoryUsed":49524288}]}],"user":"dremio","planningStart":1565848550553,"planningEnd":1565848550640,"jsonPlan":"{\n \"\\\"00-00\\\"\": {\n \"\\\"op\\\"\": \"com.dremio.exec.planner.physical.ScreenPrel\",\n \"\\\"values\\\"\": {},\n \"\\\"inputs\\\"\": [\n \"00-01\"\n ],\n \"\\\"rowType\\\"\": \"RecordType(VARCHAR(65536) Fragment, BIGINT Records, VARCHAR(65536) Path, VARBINARY(65536) Metadata, INTEGER Partition, BIGINT FileSize)\",\n \"\\\"rowCount\\\"\": 1.0,\n \"\\\"cumulativeCost\\\"\": \"{31.1 rows, 1066.1000999999999 cpu, 832.0 io, 832.0 network, 80.0 memory}\"\n },\n \"\\\"00-01\\\"\": {\n \"\\\"op\\\"\": \"com.dremio.exec.planner.physical.ProjectPrel\",\n \"\\\"values\\\"\": {\n \"\\\"fields\\\"\": \"[Fragment, Records, Path, Metadata, Partition, FileSize]\",\n \"\\\"exprs\\\"\": \"[$0, $1, $2, $3, $4, $5]\"\n },\n \"\\\"inputs\\\"\": [\n \"00-02\"\n ],\n \"\\\"rowType\\\"\": \"RecordType(VARCHAR(65536) Fragment, BIGINT Records, VARCHAR(65536) Path, VARBINARY(65536) Metadata, INTEGER Partition, BIGINT FileSize)\",\n \"\\\"rowCount\\\"\": 1.0,\n \"\\\"cumulativeCost\\\"\": \"{31.0 rows, 1066.0001 cpu, 832.0 io, 832.0 network, 80.0 memory}\"\n },\n \"\\\"00-02\\\"\": {\n \"\\\"op\\\"\": \"com.dremio.exec.planner.physical.WriterCommitterPrel\",\n \"\\\"values\\\"\": {\n \"\\\"final\\\"\": \"/Users/balajiramaswamy/Downloads/dremio-enterprise-3.3.1-201907291852280797-df23756/data/pdfs/results/22ab0c19-6dca-0d0d-d146-61645a4a5100\"\n },\n \"\\\"inputs\\\"\": [\n \"00-03\"\n ],\n \"\\\"rowType\\\"\": \"RecordType(VARCHAR(65536) Fragment, BIGINT Records, VARCHAR(65536) Path, VARBINARY(65536) Metadata, INTEGER Partition, BIGINT FileSize)\",\n \"\\\"rowCount\\\"\": 1.0,\n \"\\\"cumulativeCost\\\"\": \"{30.0 rows, 1066.00004 cpu, 832.0 io, 832.0 network, 80.0 memory}\"\n },\n \"\\\"00-03\\\"\": {\n \"\\\"op\\\"\": \"com.dremio.exec.planner.physical.WriterPrel\",\n \"\\\"values\\\"\": {},\n \"\\\"inputs\\\"\": [\n \"00-04\"\n ],\n \"\\\"rowType\\\"\": \"RecordType(VARCHAR(65536) Fragment, BIGINT Records, VARCHAR(65536) Path, VARBINARY(65536) Metadata, INTEGER Partition, BIGINT FileSize)\",\n \"\\\"rowCount\\\"\": 1.0,\n \"\\\"cumulativeCost\\\"\": \"{29.0 rows, 1065.00004 cpu, 832.0 io, 832.0 network, 80.0 memory}\"\n },\n \"\\\"00-04\\\"\": {\n \"\\\"op\\\"\": \"com.dremio.exec.planner.physical.ProjectPrel\",\n \"\\\"values\\\"\": {\n \"\\\"fields\\\"\": \"[line_id, EXPR$1]\",\n \"\\\"exprs\\\"\": \"[$0, $1]\"\n },\n \"\\\"inputs\\\"\": [\n \"00-05\"\n ],\n \"\\\"rowType\\\"\": \"RecordType(INTEGER line_id, BIGINT EXPR$1)\",\n \"\\\"rowCount\\\"\": 1.0,\n \"\\\"cumulativeCost\\\"\": \"{28.0 rows, 1064.00004 cpu, 832.0 io, 832.0 network, 80.0 memory}\"\n },\n \"\\\"00-05\\\"\": {\n \"\\\"op\\\"\": \"com.dremio.exec.planner.physical.ProjectPrel\",\n \"\\\"values\\\"\": {\n \"\\\"fields\\\"\": \"[line_id, EXPR$1]\",\n \"\\\"exprs\\\"\": \"[$0, $1]\"\n },\n \"\\\"inputs\\\"\": [\n \"00-06\"\n ],\n \"\\\"rowType\\\"\": \"RecordType(INTEGER line_id, BIGINT EXPR$1)\",\n \"\\\"rowCount\\\"\": 1.0,\n \"\\\"cumulativeCost\\\"\": \"{27.0 rows, 1064.00002 cpu, 832.0 io, 832.0 network, 80.0 memory}\"\n },\n \"\\\"00-06\\\"\": {\n \"\\\"op\\\"\": \"com.dremio.exec.planner.physical.SortPrel\",\n \"\\\"values\\\"\": {\n \"\\\"collation\\\"\": \"[0]\"\n },\n \"\\\"inputs\\\"\": [\n \"00-07\"\n ],\n \"\\\"rowType\\\"\": \"RecordType(INTEGER line_id, BIGINT EXPR$1)\",\n \"\\\"rowCount\\\"\": 1.0,\n \"\\\"cumulativeCost\\\"\": \"{26.0 rows, 1064.0 cpu, 832.0 io, 832.0 network, 80.0 memory}\"\n },\n \"\\\"00-07\\\"\": {\n \"\\\"op\\\"\": \"com.dremio.exec.planner.physical.FilterPrel\",\n \"\\\"values\\\"\": {\n \"\\\"condition\\\"\": \">($1, 1)\"\n },\n \"\\\"inputs\\\"\": [\n \"00-08\"\n ],\n \"\\\"rowType\\\"\": \"RecordType(INTEGER line_id, BIGINT EXPR$1)\",\n \"\\\"rowCount\\\"\": 1.0,\n \"\\\"cumulativeCost\\\"\": \"{25.0 rows, 1064.0 cpu, 832.0 io, 832.0 network, 64.0 memory}\"\n },\n \"\\\"00-08\\\"\": {\n \"\\\"op\\\"\": \"com.dremio.exec.planner.physical.StreamAggPrel\",\n \"\\\"values\\\"\": {\n \"\\\"group\\\"\": \"{0}\",\n \"\\\"aggs\\\"\": \"[COUNT()]\"\n },\n \"\\\"inputs\\\"\": [\n \"00-09\"\n ],\n \"\\\"rowType\\\"\": \"RecordType(INTEGER line_id, BIGINT EXPR$1)\",\n \"\\\"rowCount\\\"\": 1.0,\n \"\\\"cumulativeCost\\\"\": \"{24.0 rows, 1056.0 cpu, 832.0 io, 832.0 network, 64.0 memory}\"\n },\n \"\\\"00-09\\\"\": {\n \"\\\"op\\\"\": \"com.dremio.exec.planner.physical.SortPrel\",\n \"\\\"values\\\"\": {\n \"\\\"collation\\\"\": \"[0]\"\n },\n \"\\\"inputs\\\"\": [\n \"00-10\"\n ],\n \"\\\"rowType\\\"\": \"RecordType(INTEGER line_id)\",\n \"\\\"rowCount\\\"\": 8.0,\n \"\\\"cumulativeCost\\\"\": \"{16.0 rows, 928.0 cpu, 832.0 io, 832.0 network, 64.0 memory}\"\n },\n \"\\\"00-10\\\"\": {\n \"\\\"op\\\"\": \"com.dremio.exec.store.hive.HiveRulesFactory$HiveScanPrel\",\n \"\\\"values\\\"\": {\n \"\\\"table\\\"\": \"localhive.\\\"default\\\".dim_order\",\n \"\\\"columns\\\"\": \"`line_id`\",\n \"\\\"splits\\\"\": \"4\",\n \"\\\"mode\\\"\": \"BASIC\"\n },\n \"\\\"inputs\\\"\": [],\n \"\\\"rowType\\\"\": \"RecordType(INTEGER line_id)\",\n \"\\\"rowCount\\\"\": 8.0,\n \"\\\"cumulativeCost\\\"\": \"{8.0 rows, 832.0 cpu, 832.0 io, 832.0 network, 0.0 memory}\"\n }\n}","clientInfo":{"name":"Dremio Java local client","version":"3.3.1-201907291852280797-df23756","majorVersion":3,"minorVersion":3,"patchVersion":1,"application":"17939@Bali.local","buildNumber":1,"versionQualifier":"201907291852280797-df23756"},"planPhases":[{"phaseName":"Validation","durationMillis":1,"plan":""},{"phaseName":"Convert To Rel","durationMillis":1,"plan":"LogicalSort(sort0=[$0], dir0=[ASC])\n LogicalFilter(condition=[>($1, 1)])\n LogicalAggregate(group=[{0}], EXPR$1=[COUNT()])\n LogicalProject(line_id=[$1])\n ScanCrel(table=[localhive.\"default\".dim_order], columns=[`orderno`, `line_id`, `promise_date`, `orig_promise_date`], splits=[4])\n"},{"phaseName":"Reduce Expressions","durationMillis":0,"plan":""},{"phaseName":"JDBC Pushdown","durationMillis":0,"plan":""},{"phaseName":"Window Function Rewrites","durationMillis":0,"plan":""},{"phaseName":"Pre-Logical Filter Pushdown","durationMillis":0,"plan":""},{"phaseName":"Find Materializations","durationMillis":0,"plan":""},{"phaseName":"Substitution","durationMillis":0,"plan":""},{"phaseName":"Logical Planning","durationMillis":2,"plan":"SortRel(sort0=[$0], dir0=[ASC])\n FilterRel(condition=[>($1, 1)])\n AggregateRel(group=[{0}], EXPR$1=[COUNT()])\n HiveScanDrel(table=[localhive.\"default\".dim_order], columns=[`line_id`], splits=[4])\n"},{"phaseName":"Multi-Join analysis","durationMillis":0,"plan":""},{"phaseName":"LOPT Join Planning","durationMillis":0,"plan":""},{"phaseName":"Physical Planning","durationMillis":45,"plan":""},{"phaseName":"Final Physical Transformation","durationMillis":4,"plan":"00-00 Screen : rowType = RecordType(VARCHAR(65536) Fragment, BIGINT Records, VARCHAR(65536) Path, VARBINARY(65536) Metadata, INTEGER Partition, BIGINT FileSize): rowcount = 1.0, cumulative cost = {31.1 rows, 1066.1000999999999 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15270\n00-01 Project(Fragment=[$0], Records=[$1], Path=[$2], Metadata=[$3], Partition=[$4], FileSize=[$5]) : rowType = RecordType(VARCHAR(65536) Fragment, BIGINT Records, VARCHAR(65536) Path, VARBINARY(65536) Metadata, INTEGER Partition, BIGINT FileSize): rowcount = 1.0, cumulative cost = {31.0 rows, 1066.0001 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15269\n00-02 WriterCommitter(final=[/Users/balajiramaswamy/Downloads/dremio-enterprise-3.3.1-201907291852280797-df23756/data/pdfs/results/22ab0c19-6dca-0d0d-d146-61645a4a5100]) : rowType = RecordType(VARCHAR(65536) Fragment, BIGINT Records, VARCHAR(65536) Path, VARBINARY(65536) Metadata, INTEGER Partition, BIGINT FileSize): rowcount = 1.0, cumulative cost = {30.0 rows, 1066.00004 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15268\n00-03 Writer : rowType = RecordType(VARCHAR(65536) Fragment, BIGINT Records, VARCHAR(65536) Path, VARBINARY(65536) Metadata, INTEGER Partition, BIGINT FileSize): rowcount = 1.0, cumulative cost = {29.0 rows, 1065.00004 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15267\n00-04 Project(line_id=[$0], EXPR$1=[$1]) : rowType = RecordType(INTEGER line_id, BIGINT EXPR$1): rowcount = 1.0, cumulative cost = {28.0 rows, 1064.00004 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15266\n00-05 Project(line_id=[$0], EXPR$1=[$1]) : rowType = RecordType(INTEGER line_id, BIGINT EXPR$1): rowcount = 1.0, cumulative cost = {27.0 rows, 1064.00002 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15265\n00-06 Sort(sort0=[$0], dir0=[ASC]) : rowType = RecordType(INTEGER line_id, BIGINT EXPR$1): rowcount = 1.0, cumulative cost = {26.0 rows, 1064.0 cpu, 832.0 io, 832.0 network, 80.0 memory}, id = 15264\n00-07 Filter(condition=[>($1, 1)]) : rowType = RecordType(INTEGER line_id, BIGINT EXPR$1): rowcount = 1.0, cumulative cost = {25.0 rows, 1064.0 cpu, 832.0 io, 832.0 network, 64.0 memory}, id = 15263\n00-08 StreamAgg(group=[{0}], EXPR$1=[COUNT()]) : rowType = RecordType(INTEGER line_id, BIGINT EXPR$1): rowcount = 1.0, cumulative cost = {24.0 rows, 1056.0 cpu, 832.0 io, 832.0 network, 64.0 memory}, id = 15262\n00-09 Sort(sort0=[$0], dir0=[ASC]) : rowType = RecordType(INTEGER line_id): rowcount = 8.0, cumulative cost = {16.0 rows, 928.0 cpu, 832.0 io, 832.0 network, 64.0 memory}, id = 15261\n00-10 HiveScan(table=[localhive.\"default\".dim_order], columns=[`line_id`], splits=[4], mode=[BASIC]) : rowType = RecordType(INTEGER line_id): rowcount = 8.0, cumulative cost = {8.0 rows, 832.0 cpu, 832.0 io, 832.0 network, 0.0 memory}, id = 15260\n"},{"phaseName":"Execution Plan: Executor Selection","durationMillis":0,"plan":"idealNumFragments: 1\nidealNumNodes : 1\nnumExecutors : 1\ndetails : Selected collection: 0, pre-selection state: (1/2147483647):0, (0/u), (0/b)"},{"phaseName":"Execution Plan: Fragment Assignment","durationMillis":1,"plan":""},{"phaseName":"Execution Plan: Plan Generation","durationMillis":1,"plan":""},{"phaseName":"Fragment Start RPCs","durationMillis":3,"sizeStats":{"sizePerNode":5888,"fragments":[{"majorId":0,"majorPortionSize":3713,"minorPortionSize":660}],"minorSpecificAttrs":[{"name":"scan-splits-normalized","size":622}],"sharedAttrs":[{"name":"scan-partitions-normalized::74","size":917}]}},{"phaseName":"Fragment Activate RPCs","durationMillis":0},{"phaseName":"localhive.\"default\".dim_order: PERMISSION_CACHE_HIT","durationMillis":0},{"phaseName":"localhive.\"default\".dim_order: PERMISSION_CACHE_HIT","durationMillis":0},{"phaseName":"localhive.\"default\".dim_order: CACHED_METADATA","durationMillis":0},{"phaseName":"localhive.\"default\".dim_order: PERMISSION_CACHE_HIT","durationMillis":0},{"phaseName":"localhive.\"default\".dim_order: PERMISSION_CACHE_HIT","durationMillis":0},{"phaseName":"localhive.\"default\".dim_order: CACHED_METADATA","durationMillis":0}],"accelerationProfile":{"accelerated":false,"numSubstitutions":0,"millisTakenGettingMaterializations":0,"millisTakenNormalizing":0,"millisTakenSubstituting":0,"layoutProfiles":[],"normalizedQueryPlans":[],"accelerationDetails":""},"nonDefaultOptionsJSON":"[ {\n \"kind\" : \"STRING\",\n \"type\" : \"QUERY\",\n \"name\" : \"planner.query_results_store_path\",\n \"string_val\" : \"__jobResultsStore\"\n}, {\n \"kind\" : \"BOOLEAN\",\n \"type\" : \"QUERY\",\n \"name\" : \"planner.output_limit_enable\",\n \"bool_val\" : true\n}, {\n \"kind\" : \"STRING\",\n \"type\" : \"QUERY\",\n \"name\" : \"planner.store_query_results\",\n \"string_val\" : \"PATH_AND_ATTEMPT_ID\"\n}, {\n \"kind\" : \"LONG\",\n \"type\" : \"SYSTEM\",\n \"name\" : \"limits.single_field_size_bytes\",\n \"num_val\" : 60086\n} ]","datasetProfile":[{"datasetPath":"localhive.\"default\".dim_order","type":1,"batchSchema":"EAAAAAAACgAMAAAACAAEAAoAAAAIAAAACAAAAAAAAAAEAAAA1AAAAIQAAABEAAAABAAAAE7///8UAAAAFAAAABQAAAAAAAgBEAAAAAAAAAAAAAAAxP///xEAAABvcmlnX3Byb21pc2VfZGF0ZQAAAIr///8UAAAAFAAAABgAAAAAAAgBFAAAAAAAAAAAAAAABAAEAAQAAAAMAAAAcHJvbWlzZV9kYXRlAAAAAMb///8UAAAAFAAAABQAAAAAAAIBGAAAAAAAAAAAAAAAtP///wAAAAEgAAAABwAAAGxpbmVfaWQAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAABQAAAAcAAAAAAACASAAAAAAAAAAAAAAAAgADAAIAAcACAAAAAAAAAEgAAAABwAAAG9yZGVybm8A","allowApproxStats":false},{"datasetPath":"localhive.\"default\".dim_order","type":1,"batchSchema":"EAAAAAAACgAMAAAACAAEAAoAAAAIAAAACAAAAAAAAAAEAAAA1AAAAIQAAABEAAAABAAAAE7///8UAAAAFAAAABQAAAAAAAgBEAAAAAAAAAAAAAAAxP///xEAAABvcmlnX3Byb21pc2VfZGF0ZQAAAIr///8UAAAAFAAAABgAAAAAAAgBFAAAAAAAAAAAAAAABAAEAAQAAAAMAAAAcHJvbWlzZV9kYXRlAAAAAMb///8UAAAAFAAAABQAAAAAAAIBGAAAAAAAAAAAAAAAtP///wAAAAEgAAAABwAAAGxpbmVfaWQAAAASABgAFAATABIADAAAAAgABAASAAAAFAAAABQAAAAcAAAAAAACASAAAAAAAAAAAAAAAAgADAAIAAcACAAAAAAAAAEgAAAABwAAAG9yZGVybm8A","allowApproxStats":false}],"nodeProfile":[{"endpoint":{"address":"localhost","userPort":31010,"fabricPort":45678},"maxMemoryUsed":49524288,"timeEnqueuedBeforeSubmitMs":0}],"dremioVersion":"3.3.1-201907291852280797-df23756","operatorTypeMetricsMap":{"metricsDef":[{"metricDef":[{"id":0,"name":"BYTES_SENT"}]},{"metricDef":[{"id":0,"name":"N_RECEIVERS"},{"id":1,"name":"BYTES_SENT"}]},{"metricDef":[{"id":0,"name":"JAVA_BUILD_TIME"},{"id":1,"name":"JAVA_EXECUTE_TIME"},{"id":2,"name":"GANDIVA_BUILD_TIME"},{"id":3,"name":"GANDIVA_EXECUTE_TIME"},{"id":4,"name":"JAVA_EXPRESSIONS"},{"id":5,"name":"GANDIVA_EXPRESSIONS"},{"id":6,"name":"MIXED_SPLITS"}]},{"metricDef":[{"id":0,"name":"NUM_BUCKETS"},{"id":1,"name":"NUM_ENTRIES"},{"id":2,"name":"NUM_RESIZING"},{"id":3,"name":"RESIZING_TIME"},{"id":4,"name":"PIVOT_TIME"},{"id":5,"name":"INSERT_TIME"},{"id":6,"name":"ACCUMULATE_TIME"},{"id":7,"name":"REVERSE_TIME"},{"id":8,"name":"UNPIVOT_TIME"},{"id":9,"name":"VECTORIZED"},{"id":10,"name":"NUM_HASH_PARTITIONS"},{"id":11,"name":"MAX_HASHTABLE_BATCH_SIZE"},{"id":12,"name":"MIN_HASHTABLE_ENTRIES"},{"id":13,"name":"MAX_HASHTABLE_ENTRIES"},{"id":14,"name":"MIN_REHASH_COUNT"},{"id":15,"name":"MAX_REHASH_COUNT"},{"id":16,"name":"PREALLOCATED_MEMORY"},{"id":17,"name":"SPILL_COUNT"},{"id":18,"name":"PARTITIONS_SPILLED"},{"id":19,"name":"ITERATIONS"},{"id":20,"name":"RAN_OUT_OF_MEMORY"},{"id":21,"name":"SPILL_TIME"},{"id":22,"name":"READ_SPILLED_BATCH_TIME"},{"id":23,"name":"TOTAL_BATCHES_SPILLED"},{"id":24,"name":"MAX_BATCHES_SPILLED"},{"id":25,"name":"TOTAL_RECORDS_SPILLED"},{"id":26,"name":"MAX_RECORDS_SPILLED"},{"id":27,"name":"RECURSION_DEPTH"},{"id":28,"name":"TOTAL_SPILLED_DATA_SIZE"},{"id":29,"name":"MAX_SPILLED_DATA_SIZE"},{"id":30,"name":"MAX_TOTAL_NUM_BUCKETS"},{"id":31,"name":"MAX_TOTAL_NUM_ENTRIES"},{"id":32,"name":"ALLOCATED_FOR_FIXED_KEYS"},{"id":33,"name":"UNUSED_FOR_FIXED_KEYS"},{"id":34,"name":"ALLOCATED_FOR_VARIABLE_KEYS"},{"id":35,"name":"UNUSED_FOR_VARIABLE_KEYS"},{"id":36,"name":"MAX_VARIABLE_BLOCK_LENGTH"},{"id":37,"name":"OOB_SENDS"},{"id":38,"name":"OOB_RECEIVES"},{"id":39,"name":"OOB_DROP_LOCAL"},{"id":40,"name":"OOB_DROP_WRONG_STATE"},{"id":41,"name":"OOB_DROP_UNDER_THRESHOLD"},{"id":42,"name":"OOB_DROP_NO_VICTIM"},{"id":43,"name":"OOB_SPILL"},{"id":44,"name":"OOB_DROP_ALREADY_SPILLING"}]},{"metricDef":[{"id":0,"name":"NUM_BUCKETS"},{"id":1,"name":"NUM_ENTRIES"},{"id":2,"name":"NUM_RESIZING"},{"id":3,"name":"RESIZING_TIME_NANOS"},{"id":4,"name":"PIVOT_TIME_NANOS"},{"id":5,"name":"INSERT_TIME_NANOS"},{"id":6,"name":"ACCUMULATE_TIME_NANOS"},{"id":7,"name":"REVERSE_TIME_NANOS"},{"id":8,"name":"UNPIVOT_TIME_NANOS"},{"id":9,"name":"VECTORIZED"},{"id":10,"name":"PROBE_PIVOT_NANOS"},{"id":11,"name":"PROBE_MATCH_NANOS"},{"id":12,"name":"PROBE_LIST_NANOS"},{"id":13,"name":"PROBE_FIND_NANOS"},{"id":14,"name":"PROBE_COPY_NANOS"},{"id":15,"name":"BUILD_COPY_NANOS"},{"id":16,"name":"BUILD_COPY_NOMATCH_NANOS"},{"id":17,"name":"LINK_TIME_NANOS"},{"id":18,"name":"UNMATCHED_BUILD_KEY_COUNT"},{"id":19,"name":"UNMATCHED_PROBE_COUNT"},{"id":20,"name":"OUTPUT_RECORDS"},{"id":21,"name":"HASHCOMPUTATION_TIME_NANOS"},{"id":22,"name":"PROBE_HASHCOMPUTATION_TIME_NANOS"}]},{"metricDef":[]},{"metricDef":[{"id":0,"name":"BATCHES_SENT"},{"id":1,"name":"RECORDS_SENT"},{"id":2,"name":"MIN_RECORDS"},{"id":3,"name":"MAX_RECORDS"},{"id":4,"name":"N_RECEIVERS"},{"id":5,"name":"BYTES_SENT"},{"id":6,"name":"SENDING_THREADS_COUNT"},{"id":7,"name":"COST"},{"id":8,"name":"NUM_COPIES"},{"id":9,"name":"FIXED_COPY_NS"},{"id":10,"name":"VARIABLE_COPY_NS"},{"id":11,"name":"BINARY_COPY_NS"},{"id":12,"name":"GENERIC_COPY_NS"},{"id":13,"name":"PRECOPY_NS"},{"id":14,"name":"FLUSH_NS"},{"id":15,"name":"NUM_FLUSHES"},{"id":16,"name":"BUCKET_SIZE"}]},{"metricDef":[]},{"metricDef":[{"id":0,"name":"BYTES_RECEIVED"},{"id":1,"name":"NUM_SENDERS"},{"id":2,"name":"NEXT_WAIT_NANOS"}]},{"metricDef":[]},{"metricDef":[{"id":0,"name":"JAVA_EXPRESSIONS"},{"id":1,"name":"JAVA_BUILD_TIME"},{"id":2,"name":"JAVA_EVALUATE_TIME"},{"id":3,"name":"GANDIVA_EXPRESSIONS"},{"id":4,"name":"GANDIVA_BUILD_TIME"},{"id":5,"name":"GANDIVA_EVALUATE_TIME"},{"id":6,"name":"MIXED_EXPRESSIONS"},{"id":7,"name":"MIXED_SPLITS"}]},{"metricDef":[{"id":0,"name":"BYTES_RECEIVED"},{"id":1,"name":"NUM_SENDERS"}]},{"metricDef":[]},{"metricDef":[{"id":0,"name":"BYTES_SENT"}]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[{"id":0,"name":"SPILL_COUNT"},{"id":1,"name":"MERGE_COUNT"},{"id":2,"name":"PEAK_BATCHES_IN_MEMORY"},{"id":3,"name":"MAX_BATCH_SIZE"},{"id":4,"name":"AVG_BATCH_SIZE"},{"id":5,"name":"SPILL_TIME_NANOS"},{"id":6,"name":"MERGE_TIME_NANOS"},{"id":7,"name":"TOTAL_SPILLED_DATA_SIZE"},{"id":8,"name":"OOB_SENDS"},{"id":9,"name":"OOB_RECEIVES"},{"id":10,"name":"OOB_DROP_LOCAL"},{"id":11,"name":"OOB_DROP_WRONG_STATE"},{"id":12,"name":"OOB_DROP_UNDER_THRESHOLD"},{"id":13,"name":"OOB_SPILL"}]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[{"id":0,"name":"SETUP_NS"},{"id":1,"name":"NUM_READERS"},{"id":2,"name":"NUM_REMOTE_READERS"},{"id":3,"name":"NUM_ROW_GROUPS"},{"id":4,"name":"NUM_VECTORIZED_COLUMNS"},{"id":5,"name":"NUM_NON_VECTORIZED_COLUMNS"},{"id":6,"name":"COPY_MS"},{"id":7,"name":"FILTER_MS"},{"id":8,"name":"PARQUET_EXEC_PATH"},{"id":9,"name":"FILTER_EXISTS"},{"id":10,"name":"PARQUET_BYTES_READ"},{"id":11,"name":"NUM_ASYNC_STREAMS"},{"id":12,"name":"NUM_ASYNC_READS"},{"id":13,"name":"NUM_ASYNC_BYTES_READ"},{"id":14,"name":"NUM_EXTENDING_READS"},{"id":15,"name":"EXTENDING_READ_BYTES"},{"id":16,"name":"TOTAL_BYTES_READ"},{"id":17,"name":"LOCAL_BYTES_READ"},{"id":18,"name":"SHORT_CIRCUIT_BYTES_READ"},{"id":19,"name":"PRELOADED_BYTES"}]},{"metricDef":[{"id":0,"name":"SETUP_NS"},{"id":1,"name":"NUM_READERS"},{"id":2,"name":"NUM_REMOTE_READERS"},{"id":3,"name":"NUM_ROW_GROUPS"},{"id":4,"name":"NUM_VECTORIZED_COLUMNS"},{"id":5,"name":"NUM_NON_VECTORIZED_COLUMNS"},{"id":6,"name":"COPY_MS"},{"id":7,"name":"FILTER_MS"},{"id":8,"name":"PARQUET_EXEC_PATH"},{"id":9,"name":"FILTER_EXISTS"},{"id":10,"name":"PARQUET_BYTES_READ"},{"id":11,"name":"NUM_ASYNC_STREAMS"},{"id":12,"name":"NUM_ASYNC_READS"},{"id":13,"name":"NUM_ASYNC_BYTES_READ"},{"id":14,"name":"NUM_EXTENDING_READS"},{"id":15,"name":"EXTENDING_READ_BYTES"},{"id":16,"name":"TOTAL_BYTES_READ"},{"id":17,"name":"LOCAL_BYTES_READ"},{"id":18,"name":"SHORT_CIRCUIT_BYTES_READ"},{"id":19,"name":"PRELOADED_BYTES"}]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[{"id":0,"name":"NUM_FILES_WRITTEN"},{"id":1,"name":"MIN_FILE_SIZE"},{"id":2,"name":"MAX_FILE_SIZE"},{"id":3,"name":"AVG_FILE_SIZE"},{"id":4,"name":"MIN_RECORD_COUNT_IN_FILE"},{"id":5,"name":"MAX_RECORD_COUNT_IN_FILE"}]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[{"id":0,"name":"BYTES_WRITTEN"},{"id":1,"name":"OUTPUT_LIMITED"}]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[]},{"metricDef":[{"id":0,"name":"N_RECEIVERS"},{"id":1,"name":"BYTES_SENT"}]}]},"resourceSchedulingProfile":{"queueName":"Low Cost User Queries","queueId":"a91ff0cc-ee2f-409a-85b7-57284778d057","ruleContent":"query_cost() < 30000000","ruleName":"Low Cost User Queries","ruleAction":"PLACE","schedulingProperties":{"queryCost":25.0,"queryType":"UI Run"},"resourceSchedulingStart":1565848550620,"resourceSchedulingEnd":1565848550637},"commandPoolWaitMillis":0}