Skip to main content
Version: current [24.3.x]

Configuring via dremio.conf

This topic describes how to configure Dremio, in particular, what properties you need to modify in the dremio.conf file and how to accomplish it.

note

The dremio.conf file uses the HOCON syntax.

This topic covers following:

Disk Space Recommendations

For cluster deployments, we recommend the query spill volume contain 128 GB per node or 2 * executor RAM, whichever is larger. The Dremio administrator should watch for "out of spill space" messages.

Template for dremio.conf

Template for dremio.conf
paths: {

# the local path for dremio to store data.
local: ${DREMIO_HOME}"/data"

# the distributed path Dremio data including job results, downloads, uploads, etc
dist: "pdfs://"${paths.local}"/pdfs"

# location for catalog database (if master node)
db: ${paths.local}/db,

spilling: [${paths.local}/spill]

# storage area for the accelerator cache.
accelerator: ${paths.dist}/accelerator

# staging area for json and csv ui downloads
downloads: ${paths.dist}/downloads

# stores uploaded data associated with user home directories
uploads: ${paths.dist}/uploads

# stores data associated with the job results cache.
results: ${paths.dist}/results

# shared scratch space for creation of tables.
scratch: ${paths.dist}/scratch

}

services: {
coordinator: {
enabled: true,

# Auto-upgrade Dremio at startup if needed
auto-upgrade: false,

master: {
enabled: true,
# configure an embedded ZooKeeper server on the same node as master
embedded-zookeeper: {
enabled: true,
port: 2181,
path: ${paths.local}/zk
}
},

web: {
enabled: true,
port: 9047,
ssl: {
# If SSL for communication path between browsers (or REST clients) and Dremio should be enabled.
enabled: false,

# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: true

# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties

# KeyStore type
keyStoreType: ${javax.net.ssl.keyStoreType},

# Path to KeyStore file
keyStore: ${javax.net.ssl.keyStore},

# Password to access the keystore file
keyStorePassword: ${javax.net.ssl.keyStorePassword},

# Password to access the key
keyPassword: ${javax.net.ssl.keyPassword},

# TrustStore type
trustStoreType: ${javax.net.ssl.trustStoreType},

# Path to TrustStore file
trustStore: ${javax.net.ssl.trustStore},

# Password to access the truststore file
trustStorePassword: ${javax.net.ssl.trustStorePassword}
},
auth: {
type: "internal", # Possible values are "internal", "ldap" (includes Active directory)
# LDAP config file for configuration when auth type is "ldap"
ldap_config: "ldap.json" # file name provided should be in the classpath.
}
tokens: {
cache: {
# number of tokens to store locally on this coordinator node
# (set to 0, if all requests should be made directly to the database of tokens)
size: 100
# time (in minutes) after which the token needs to be obtained from the token store
expiration_minutes: 5
}
}
},

client-endpoint: {
port: 31010
},

scheduler: {
threads: 24
},

command-pool: {
enabled: true
size: 0 # 0 defaults to the machine's number of cores
}
},

executor: {
enabled: true
},

flight: {
enabled: true,
port: 32010,
# Authentication mode for the Arrow FlightServer endpoint.
# There are two modes:
# - legacy.arrow.flight.auth
# - arrow.flight.auth2
# legacy.arrow.flight.auth is backwards compatible and will be deprecated in the future.
# arrow.flight.auth2 is the new and preferred Arrow Flight authentication method.
auth.mode: "arrow.flight.auth2"
}

fabric: {
port: 45678,

memory: {
reservation: 100M
}
},

conduit: {
# If set to 0, a port is automatically allocated (typically in ephemeral range). Otherwise, the configured value
# is used.
port: 0

ssl: {
# If SSL for communication path between Dremio instances should be enabled.
enabled: ${services.fabric.ssl.enabled},

# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: ${services.fabric.ssl.auto-certificate.enabled},

# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties

# KeyStore type
keyStoreType: ${services.fabric.ssl.keyStoreType},

# Path to KeyStore file
keyStore: ${services.fabric.ssl.keyStore},

# Password to access the keystore file
keyStorePassword: ${services.fabric.ssl.keyStorePassword},

# Password to access the key
keyPassword: ${services.fabric.ssl.keyPassword},

# TrustStore type
trustStoreType: ${services.fabric.ssl.trustStoreType},

# Path to TrustStore file
trustStore: ${services.fabric.ssl.trustStore},

# Password to access the truststore file
trustStorePassword: ${services.fabric.ssl.trustStorePassword}
}
}

# Set up kerberos credentials in server (applicable for both coordinator and executor)
kerberos: {
principal: "",
keytab.file.path: ""
}

web-admin: {
enabled: true,
# Port, bound to loopback interface, on which the daemon responds to liveness HTTP requests (0 == auto-allocated)
port: 0
}
}

provisioning: {
yarn: {
jvmoptions: "",
# list of jars to be added to the main container classpath
classpath: []
app {
# list of jars to be added to the Dremio application classpath
classpath: []
# list of jars to be added to the Dremio application classpath (at the front)
classpath-prefix: []
}
watchdog: {
# how long to wait for server reply before considering it failed
poll.timeout: 1000ms
# interval between two server polls
poll.interval: 10000ms
# how many consecutive failed attempts before killing server
missed.polls.before.kill: 6
# maximum attempts at killing server
max.kill.attempts: 10
# how long before reattempting killing server
kill.reattempt.interval: 1000ms
}
}
}

# the zookeeper quorum for the cluster
zookeeper: "localhost:"${services.coordinator.master.embedded-zookeeper.port}
zk.client.session.timeout: 90000


# These system properties are listed here to allow substitution of system property values for DAC Web SSL properties
# listed in services.web.ssl section. Currently we consider only the system properties listed in this file for
# substitution.
javax.net.ssl {
keyStoreType: "",
keyStore: "",
keyStorePassword: "",
keyPassword: "",
trustStoreType: "",
trustStore:"",
trustStorePassword: ""
}

registration.publish-host: ""