Configuring via dremio.conf
This topic describes how to configure Dremio, in particular, what properties you need to modify in the dremio.conf file and how to accomplish it.
note:
The dremio.conf file uses the HOCON syntax.
This topic covers following:
- Configuring Dremio Services
- Configuring Metadata Storage
- Configuring Distributed Storage
- Configuring Zookeeper
- Configuring High Availability
- Configuring Wire Encryption
Template for dremio.conf
Template for dremio.confpaths: {
# the local path for dremio to store data.
local: ${DREMIO_HOME}"/data"
# the distributed path Dremio data including job results, downloads, uploads, etc
dist: "pdfs://"${paths.local}"/pdfs"
# location for catalog database (if master node)
db: ${paths.local}/db,
spilling: [${paths.local}/spill]
# storage area for the accelerator cache.
accelerator: ${paths.dist}/accelerator
# staging area for json and csv ui downloads
downloads: ${paths.dist}/downloads
# stores uploaded data associated with user home directories
uploads: ${paths.dist}/uploads
# stores data associated with the job results cache.
results: ${paths.dist}/results
# shared scratch space for creation of tables.
scratch: ${paths.dist}/scratch
}
services: {
coordinator: {
enabled: true,
# Auto-upgrade Dremio at startup if needed
auto-upgrade: false,
master: {
enabled: true,
# configure an embedded ZooKeeper server on the same node as master
embedded-zookeeper: {
enabled: true,
port: 2181,
path: ${paths.local}/zk
}
},
web: {
enabled: true,
port: 9047,
ssl: {
# If SSL for communication path between browsers (or REST clients) and Dremio should be enabled.
enabled: false,
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: true
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${javax.net.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${javax.net.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${javax.net.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${javax.net.ssl.keyPassword},
# TrustStore type
trustStoreType: ${javax.net.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${javax.net.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${javax.net.ssl.trustStorePassword}
},
auth: {
type: "internal", # Possible values are "internal", "ldap" (includes Active directory)
# LDAP config file for configuration when auth type is "ldap"
ldap_config: "ldap.json" # file name provided should be in the classpath.
}
ui: {
# Configuration for Intercom
intercom: {
enabled: true
appid: "@dremio.ui.intercom.appid@"
}
}
tokens: {
cache: {
# number of tokens to store locally on this coordinator node
# (set to 0, if all requests should be made directly to the database of tokens)
size: 100
# time (in minutes) after which the token needs to be obtained from the token store
expiration_minutes: 5
}
}
},
client-endpoint: {
port: 31010
},
scheduler: {
threads: 24
},
command-pool: {
enabled: true
size: 0 # 0 defaults to the machine's number of cores
}
},
executor: {
enabled: true
},
flight: {
enabled: true,
port: 32010,
# Authentication mode for the Arrow FlightServer endpoint.
# There are two modes:
# - legacy.arrow.flight.auth
# - arrow.flight.auth2
# legacy.arrow.flight.auth is backwards compatible and will be deprecated in the future.
# arrow.flight.auth2 is the new and preferred Arrow Flight authentication method.
auth.mode: "arrow.flight.auth2"
}
fabric: {
port: 45678,
memory: {
reservation: 100M
}
},
conduit: {
# If set to 0, a port is automatically allocated (typically in ephemeral range). Otherwise, the configured value
# is used.
port: 0
ssl: {
# If SSL for communication path between Dremio instances should be enabled.
enabled: ${services.fabric.ssl.enabled},
# Allow for auto-generated certificates if keyStore option is not set
# Auto-generated self-signed certificates are considered insecure, and this
# option should be set to false in production environment
auto-certificate.enabled: ${services.fabric.ssl.auto-certificate.enabled},
# KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.
# If needed to be overridden, then change the below properties
# KeyStore type
keyStoreType: ${services.fabric.ssl.keyStoreType},
# Path to KeyStore file
keyStore: ${services.fabric.ssl.keyStore},
# Password to access the keystore file
keyStorePassword: ${services.fabric.ssl.keyStorePassword},
# Password to access the key
keyPassword: ${services.fabric.ssl.keyPassword},
# TrustStore type
trustStoreType: ${services.fabric.ssl.trustStoreType},
# Path to TrustStore file
trustStore: ${services.fabric.ssl.trustStore},
# Password to access the truststore file
trustStorePassword: ${services.fabric.ssl.trustStorePassword}
}
}
# Set up kerberos credentials in server (applicable for both coordinator and executor)
kerberos: {
principal: "",
keytab.file.path: ""
}
web-admin: {
enabled: true,
# Port, bound to loopback interface, on which the daemon responds to liveness HTTP requests (0 == auto-allocated)
port: 0
}
}
provisioning: {
yarn: {
jvmoptions: "",
# list of jars to be added to the main container classpath
classpath: []
app {
# list of jars to be added to the Dremio application classpath
classpath: []
# list of jars to be added to the Dremio application classpath (at the front)
classpath-prefix: []
}
watchdog: {
# how long to wait for server reply before considering it failed
poll.timeout: 1000ms
# interval between two server polls
poll.interval: 10000ms
# how many consecutive failed attempts before killing server
missed.polls.before.kill: 6
# maximum attempts at killing server
max.kill.attempts: 10
# how long before reattempting killing server
kill.reattempt.interval: 1000ms
}
}
}
# the zookeeper quorum for the cluster
zookeeper: "localhost:"${services.coordinator.master.embedded-zookeeper.port}
zk.client.session.timeout: 90000
# These system properties are listed here to allow substitution of system property values for DAC Web SSL properties
# listed in services.web.ssl section. Currently we consider only the system properties listed in this file for
# substitution.
javax.net.ssl {
keyStoreType: "",
keyStore: "",
keyStorePassword: "",
keyPassword: "",
trustStoreType: "",
trustStore:"",
trustStorePassword: ""
}
registration.publish-host: ""
Was this page helpful?
Glad to hear it! Thank you for your feedback.
Sorry to hear that. Thank you for your feedback.