dremio 配置文件

因为官方文档对于配置文件的介绍太简单了,比如默认zk 注册的时候会有主机名称的问题，不然我们就需要自己配置
hosts，以下是来自官方代码仓库的一个参考配置，方便使用(注意此配置主要是核心组件的，还有其他组件自己的配置)

配置文件

• 文件位置
  源码的common resources dremio-reference.conf
• 内容

 

## Copyright (C) 2017-2019 Dremio Corporation

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#     http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

#

 

include classpath("dremio-reference-ext.conf")

 

paths: {

 

  # the local path for dremio to store data.

  local: "/tmp/dremio",

 

  # the distributed path Dremio data including job results, downloads, uploads, etc

  dist: "pdfs://"${paths.local}"/pdfs"

 

  # location for catalog database (if master node)

  db: ${paths.local}/db,

 

  spilling: [${paths.local}/spill]

 

  # storage area for the accelerator cache.

  accelerator: ${paths.dist}/accelerator

 

  # staging area for json and csv ui downloads

  downloads: ${paths.dist}/downloads

 

  # stores uploaded data associated with user home directories

  uploads: ${paths.dist}/uploads

 

  # stores data associated with the job results cache.

  results: ${paths.dist}/results

 

  # shared scratch space for creation of tables.

  scratch: ${paths.dist}/scratch

 

}

 

services: {

  node-tag: "",

  coordinator: {

    enabled: true,

 

    # Auto-upgrade Dremio at startup if needed

    auto-upgrade: false,

 

    master: {

      enabled: true,

      # configure an embedded ZooKeeper server on the same node as master

      embedded-zookeeper: {

        enabled: true,

        port: 2181,

        path: ${paths.local}/zk

      }

    },

 

    web: {

      enabled: true,

      port: 9047,

      ssl: {

        # If SSL for communication path between browsers (or REST clients) and Dremio should be enabled.

        enabled: false,

 

        # Allow for auto-generated certificates if keyStore option is not set

        # Auto-generated self-signed certificates are considered insecure, and this

        # option should be set to false in production environment

        auto-certificate.enabled: true

 

        # KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.

        # If needed to be overridden, then change the below properties

 

        # KeyStore type

        keyStoreType: ${javax.net.ssl.keyStoreType},

 

        # Path to KeyStore file

        keyStore: ${javax.net.ssl.keyStore},

 

        # Password to access the keystore file

        keyStorePassword: ${javax.net.ssl.keyStorePassword},

 

        # Password to access the key

        keyPassword: ${javax.net.ssl.keyPassword},

 

        # TrustStore type

        trustStoreType: ${javax.net.ssl.trustStoreType},

 

        # Path to TrustStore file

        trustStore: ${javax.net.ssl.trustStore},

 

        # Password to access the truststore file

        trustStorePassword: ${javax.net.ssl.trustStorePassword}

      },

      auth: {

        type: "internal"

      }

      ui: {

        # Configuration for Intercom

        intercom: {

            enabled: true

            appid:  "@dremio.ui.intercom.appid@"

        }

      }

      tokens: {

        cache: {

          # number of tokens to store locally on this coordinator node

          # (set to 0, if all requests should be made directly to the database of tokens)

          size: 100

          # time (in minutes) after which the token needs to be obtained from the token store

          expiration_minutes: 5

        }

      }

    },

 

    client-endpoint: {

      port: 31010

    },

 

    scheduler: {

      threads: 24

    },

 

    command-pool: {

      enabled: true

      size: 0 # 0 defaults to the machine's number of cores

    },

  },

 

  executor: {

    enabled: true

 

    # enable/disable local cache manager

    # storage space for cache manager

    # control max percentage of disk cache manager db instance and fs mount points can consume

    cache: {

      enabled: true,

      path: {

        db: ${paths.local},

        fs: [${services.executor.cache.path.db}]

      },

      pctquota: {

        db: 70,

        fs: [${services.executor.cache.pctquota.db}]

      },

      ensurefreespace: {

        fs: [10]

      }

    }

  },

 

  fabric: {

    port: 45678,

 

    memory: {

      reservation: 100M

    }

    ssl: {

      # If SSL for communication path between Dremio instances should be enabled.

      enabled: false,

 

      # Allow for auto-generated certificates if keyStore option is not set

      # Auto-generated self-signed certificates are considered insecure, and this

      # option should be set to false in production environment

      auto-certificate.enabled: false,

 

      # KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.

      # If needed to be overridden, then change the below properties

 

      # KeyStore type

      keyStoreType: ${javax.net.ssl.keyStoreType},

 

      # Path to KeyStore file

      keyStore: ${javax.net.ssl.keyStore},

 

      # Password to access the keystore file

      keyStorePassword: ${javax.net.ssl.keyStorePassword},

 

      # Password to access the key

      keyPassword: ${javax.net.ssl.keyPassword},

 

      # TrustStore type

      trustStoreType: ${javax.net.ssl.trustStoreType},

 

      # Path to TrustStore file

      trustStore: ${javax.net.ssl.trustStore},

 

      # Password to access the truststore file

      trustStorePassword: ${javax.net.ssl.trustStorePassword}

    }

  },

 

  flight: {

    enabled: true

    port: 32010

    # Authentication mode to configure the FlightServer Endpoint to use.

    # There are two modes:

    #       - legacy.arrow.flight.auth

    #       - arrow.flight.auth2

    # legacy.arrow.flight.auth is backwards compatible and will be deprecated in the future.

    # arrow.flight.auth2 is the new and preferred Arrow Flight authentication method.

    auth.mode: "arrow.flight.auth2"

 

    ssl: {

      # If SSL for communication path between Flight clients and Dremio should be enabled.

      enabled: false,

 

      # Allow for auto-generated certificates if keyStore option is not set

      # Auto-generated self-signed certificates are considered insecure, and this

      # option should be set to false in production environment

      auto-certificate.enabled: true

 

      # KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.

      # If needed to be overridden, then change the below properties

 

      # KeyStore type

      keyStoreType: ${javax.net.ssl.keyStoreType},

 

      # Path to KeyStore file

      keyStore: ${javax.net.ssl.keyStore},

 

      # Password to access the keystore file

      keyStorePassword: ${javax.net.ssl.keyStorePassword},

 

      # Password to access the key

      keyPassword: ${javax.net.ssl.keyPassword},

 

      # TrustStore type

      trustStoreType: ${javax.net.ssl.trustStoreType},

 

      # Path to TrustStore file

      trustStore: ${javax.net.ssl.trustStore},

 

      # Password to access the truststore file

      trustStorePassword: ${javax.net.ssl.trustStorePassword}

    }

  },

 

  conduit: {

    # If set to 0, a port is automatically allocated (typically in ephemeral range). Otherwise, the configured value

    # is used.

    port: 0

 

    ssl: {

      # If SSL for communication path between Dremio instances should be enabled.

      enabled: ${services.fabric.ssl.enabled},

 

      # Allow for auto-generated certificates if keyStore option is not set

      # Auto-generated self-signed certificates are considered insecure, and this

      # option should be set to false in production environment

      auto-certificate.enabled: ${services.fabric.ssl.auto-certificate.enabled},

 

      # KeyStore and TrustStore settings default to Java keystore and truststore JVM arguments.

      # If needed to be overridden, then change the below properties

 

      # KeyStore type

      keyStoreType: ${services.fabric.ssl.keyStoreType},

 

      # Path to KeyStore file

      keyStore: ${services.fabric.ssl.keyStore},

 

      # Password to access the keystore file

      keyStorePassword: ${services.fabric.ssl.keyStorePassword},

 

      # Password to access the key

      keyPassword: ${services.fabric.ssl.keyPassword},

 

      # TrustStore type

      trustStoreType: ${services.fabric.ssl.trustStoreType},

 

      # Path to TrustStore file

      trustStore: ${services.fabric.ssl.trustStore},

 

      # Password to access the truststore file

      trustStorePassword: ${services.fabric.ssl.trustStorePassword}

    }

  }

 

  # Set up kerberos credentials in server (applicable for both coordinator and executor)

  kerberos: {

    principal: "",

    keytab.file.path: ""

  }

 

  web-admin: {

    enabled: true,

    # Port, bound to loopback interface, on which the daemon responds to liveness HTTP requests (0 == auto-allocated)

    port: 0

  }

 

  jobs: {

    # if jobs server should be enabled in a coorinator

    enabled: true

  }

}

 

provisioning: {

  yarn: {

    jvmoptions: "",

    # list of jars to be added to the main container classpath

    classpath: []

    app {

      # list of jars to be added to the Dremio application classpath

      classpath: []

      # list of jars to be added to the Dremio application classpath (at the front)

      classpath-prefix: []

    }

    watchdog: {

      # how long to wait for server reply before considering it failed

      poll.timeout: 1000ms

      # interval between two server polls

      poll.interval: 10000ms

      # how many consecutive failed attempts before killing server

      missed.polls.before.kill: 6

      # maximum attempts at killing server

      max.kill.attempts: 10

      # how long before reattempting killing server

      kill.reattempt.interval: 1000ms

    }

    nodemanager: {

      # Enable certificate validation when connecting to Yarn Node-manager REST API

      certificate-validation.enabled: true

    }

    enabled:false

  }

}

 

# the zookeeper quorum for the cluster

zookeeper: "localhost:"${services.coordinator.master.embedded-zookeeper.port}

zk.client.session.timeout: 90000

zk.client.retry.unlimited: true

zk.client.retry.limit: -1

zk.client.retry.initial_timeout_ms: -1

 

debug: {

  enabled: false,

  autoPort: false,

  prepopulate: false,

  singleNode: false,

  verboseAccessLog: false,

  allowTestApis: false,

  forceRemote: false,

  useMemoryStorage: false,

  addDefaultUser: false,

  allowNewerKVStore: true,

  # to enable remote debugging of the DremioDaemon running in YARN container

  yarnremote.enabled: false

  # UI Red Screen Of Death

  rsod.enabled: false

  # UI File A Bug option

  bug.filing.enabled: false

  # DX-9126: enable on-idle load shedding

  task.on_idle_load_shed: true

  # DX-23620: enable eager load shedding

  task.eager_load_shed: true

  # DX-9911: enable rescheduling task on unblock

  task.reschedule_on_unblock: true

  # Use election service to elect between multiple master candidates

  # has to be set to false if multiple master candidates

  master.election.disabled: false,

  alloc.max_occupancy_percent: 75,

  alloc.est_heap_buf_size_bytes: 800,

  # Enable caching for distributed storage, it is turned off by default

  dist.caching.enabled: false,

  # Max percent of total available cache space to use when possible for distributed storage

  dist.max.cache.space.percent: 100,

  # Enable asynchronous access for distributed stores when possible

  dist.async.enabled: true,

  # Enable asynchronous access for uploaded data associated with user home directories

  uploads.async.enabled: true,

  # Enable asynchronous access for temporary support stores

  support.async.enabled: true,

  # Enable asynchronous access for the job results cache store

  results.async.enabled: true,

  # Enable asynchronous access for the create table scratch space

  scratch.async.enabled: true,

  # Enable asynchronous access for the json and csv download staging space

  download.async.enabled: true,

  # Enable asynchronous access for the logs store

  logs.async.enabled: true,

  # NoOp cluster coordinator

  noop.cluster.coordinator.enabled: false,

  remote.datastore.rpc_timeout_seconds: 60,

  dist.s3_file_status_check.enabled: true,

}

 

# These system properties are listed here to allow substitution of system property values for DAC Web SSL properties

# listed in services.web.ssl section. Currently we consider only the system properties listed in this file for

# substitution.

javax.net.ssl {

  keyStoreType: "",

  keyStore: "",

  keyStorePassword: "",

  keyPassword: "",

  trustStoreType: "",

  trustStore:"",

  trustStorePassword: ""

}

 

registration.publish-host: ""

一个比较有用的配置

registration.publish-host 是一个比较方便的配置，在zk 注册的时候会使用主机名，我们通过此配置可以解决zk 注册主机名的问题
可以方便执行节点的运行（不需要配置hosts了）