airbyte 整体架构

内容来自官方文档，基于此图我们可以了解airbyte 的整体运行&&方便问题解决&&环境部署

参考图

 

简单说明

• ui 进行配置，以及通过界面查看同步&&日志
• config store 关于认证&&同步次信息存储
• scheduler 存储关于调度执行情况
• config api 方便ui 进行连接配置
• scheduler api ui 进行调度job 配置
• scheduler 进行数据任务的调度编排以及状态追踪
• worker 具体数据从source 到sink 的操作
• tep store 临时存储（需要数据写入磁盘的场景）

官方参考部署

注意目前关于ha以及集群的部署官方缺少说明，后边可以通过源码分析下

• docker-compose

version: "3.7"

services:

  # hook in case we need to add init behavior

  # every root service (no depends_on) should depend on init

  init:

    image: airbyte/init:${VERSION}

    container_name: init

    command: /bin/sh -c "./scripts/create_mount_directories.sh /local_parent ${HACK_LOCAL_ROOT_PARENT} ${LOCAL_ROOT}"

    environment:

      - LOCAL_ROOT=${LOCAL_ROOT}

      - HACK_LOCAL_ROOT_PARENT=${HACK_LOCAL_ROOT_PARENT}

    volumes:

      - ${HACK_LOCAL_ROOT_PARENT}:/local_parent

  db:

    image: airbyte/db:${VERSION}

    container_name: airbyte-db

    environment:

      - POSTGRES_USER=${DATABASE_USER}

      - POSTGRES_PASSWORD=${DATABASE_PASSWORD}

      - POSTGRES_DB=db-${DATABASE_DB}

    volumes:

      - db:/var/lib/postgresql/data

  seed:

    image: airbyte/seed:${VERSION}

    container_name: airbyte-data-seed

    # Pre-populate the volume if it is empty.

    # See: https://docs.docker.com/storage/volumes/#populate-a-volume-using-a-container

    volumes:

      - data:/app/seed

  scheduler:

    image: airbyte/scheduler:${VERSION}

    container_name: airbyte-scheduler

    environment:

      - DATABASE_USER=${DATABASE_USER}

      - DATABASE_PASSWORD=${DATABASE_PASSWORD}

      - DATABASE_URL=jdbc:postgresql://db:5432/${DATABASE_DB}

      - WAIT_BEFORE_HOSTS=5

      - WAIT_HOSTS_TIMEOUT=45

      - WAIT_HOSTS=db:5432

      - WORKSPACE_ROOT=${WORKSPACE_ROOT}

      - WORKSPACE_DOCKER_MOUNT=${WORKSPACE_DOCKER_MOUNT}

      - LOCAL_DOCKER_MOUNT=${LOCAL_DOCKER_MOUNT}

      - CONFIG_ROOT=${CONFIG_ROOT}

      - TRACKING_STRATEGY=${TRACKING_STRATEGY}

      - AIRBYTE_VERSION=${VERSION}

      - AIRBYTE_ROLE=${AIRBYTE_ROLE:-}

    volumes:

      - /var/run/docker.sock:/var/run/docker.sock

      - workspace:${WORKSPACE_ROOT}

      - ${LOCAL_ROOT}:${LOCAL_ROOT}

      - data:${CONFIG_ROOT}

  server:

    image: airbyte/server:${VERSION}

    container_name: airbyte-server

    environment:

      - DATABASE_USER=${DATABASE_USER}

      - DATABASE_PASSWORD=${DATABASE_PASSWORD}

      - DATABASE_URL=jdbc:postgresql://db:5432/${DATABASE_DB}

      - WAIT_BEFORE_HOSTS=5

      - WAIT_HOSTS_TIMEOUT=45

      - WAIT_HOSTS=db:5432

      - CONFIG_ROOT=${CONFIG_ROOT}

      - TRACKING_STRATEGY=${TRACKING_STRATEGY}

      - AIRBYTE_VERSION=${VERSION}

      - AIRBYTE_ROLE=${AIRBYTE_ROLE:-}

      - WORKSPACE_ROOT=${WORKSPACE_ROOT}

    ports:

      - 8001:8001

    volumes:

      - workspace:${WORKSPACE_ROOT}

      - data:${CONFIG_ROOT}

  webapp:

    image: airbyte/webapp:${VERSION}

    container_name: airbyte-webapp

    ports:

      - 8000:80

    environment:

      - AIRBYTE_ROLE=${AIRBYTE_ROLE:-}

      - AIRBYTE_VERSION=${VERSION}

      - API_URL=${API_URL}

      - IS_DEMO=${IS_DEMO:-}

      - PAPERCUPS_STORYTIME=${PAPERCUPS_STORYTIME:-}

      - TRACKING_STRATEGY=${TRACKING_STRATEGY}

volumes:

  workspace:

    name: ${WORKSPACE_DOCKER_MOUNT}

  data:

    name: ${DATA_DOCKER_MOUNT}

  db:

    name: ${DB_DOCKER_MOUNT}

参考资料

https://docs.airbyte.io/architecture/high-level-view