mongoShake docker 部署 （mongoshake:2.6.5）

mongoshake:2.6.5  docker 

 1. 下载镜像

sudo docker pull happysea/mongoshake:2.6.5

2. 同步mongo 数据 (192.168.18.176:27018,192.168.18.176:27019,192.168.18.176:27020)到kafka(192.168.18.51:9092)   topic  mongosheketopic

sudo docker run -itd --rm --name=mongoshake --restart unless-stopped  -p9101:9101 -p 9100:9100 
-e tunnel_address=seamgtest@192.168.18.51:9092  
-e log_dir=./logs/   
-e sync_mode=incr    
-e mongo_urls=mongodb://root:root@192.168.18.176:27018,192.168.18.176:27019,192.168.18.176:27020   
-e tunnel=kafka  
-e tunnel_address=mongosheketopic@192.168.18.51:9092　
-e tunnel_kafka_partition_number=1 
-e tunnel_message=json  
-e incr_sync_mongo_fetch_method=oplog  
-v  /opt/docker/mongoshake/logs:/mongo-shake-v2.6.5/logs:rw 
happysea/mongoshake:2.6.5

 2.1  同步到其它地方，自己配置（略）

3. 变量参数

collector.conf 中的配置名， 用 "_" 替换掉点 " . "

eg:   tunnel.address    变为   tunnel_address

然后指定  -e   tunnel_address=topic@192.168.18.51:9092  即可

详见：

master_quorum = '${master_quorum:=false}'
full_sync.http_port = 9101
incr_sync.http_port = 9100
# profiling on net/http/profile
# profiling端口，用于查看内部go堆栈。
system_profile_port = 9200
# global log level: debug, info, warning, error. lower level message will be filter
log.level = '${log_level:=info}'
log.dir = '${log_dir}'
log.flush = '${log_flush:=false}'
sync_mode = '${sync_mode:=incr}'
mongo_urls = '${mongo_urls:=mongodb://root:root@192.168.18.176:27018,192.168.18.176:27019,192.168.18.176:27020}'
# please fill the source config server url if source mongodb is sharding.  #############
mongo_cs_url = '${mongo_cs_url}'
mongo_s_url = '${mongo_s_url}'
# enable source ssl
mongo_ssl_root_ca_file = '${mongo_ssl_root_ca_file}'
tunnel = '${tunnel:=kafka}'
############################################### tunnel.address mgtest1@192.168.18.51:9092 ###################################
tunnel.address = '${tunnel_address:=mongosheketest@192.168.18.51:9092}'
tunnel.message = '${tunnel_message:=json}'
tunnel.kafka.partition_number = '${tunnel_kafka_partition_number:=1}'
# canonical_extended_json
tunnel.json.format = '${tunnel_json_format}'
tunnel.mongo_ssl_root_ca_file =　'${tunnel_mongo_ssl_root_ca_file}'
mongo_connect_mode = '${mongo_connect_mode:=secondaryPreferred}'
filter.namespace.black = '${filter_namespace_black}'
filter.namespace.white = '${filter_namespace_white}'
filter.pass.special.db = '${filter_pass_special_db}'
filter.ddl_enable = '${filter_ddl_enable:=false}'
filter.oplog.gids = '${filter_oplog_gids:=false}'
# 2.4版本以后不需要配置为源端cs的地址。
checkpoint.storage.url = '${checkpoint_storage_url}'
checkpoint.storage.db = '${checkpoint_storage_db:=mongoshake}'
# checkpoint collection,s name.
# checkpoint存储的表的名字，如果启动多个mongoshake拉取同一个源可以修改这个表名以防止冲突。
checkpoint.storage.collection = '${checkpoint_storage_collection:=ckpt_default}'
# set if enable ssl
checkpoint.storage.url.mongo_ssl_root_ca_file = '${checkpoint_storage_url_mongo_ssl_root_ca_file}'
# 大于给定的时间，如果是则会直接报错退出。
checkpoint.start_position = 1970-01-01T00:00:00Z
transform.namespace = '${transform_namespace}'
full_sync.reader.collection_parallel = '${full_sync_reader_collection_parallel:=6}'
# the number of document writer thread in each collection.
# 同一个表内并发写的线程数，例如，8表示对于同一个表，将会有8个写线程进行并发写入。#######
full_sync.reader.write_document_parallel = '${full_sync_reader_write_document_parallel:=8}'
# number of documents in a batch insert in a document concurrence
# 目的端写入的batch大小，例如，128表示一个线程将会一次聚合128个文档然后再写入。#######
full_sync.reader.document_batch_size = '${full_sync_reader_document_batch_size:=128}'
# max number of fetching thread per table. default is 1
# 单个表最大拉取的线程数，默认是单线程拉取。需要具备splitVector权限。
# 注意：对单个表来说，仅支持索引对应的value是同种类型，如果有不同类型请勿启用该配置项！
full_sync.reader.parallel_thread = '${full_sync_reader_parallel_thread:=1}'
# the parallel query index if set full_sync.reader.parallel_thread. index should only has
# 1 field.
# 如果设置了full_sync.reader.parallel_thread，还需要设置该参数，并行拉取所扫描的index，value
# 必须是同种类型。对于副本集，建议设置_id；对于集群版，建议设置shard_key。key只能有1个field。
full_sync.reader.parallel_index = '${full_sync_reader_parallel_index:=_id}'
# drop the same name of collection in dest mongodb in full synchronization
# 同步时如果目的库存在，是否先删除目的库再进行同步，true表示先删除再同步，false表示不删除。
full_sync.collection_exist_drop = '${full_sync_collection_exist_drop:=true}'
# create index option.

# background表示创建后台索引。
full_sync.create_index = '${full_sync_create_index:=none}'
# convert insert to update when duplicate key found
# 如果_id存在在目的库，是否将insert语句修改为update语句。
full_sync.executor.insert_on_dup_update = '${full_sync_executor_insert_on_dup_update:=false}'
# filter orphan document for source type is sharding.
# 源端是sharding，是否需要过滤orphan文档
full_sync.executor.filter.orphan_document = '${full_sync_executor_filter_orphan_document:=false}'
# enable majority write in full sync.
# the performance will degrade if enable.
# 全量阶段写入端是否启用majority write
full_sync.executor.majority_enable = '${full_sync_executor_majority_enable:=false}'
# --------------------------- incrmental sync configuration ---------------------------

#######################################   incr_sync.mongo_fetch_method ###########################################
incr_sync.mongo_fetch_method = '${incr_sync_mongo_fetch_method:=oplog}'

incr_sync.change_stream.watch_full_document = '${incr_sync_change_stream_watch_full_document:=false}'

incr_sync.oplog.gids = '${incr_sync_oplog_gids}'

incr_sync.shard_key = '${incr_sync_shard_key:=collection}' 

incr_sync.shard_by_object_id_whitelist = '${incr_sync_shard_by_object_id_whitelist}' 

incr_sync.worker = '${incr_sync_worker:=8}'

incr_sync.tunnel.write_thread = '${incr_sync_tunnel_write_thread:=8}'
# set the sync delay just like mongodb secondary slaveDelay parameter. unit second.
# 设置目的端的延迟，比如延迟源端20分钟，类似MongoDB本身主从同步slaveDelay参数，单位：秒
# 0表示不启用
incr_sync.target_delay = '${incr_sync_target_delay:=0}'
# memory queue configuration, plz visit FAQ document to see more details.
# do not modify these variables if the performance and resource usage can
# meet your needs.
# 内部队列的配置参数，如果目前性能足够不建议修改，详细信息参考FAQ。
incr_sync.worker.batch_queue_size = '${incr_sync_worker_batch_queue_size:=64}'
incr_sync.adaptive.batching_max_size = '${incr_sync_adaptive_batching_max_size:=1024}'
incr_sync.fetcher.buffer_capacity = '${incr_sync_fetcher_buffer_capacity:=256}'

incr_sync.executor.upsert = '${incr_sync_executor_upsert:=false}'
# oplog changes to Update while Insert found duplicated key (_id or unique-index)
# 如果_id存在在目的库，是否将insert语句修改为update语句。
incr_sync.executor.insert_on_dup_update = '${incr_sync_executor_insert_on_dup_update:=false}'

incr_sync.conflict_write_to = '${incr_sync_conflict_write_to:=none}'

incr_sync.executor.majority_enable = '${incr_sync_executor_majority_enable:=false}'

special.source.db.flag = '${special_source_db_flag}'