mongodb同步ElasticSearch性能测试脚本记录

一、逐条写入性能测试脚本

[root@wlan-cloudserver8 bin]# ./mongo 172.27.8.118:40000

  SHARDING_2:PRIMARY> use admin
  switched to db admin
  SHARDING_2:PRIMARY> db.auth('wlan','wlan'）

  SHARDING_2:PRIMARY> use estestdbnew

  switched to db estestdbnew

SHARDING_2:PRIMARY> for(var i=0;i<1000;i++){db.estestdbnew.insert({name:"new", age:999})}

二、通过命令行批量写入性能测试脚本

var bulk = db.test.initializeUnorderedBulkOp();
for(var i=0;i<100000;i++){bulk.insert( { item: "abc123", defaultQty: 100, status: "A", points: 100 } )}   ---写入十万条
bulk.execute();

三、通过代码批量写入性能测试脚本

package mongodbtest;

import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.TimeZone;

public class BulkWriting
{

    public static void main(String[] args) throws IOException, ParseException
    {

        MongoDBConnector mongoDBConnector = new MongoDBConnector("admin", "admin", "estestdbnew", "172.27.8.110", 27017,
                "test");

        mongoDBConnector.connect();

        // mongoDBConnector.insertone();
        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
        sdf.setTimeZone(TimeZone.getTimeZone("GMT+8"));

        System.out.println("start:" + sdf.format(new Date()));
        mongoDBConnector.bulkwirte();
        System.out.println("finish:" + sdf.format(new Date()));
    }
}

package mongodbtest;

import java.util.ArrayList;
import java.util.List;

import org.bson.Document;
import org.bson.conversions.Bson;

import com.mongodb.MongoClient;
import com.mongodb.MongoCredential;
import com.mongodb.ReadPreference;
import com.mongodb.ServerAddress;
import com.mongodb.bulk.BulkWriteResult;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoCursor;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.InsertOneModel;
import com.mongodb.client.model.WriteModel;

public class MongoDBConnector
{
    private String                        mongodbUserName;
    private String                        mongodbPassWord;
    private String                        mongodbDBName;
    private String                        mongodbServerAddr;
    private int                            mongodbServerport;
    private String                        mongodbCollection;
    private MongoClient                    mongoClient;
    private MongoCollection<Document>    collection;

    public MongoDBConnector(String mongodbUserName, String mongodbPassWord, String mongodbDBName,
            String mongodbServerAddr, int mongodbServerport, String mongodbCollection)
    {
        this.mongodbUserName = mongodbUserName;
        this.mongodbPassWord = mongodbPassWord;
        this.mongodbDBName = mongodbDBName;
        this.mongodbServerAddr = mongodbServerAddr;
        this.mongodbServerport = mongodbServerport;
        this.mongodbCollection = mongodbCollection;
    }

    public void connect()
    {
        ServerAddress serverAddress = new ServerAddress(this.mongodbServerAddr, this.mongodbServerport);
        List<ServerAddress> addrs = new ArrayList<ServerAddress>();
        addrs.add(serverAddress);

        MongoCredential credential = MongoCredential.createScramSha1Credential(this.mongodbUserName, this.mongodbDBName,
                this.mongodbPassWord.toCharArray());

        List<MongoCredential> credentials = new ArrayList<MongoCredential>();
        credentials.add(credential);

        MongoClient mongoClient = new MongoClient(addrs, credentials);
        this.mongoClient = mongoClient;

        MongoDatabase mongoDatabase = mongoClient.getDatabase(this.mongodbDBName)
                .withReadPreference(ReadPreference.secondary());
        MongoCollection<Document> collection = mongoDatabase.getCollection(this.mongodbCollection);
        this.collection = collection;
        return;
    }

    public MongoCursor<Document> find(Bson filter)
    {
        FindIterable<Document> findIterable = this.collection.find(filter);// .batchSize(100000);
        MongoCursor<Document> mongoCursor = findIterable.iterator();

        return mongoCursor;
    }

    public void insertone()
    {
        Document document = new Document("x", 1);
        collection.insertOne(document);
        document.append("x", 2).append("y", 3);
        System.out.println("insert one document has finished!");
    }

    public void bulkwirte()
    {
        List<WriteModel<Document>> writes = new ArrayList<WriteModel<Document>>();
        for (int i = 100000; i < 300000; i++)
        {
            writes.add(new InsertOneModel<Document>(
                    new Document("_id", i).append("char", "Taeln").append("class", "fighter").append("lvl", 4)));
        }

        BulkWriteResult bulkWriteResult = collection.bulkWrite(writes);
        System.out.println(bulkWriteResult);
    }

    public void close()
    {
        this.mongoClient.close();
        return;
    }
}

测试结果：

start:2016-11-22 11:49:50.771
AcknowledgedBulkWriteResult{insertedCount=200000, matchedCount=0, removedCount=0, modifiedCount=0, upserts=[]}
finish:2016-11-22 11:50:25.201

观察ES，大概在mongodb写完3分钟后才完全同步过去

[root@master ~]# curl '172.27.8.132:9200/_cat/indices?v'
health status index                   pri rep docs.count docs.deleted store.size pri.store.size 
green  open   estestdb                  5   1      27826            0     11.6mb          5.8mb 
green  open   logstash-2015.05.20       5   1       4750            0     35.9mb         17.9mb 
green  open   mongodb_meta              5   1     327826            0     56.2mb         28.1mb 
green  open   .marvel-es-1-2016.11.22   1   1      34554           96     33.8mb         16.9mb 
green  open   .marvel-es-1-2016.11.21   1   1     211946          564    203.8mb        101.8mb 
green  open   .marvel-es-1-2016.11.20   1   1     204352          740    201.5mb        100.7mb 
green  open   bank                      5   1       1000            0    885.2kb        442.6kb 
green  open   .kibana                   1   1          5            0     51.7kb         25.8kb 
green  open   .marvel-es-data-1         1   1         20            4      7.4mb          3.7mb 
green  open   .marvel-es-1-2016.11.19   1   1     195921          728    191.4mb         95.7mb 
green  open   .marvel-es-1-2016.11.18   1   1     179399          534    167.2mb         83.6mb 
green  open   .marvel-es-1-2016.11.17   1   1      97604          228     77.9mb         38.9mb 
green  open   shakespeare               5   1     111396            0     37.3mb         18.6mb 
green  open   estestdbnew               5   1     300000            0      9.3mb          4.6mb 
green  open   logstash-2015.05.18       5   1       4631            0     34.6mb         17.3mb 
green  open   logstash-2015.05.19       5   1       4624            0     34.9mb         17.4mb