mongo 增量同步之 MongoShake(2) kafka json to python json

目的: MongoShake 数据同步到kafka(json格式),是带有{"Name":"key","Value":"123"} 的json, 实际上这个json 为 {"key":"123"}

转换如下:

import json

'''
k:节点name
v:节点value
cn:当前节点
'''


def setNode(k, v, cn):
    pass
    if type(v) == list:  # 无论是map还是list,都是list封装
        # get type
        if v:
            v_type = type(v[0])  # 子节点的type
            if v_type == dict:  # dict
                c_value = {}
                for v1 in v:
                    k2 = v1["Name"]
                    v2 = v1["Value"]
                    # v_value[v1["Name"]] = v1["Value"]
                    setNode(k2, v2, c_value)
                cn[k] = c_value
            if v_type == list:  # list
                c_value_dict = {}  # list_dict
                c_value_list = []  # list list
                for v2 in v:  # list 子节点可能为list,和 dict
                    if type(v2) == dict:
                        # v2 is dict
                        k2 = v2["Name"]
                        v2 = v2["Value"]
                        setNode(k2, v2, c_value_dict)
                    if type(v2) == list:
                        # # TODO if list in dict,here just add it
                        # c_value_list.append(v2)
                        pass
                        set_list_list(v2, c_value_list)
                if c_value_list:
                    cn[k] = c_value_list
                if c_value_dict:
                    cn[k] = c_value_dict
            if (not v_type == list) and (not v_type == dict):  # 子节点的节点为普通类型
                cn[k] = v
    else:  # 普通节点
        cn[k] = v


'''
nodelist: list 节点
cnode: 当前节点_list
'''


def set_list_list(nodelist, cnode):
    c_value_dict = {}  # list_dict
    c_value_list = []  # list list
    c_value_other = []  # list other
    for v2 in nodelist:  # list 子节点可能为list,和 dict
        if type(v2) == dict:  # list dict
            # v2 is dict
            k2 = v2["Name"]
            v2 = v2["Value"]
            setNode(k2, v2, c_value_dict)
        if type(v2) == list:  # list list
            set_list_list(v2, c_value_list)
        if (not type(v2) == list) and (not type(v2) == dict):
            c_value_other.append(v2)
    if c_value_list:
        cnode.append(c_value_list)
    if c_value_dict:
        cnode.append(c_value_dict)
    if (not c_value_dict) or (not c_value_list):
        cnode.append(c_value_other)


'''
dict to dict
'''


def json_json(tb_value_json_dict):
    etl_dict = {}
    for kv in tb_value_json_dict:
        setNode(kv["Name"], kv["Value"], etl_dict)
    return etl_dict


"""
    json_str to python dict or list
"""


def json_to_json(tb_value_json_str):
    etl_dict = {}
    etl_list = []
    tb_value_json = json.loads(tb_value_json_str)
    if type(tb_value_json) == list:
        for tb_value_dict in tb_value_json:
            node_dict = {}
            for kv in tb_value_dict:
                setNode(kv["Name"], kv["Value"], node_dict)
            if node_dict:
                etl_list.append(node_dict)
        return etl_list
    if type(tb_value_json) == dict:
        for kv in tb_value_json:
            setNode(kv["Name"], kv["Value"], etl_dict)
        return etl_dict


if __name__ == '__main__':
    # start_kafkaListener()
    value_dict = {"ts": 7012463593182461954, "v": 2, "op": "u", "ns": "sea.test",
                  "o": [[{"Name": "_id", "Value": "6150239397b76701bac32d04"},
                         {"Name": "sea", "Value": "lshan523@163.com"},
                         {"Name": "age", "Value": 22},
                         {"Name": "mapmap",
                          "Value": [{"Name": "sea",
                                     "Value": [{"Name": "ss1", "Value": "ss1v"}, {"Name": "ss2", "Value": "ss2v"}]},
                                    {"Name": "sa", "Value": "s"}]},
                         {"Name": "idList", "Value": ["21", "22", "23"]},
                         {"Name": "idListList", "Value": [["1", "2", "3", "4"], ["21", "22", "23"]]},
                         {"Name": "iListdict",
                          "Value": [[{"Name": "lld", "Value": "lldv"}], [{"Name": "lld1", "Value": "lldv1"}]]}
                         ]]
                  }
    tb_values = value_dict["o"]
    print(tb_values)
    print("=" * 100)
    etl = json_json(tb_values[0])
    print(etl)
    print("*" * 100)
    etl = json_to_json(json.dumps(tb_values))
    print(etl)

result :

[[{'Name': '_id', 'Value': '6150239397b76701bac32d04'}, {'Name': 'sea', 'Value': 'test32'}, {'Name': 'age', 'Value': 22}, {'Name': 'mapmap', 'Value': [{'Name': 'sea', 'Value': [{'Name': 'ss1', 'Value': 'ss1v'}, {'Name': 'ss2', 'Value': 'ss2v'}]}, {'Name': 'sa', 'Value': 's'}]}, {'Name': 'idList', 'Value': ['21', '22', '23']}, {'Name': 'idListList', 'Value': [['1', '2', '3', '4'], ['21', '22', '23']]}, {'Name': 'iListdict', 'Value': [[{'Name': 'lld', 'Value': 'lldv'}], [{'Name': 'lld1', 'Value': 'lldv1'}]]}]]
====================================================================================================
{'_id': '6150239397b76701bac32d04', 'sea': 'test32', 'age': 22, 'mapmap': {'sea': {'ss1': 'ss1v', 'ss2': 'ss2v'}, 'sa': 's'}, 'idList': ['21', '22', '23'], 'idListList': [['1', '2', '3', '4'], ['21', '22', '23']], 'iListdict': [{'lld': 'lldv'}, ['lldv'], {'lld1': 'lldv1'}, ['lldv1']]}
****************************************************************************************************
[{'_id': '6150239397b76701bac32d04', 'sea': 'test32', 'age': 22, 'mapmap': {'sea': {'ss1': 'ss1v', 'ss2': 'ss2v'}, 'sa': 's'}, 'idList': ['21', '22', '23'], 'idListList': [['1', '2', '3', '4'], ['21', '22', '23']], 'iListdict': [{'lld': 'lldv'}, ['lldv'], {'lld1': 'lldv1'}, ['lldv1']]}]