• 把对象写入Postgresql中


    工作中,遇到把大对象写入Postgresql数据库中

    package com.geni_sage.gdme.cws.dic;
    
    import java.io.BufferedInputStream;
    import java.io.BufferedOutputStream;
    import java.io.ByteArrayInputStream;
    import java.io.ByteArrayOutputStream;
    import java.io.DataInputStream;
    import java.io.File;
    import java.io.FileInputStream;
    import java.io.FileNotFoundException;
    import java.io.FileOutputStream;
    import java.io.IOException;
    import java.io.InputStream;
    import java.io.ObjectInputStream;
    import java.io.ObjectOutputStream;
    import java.io.OutputStream;
    import java.io.Serializable;
    import java.nio.ByteBuffer;
    import java.nio.ShortBuffer;
    import java.nio.channels.Channels;
    import java.nio.channels.ReadableByteChannel;
    import java.sql.Blob;
    import java.sql.Connection;
    import java.sql.DriverManager;
    import java.sql.PreparedStatement;
    import java.sql.ResultSet;
    import java.sql.SQLException;
    import java.sql.Statement;
    import java.util.HashMap;
    
    import com.geni_sage.gdme.cws.kuromoji.dict.CharacterDefinition;
    import com.geni_sage.gdme.cws.kuromoji.dict.ConnectionCosts;
    import com.geni_sage.gdme.cws.kuromoji.dict.Dictionaries;
    import com.geni_sage.gdme.cws.kuromoji.dict.TokenInfoDictionary;
    import com.geni_sage.gdme.cws.kuromoji.dict.UnknownDictionary;
    import com.geni_sage.gdme.cws.kuromoji.trie.DoubleArrayTrie;
    
    /**
     * 将日语词典写入维护库
     * 
     * @author ywf
     * 
     */
    public class WriteObj2DBTest {
        static String pathname = "D:\yuwenfeng\T_Miner_TextMing\gdme\gdm_agent_start\mode\segmentdic\japanesedic";
        static String[] dictionaries = { "tid.dat", "tid_map.dat", "unk.dat",
                "unk_map.dat", "cd.dat", "cc.dat", "dat.dat" };
    
        /**
         * @param args
         * @throws ClassNotFoundException
         * @throws IOException
         */
        public static void main(String[] args) throws IOException,
                ClassNotFoundException {
            String driver = "org.postgresql.Driver";
            String url = "jdbc:postgresql://127.0.0.1:42856/GDM";
            String user = "gsdba";
            String password = "gsdba";
            if (args.length == 4) {
                driver = args[0];
                url = args[1];
                user = args[2];
                password = args[3];
            }
            try {
                Class.forName(driver);
            } catch (ClassNotFoundException e1) {
                System.err.println("链接驱动失败");
            }
            Connection con = null;
            try {
                con = DriverManager.getConnection(url, user, password);
            } catch (SQLException e1) {
                System.err.println("创建连接失败");
            }
            try {
                createTable(con);
                write2db(con, dictionaries);
                // readfromdb(con, "gs_japanesedic", "tid.dat");
                // readfromdb(con, "gs_japanesedic", "tid_map.dat");
                // readfromdb(con, "gs_japanesedic", "unk.dat");
                // readfromdb(con, "gs_japanesedic", "unk_map.dat");
                // readfromdb(con, "gs_japanesedic", "cd.dat");
                // readfromdb(con, "gs_japanesedic", "cc.dat");
                // readfromdb(con, "gs_japanesedic", "dat.dat");
                con.close();
            } catch (SQLException e1) {
                e1.printStackTrace();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    
        /**
         * 建立日语词典表
         * 
         * @param con
         * @throws SQLException
         */
        private static void createTable(Connection con) throws SQLException {
            String sql = "create table gs_japanesedic(dictype varchar(100),japaninfo bytea)";
            Statement stmt = con.createStatement();
            stmt.execute(sql);
            stmt.close();
        }
    
        /**
         * 把流写入维护库
         * 
         * @param con
         * @param path
         * @param dicname
         * @throws Exception
         */
        private static void write2db(Connection con, String[] path)
                throws Exception {
            CharacterDefinition characterDefinition = UnknownDictionary
                    .getInstance(pathname).getCharacterDefinition();
            short[][] cost = ConnectionCosts.getInstance(pathname).getCosts();
            String sql = "insert into  gs_japanesedic values(?,?)";
            PreparedStatement pstmt = con.prepareStatement(sql);
            for (int i = 0; i < path.length; i++) {
                pstmt.setString(1, path[i]);
                byte[] by = null;
                if (path[i].equals("cd.dat") || path[i].equals("cc.dat")) {
                    if (path[i].startsWith("cd")) {// CharacterDefinition
                        by = getBytes(characterDefinition);
                    } else {
                        by = getBytes(cost);// cost
                    }
                } else {
                    InputStream input = new FileInputStream(pathname + "/"
                            + path[i]);
                    by = getbytes(input);
                }
                pstmt.setBytes(2, by);
                pstmt.addBatch();
            }
            pstmt.executeBatch();
            pstmt.close();
        }
    
        /**
         * convet inputstream into byte[]
         * 
         * @param input
         * @return
         * @throws IOException
         */
        private static byte[] getbytes(InputStream input) throws IOException {
            byte[] by = new byte[input.available()];
            input.read(by);
            return by;
        }
    
        public static DoubleArrayTrie readtrie(InputStream is) throws IOException {
            DoubleArrayTrie trie = new DoubleArrayTrie();
            DataInputStream dis = new DataInputStream(new BufferedInputStream(is));
            int baseCheckSize = dis.readInt();
            int tailSize = dis.readInt();
            ReadableByteChannel channel = Channels.newChannel(dis);
    
            ByteBuffer tmpBaseBuffer = ByteBuffer.allocateDirect(baseCheckSize * 4);
            channel.read(tmpBaseBuffer);
            tmpBaseBuffer.rewind();
            tmpBaseBuffer.asIntBuffer().asReadOnlyBuffer();
    
            ByteBuffer tmpCheckBuffer = ByteBuffer
                    .allocateDirect(baseCheckSize * 4);
            channel.read(tmpCheckBuffer);
            tmpCheckBuffer.rewind();
            tmpCheckBuffer.asIntBuffer().asReadOnlyBuffer();
    
            ByteBuffer tmpTailBuffer = ByteBuffer.allocateDirect(tailSize * 2);
            channel.read(tmpTailBuffer);
            tmpTailBuffer.rewind();
            tmpTailBuffer.asCharBuffer().asReadOnlyBuffer();
    
            is.close();
            return trie;
        }
    
        private static void readfromdb(Connection con, String table, String dictype)
                throws SQLException, IOException, ClassNotFoundException {
            String sql = "select * from  " + table + " where dictype = '" + dictype
                    + "'";
            PreparedStatement stmt = con.prepareStatement(sql);
            ResultSet rs = stmt.executeQuery();
            while (rs.next()) {
                String dic = rs.getString(1);
                byte[] by = rs.getBytes(2);
                InputStream bi = new ByteArrayInputStream(by);
                if (dictype.equals("cd.dat") || dictype.equals("cc.dat")) {
                    if (dictype.startsWith("cd")) {// CharacterDefinition
                        CharacterDefinition characterDefinition = (CharacterDefinition) loadObject(bi);
                    } else {
                        short[][] cost = (short[][]) loadObject(bi);
                    }
    
                } else if (dictype.startsWith("dat")) {
                    readtrie(bi);
                } else if (dictype.equals("tid.dat") || dictype.equals("unk.dat")) {
                    loadDictionary(bi);
                } else {
                    loadTargetMap(bi);
                }
            }
    
        }
    
        protected static void loadDictionary(InputStream is) throws IOException {
            DataInputStream dis = new DataInputStream(is);
            int size = dis.readInt();
            ByteBuffer tmpBuffer = ByteBuffer.allocateDirect(size);
            ReadableByteChannel channel = Channels.newChannel(is);
            channel.read(tmpBuffer);
            is.close();
            tmpBuffer.asReadOnlyBuffer();
        }
    
        protected static void loadTargetMap(InputStream is) throws IOException,
                ClassNotFoundException {
            ObjectInputStream ois = new ObjectInputStream(new BufferedInputStream(
                    is));
            int[][] targetMap = (int[][]) ois.readObject();
            is.close();
        }
    
        /**
         * 根据序列化对象得到byte[]
         * 
         * @param obj
         * @return
         * @throws IOException
         */
        public static byte[] getBytes(Serializable obj) throws IOException {
            ByteArrayOutputStream bout = new ByteArrayOutputStream();
            ObjectOutputStream out = new ObjectOutputStream(bout);
            out.writeObject(obj);
            out.flush();
            byte[] bytes = bout.toByteArray();
            bout.close();
            out.close();
            return bytes;
        }
    
        public static Object getObject(byte[] bytes) throws IOException,
                ClassNotFoundException {
            ByteArrayInputStream bi = new ByteArrayInputStream(bytes);
            ObjectInputStream oi = new ObjectInputStream(bi);
            Object obj = oi.readObject();
            bi.close();
            oi.close();
            return obj;
        }
    
        public static Object loadObject(InputStream in) throws IOException,
                ClassNotFoundException {
            ObjectInputStream oi = new ObjectInputStream(in);
            Object obj = oi.readObject();
            in.close();
            oi.close();
            return obj;
        }
    }
  • 相关阅读:
    50.2 Django 连接MySQL,django orm 数据库(models)操作
    50.1 Django 静态资源配置 static && form表单和 request对象
    JS备忘
    Python 单元测试 生产HTML测试报告
    python 单元测试 执行测试
    pycharm 安装插件
    Python 单元测试 实战演练
    Python 单元测试
    Pycharm 使用备忘
    Python 异常
  • 原文地址:https://www.cnblogs.com/yuwenfeng/p/4112953.html
Copyright © 2020-2023  润新知