dockerfile:
FROM founder/mysql:5.7 RUN apt-get update && apt-get update ; apt-get install -y parallel && chown -R mysql:mysql /docker-entrypoint-initdb.d ADD init.sh /docker-entrypoint-initdb.d/ COPY wisekg.tar.gz /docker-entrypoint-initdb.d/ COPY kb.tar.gz /docker-entrypoint-initdb.d/ ADD mysqld.cnf /etc/mysql/mysql.conf.d/
init.sh:
#!/bin/bash whoami tar zxvf /docker-entrypoint-initdb.d/wisekg.tar.gz -C /docker-entrypoint-initdb.d mkdir /docker-entrypoint-initdb.d/imdb/split cd /docker-entrypoint-initdb.d/imdb/split for i in ../*.txt ; do echo $i ; split -d -a 6 -l 100000 -- $i `basename $i .txt`.; ls|grep `basename $i .txt`|xargs -n1 -i{} mv {} {}.txt; rm -fv $i; done for i in `ls *.*|sed 's/^[^.]+.//'|sort -u` ; do mkdir ../split-$i for j in $i ; do mv $j ../split-$i/$i ; done done cd /docker-entrypoint-initdb.d/imdb/ time ( mysql -uroot -p"$MYSQL_ROOT_PASSWORD" -e 'drop database if exists $MYSQL_DATABASE;create database $MYSQL_DATABASE;set global FOREIGN_KEY_CHECKS=0;'; (echo "SET FOREIGN_KEY_CHECKS=0;";cat ./*.sql) | mysql -uroot -p"$MYSQL_ROOT_PASSWORD" "$MYSQL_DATABASE" ; parallel -j32 'mysqlimport --use-threads=32 -uroot -p"$MYSQL_ROOT_PASSWORD" "$MYSQL_DATABASE" /docker-entrypoint-initdb.d/imdb/{}/*.txt' ::: split-* ) whoami tar zxvf /docker-entrypoint-initdb.d/kb.tar.gz -C /docker-entrypoint-initdb.d mkdir /docker-entrypoint-initdb.d/imdbkb/split cd /docker-entrypoint-initdb.d/imdbkb/split for i in ../*.txt ; do echo $i ; split -d -a 6 -l 100000 -- $i `basename $i .txt`.; ls|grep `basename $i .txt`|xargs -n1 -i{} mv {} {}.txt; rm -fv $i; done for i in `ls *.*|sed 's/^[^.]+.//'|sort -u` ; do mkdir ../split-$i for j in $i ; do mv $j ../split-$i/$i ; done done cd /docker-entrypoint-initdb.d/imdbkb/ time ( mysql -uroot -p"$MYSQL_ROOT_PASSWORD" -e 'drop database if exists kb;create database kb;set global FOREIGN_KEY_CHECKS=0;'; (echo "SET FOREIGN_KEY_CHECKS=0;";cat ./*.sql) | mysql -uroot -p"$MYSQL_ROOT_PASSWORD" kb ; parallel -j32 'mysqlimport --use-threads=32 -uroot -p"$MYSQL_ROOT_PASSWORD" kb /docker-entrypoint-initdb.d/imdbkb/{}/*.txt' ::: split-* )
mysql.cnf
#Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA # # The MySQL Server configuration file. # # For explanations see # http://dev.mysql.com/doc/mysql/en/server-system-variables.html [mysqld] pid-file = /var/run/mysqld/mysqld.pid socket = /var/run/mysqld/mysqld.sock datadir = /var/lib/mysql #log-error = /var/log/mysql/error.log # By default we only accept connections from localhost #bind-address = 127.0.0.1 # Disabling symbolic-links is recommended to prevent assorted security risks symbolic-links=0 innodb_buffer_pool_size = 4G innodb_write_io_threads = 64 innodb_read_io_threads = 64 innodb_thread_concurrency=0 innodb_flush_method=O_DIRECT #innodb_io_capacity=2000 #innodb_io_capacity_max=4000 max_allowed_packet=2G skip-log-bin sync_binlog=0 secure_file_priv='' innodb_flush_log_at_trx_commit = 0 innodb_log_buffer_size = 1G innodb_log_file_size = 3G innodb_autoinc_lock_mode=2 innodb_doublewrite=0 innodb_old_blocks_pct=5 skip-innodb_adaptive_hash_index secure_file_priv =