• perl 多进程抽取oracle数据


    #!/usr/bin/perl 
    use DBI;
    use Parallel::ForkManager;
    use Encode;
    use HTTP::Date qw(time2iso str2time time2iso time2isoz);
    my $dbName = 'serv';
    my $dbUser = 'system';
    my $dbUserPass = 'oracle';
    
    ##########################################
    
    sub printlog
    {
      my ($LogInfo)= @_;
      my $CurrTime = time2iso(time());                   # 当前时间
      if(!defined($LogInfo) ){$LogInfo="";}
      my $StrLog="【${CurrTime}】 	 ${LogInfo} 
    "; 
      
      print $StrLog;
      #print LOGFILE $StrLog;
      };
      
    sub Exportdata{
    my $dbName = 'serv';
    my $dbUser = 'system';
    my $dbUserPass = 'oracle';
    my $dbh = DBI->connect("dbi:Oracle:$dbName", $dbUser, $dbUserPass) or die "can't connect to database ";
    my $table_name= shift; 
    my $hostSql = qq{select COLUMN_NAME from dba_tab_columns where owner=upper('$dbUser') and table_name='$table_name' order by column_id};
    my $DW_DATA_DT ="";
    my $datafile="$table_name.txt";
    my @lstRlst =();
    my @lstRlst1=();
    my ($COLUMN_NAME);
    my $selStmt = $dbh->prepare($hostSql);
    $selStmt->bind_columns(undef, $COLUMN_NAME);
    $selStmt->execute();
    while( $selStmt->fetch() ){
      print "$COLUMN_NAME
    ";
    push  (@lstRlst1 ,$COLUMN_NAME);
      }
    my @lstRlst =  @lstRlst1; 
    print @lstRlst;
    print "
    "; 
    my $exportOracleSql="SELECT ";  #数据导出的sql
    for (my $m=0;$m<@lstRlst + 0 ;$m++){
     if  ($m != @lstRlst + 0 - 1){
      $exportOracleSql = "$exportOracleSql trim($lstRlst[$m])".", "
    }
    else{
    $exportOracleSql = "$exportOracleSql trim($lstRlst[$m])"}
    print "$exportOracleSql
    ";
    }
    my $exportOracleSql="$exportOracleSql from $dbUser.$table_name";
    
    
    	    printlog "开始导出数据!";
    	    my $exportsql=$exportOracleSql;
    	    if($exportsql eq "error"){
    	    	return -1;
    	    	}
    	    my $format_sql="alter session set nls_date_format='yyyy-mm-dd'";
    	    my $stmt=$dbh->prepare($format_sql);
    	    unless ($stmt){
    			printlog "
    执行prepare SQL语句出错:
    ";
    			printlog $DBI::errstr; 
    			return -1;
    			}
    			$stmt->execute;
    			if ($dbh->err) {
    			printlog "
    执行SQL语句出错:
    "; 
    			printlog $DBI::errstr;
    			return -1;
    			}
    	     $stmt=$dbh->prepare($exportsql);
    	    unless ($stmt){
    			printlog "
    执行prepare SQL语句出错:
    ";
    			printlog $DBI::errstr;
    			return -1;
    		}
    	       $stmt->execute;
    		if ($dbh->err) {
    			printlog "
    执行SQL语句出错:
    "; 
    			printlog $DBI::errstr;
    			
    			return -1;
    		}
    	     my $row=0;
    	     my $size=0;
    	     my $curtime;
    
    	     
    	     
    	     my $writeflagsql;
    	     my $tmpstr="";
    	     $row=0;
    	     my $m=0;              
     open(DATAFILE,">", $datafile) || die (print "Open DATA file failed!!!
    ");
    	     while(my $Rows = $stmt->fetchrow_arrayref){
    	     	$m=0;
    	     	$tmpstr="";
    	     	foreach(@$Rows){
    	     		$tmpstr=$tmpstr.$Rows->[$m]."|";
    	     		$m++;
    	     	}
    	     	print DATAFILE encode_utf8($tmpstr.$DW_DATA_DT)."
    ";
    	     	#print DATAFILE $tmpstr.$DW_DATA_DT."
    ";
    	     	$row++;
    	     	if(($row%10000) == 0){
    	     		printlog "已导出数据$row条!";
    	     	}	     	     	
    	    }
         	
            	$stmt->finish;
            #	print FLAGFILE $datafile,"
    ";
            #	print FLAGFILE $row,"
    ";
            	close(DATAFILE);
            #  close(FLAGFILE);
    
            	$curtime=time2iso(time());
              printlog "数据已成功导出!";
              printlog "一共导出数据${row}条";
              $selStmt->finish;
              $dbh->disconnect;      
              return 1;	
             
    	};
    
    my $dbh = DBI->connect("dbi:Oracle:$dbName", $dbUser, $dbUserPass) or die "can't connect to database ";
    
    my $hostSql = qq{select table_name from user_tables };
    
    my @lstRlst2=();
    my ($TABLE_NAME);
    my $selStmt = $dbh->prepare($hostSql);
    $selStmt->bind_columns(undef, $TABLE_NAME);
    $selStmt->execute();
    while( $selStmt->fetch() ){
      print "$TABLE_NAME
    ";
    push  (@lstRlst2 ,$TABLE_NAME);
      }
    print @lstRlst2;
    print "
    ";
    $selStmt->finish;
    $dbh->disconnect;
    ##自定义表,默认整个SCHEMA
    my @lstRlst2=(T1,T2,T3,T4);
    my $pm = Parallel::ForkManager->new(30);
      LINKS:
    foreach (@lstRlst2){
      $pm->start and next LINKS; # do the fork
      &Exportdata($_);    
      $pm->finish; # do the exit in the child process
    };
     $pm->wait_all_children;

  • 相关阅读:
    kubernetes集群部署
    centos7通过yum安装mysql,并授权远程连接
    查看mysql主从配置的状态及修正 slave不启动问题
    ios 企业发布
    centos 安装 pip
    前端优化:DNS预解析提升页面速度
    apache mesos 安装
    Oboe 提升web 用户体验以及性能
    webpack 多entry 配置
    webpack es6支持配置
  • 原文地址:https://www.cnblogs.com/hzcya1995/p/13349542.html
Copyright © 2020-2023  润新知