php 百万数据导出 csv

1. 常规的php excel包需要所有的数据先拿到然后才能生成excel， 当生成超大数据量的excel文件时这显然是会造成内存溢出的，所以考虑使用让PHP边写入输出流边让浏览器下载的形式来完成需求。
   我们通过如下的方式写入PHP输出流

set_time_limit(0);	//防止超时
ini_set("memory_limit", "512M");//防止内存溢出

$fp = fopen('php://output', 'a');

fputs($fp, 'strings');
....
....
fclose($fp)

封装方法

	/**
     * 先设置csv相关的Header头, 然后打开
     * PHP output流, 渐进式的往output流中写入数据, 写到一定量后将系统缓冲冲刷到响应中
     * 避免缓冲溢出
     * searchName projectName team_personname accessNum 
     * csvFileName 导出文件名称
     * perSize 每页导出数量
     * accessNum 总数量
     * 
     */
 public function ExcelFun($searchName=[],$csvFileName,$columns,$Field,$accessNum = 0,$perSize=500000)
    {
     

         if(empty($csvFileName) ||empty($columns)||empty($Field)){
     
             return  json_encode(['status'=>0,'msg'=>"参数有误!"]);
         }
         
		 set_time_limit(0);	//防止超时
		 ini_set("memory_limit", "512M");//防止内存溢出
		
         $csvFileName = $csvFileName . '.csv';

         // 设置好告诉浏览器要下载excel文件的headers

         $pages   = ceil($accessNum / $perSize);
         header('Content-Description: File Transfer');
         header('Content-Type: application/vnd.ms-excel');
         header('Content-Disposition: attachment; filename="'.  $csvFileName .'"');
         header('Expires: 0');
         header('Cache-Control: must-revalidate');
         header('Pragma:public');
         $fp = fopen('php://output', 'a');//打开output流

         // 设置编码
         mb_convert_variables('GBK', 'UTF-8', $columns);

         //将数据格式化为CSV格式并写入到output流中
         fputcsv($fp, $columns);

         for($i = 1; $i <= $pages; $i++) {
     
             // 业务流程
             $accessLog = $this->getArticleAccessLog($searchName,$perSize,$lastId);
             // print_r($accessLog);exit;
             foreach($accessLog as $access) {
     
                 $rowData = [];
                 foreach ($Field as $key => $value) {
     
                     $rowData[] = $access[$value];
                 }
                 mb_convert_variables('GBK', 'UTF-8', $rowData);
                 // 写入数据
                 fputcsv($fp, $rowData);
             }
             unset($accessLog);//释放变量的内存
             //刷新输出缓冲到浏览器
             ob_flush();
             flush();//必须同时使用 ob_flush() 和flush() 函数来刷新输出缓冲。
         }
         fclose($fp);
         exit();
 }

调用封装

    // 
    public function RecordWorkersExcel(){
     

        $searchName = input('parson_name');
        $projectName = input('projectName');
        $team_personname = input('team_personname');
   
        $where = '';
        $point = 0;
        if($searchName){
     
            $point += 1;
            $where .= " parson_name like  '%".$searchName."%'  ";
        }

        if($projectName){
     
            if($point){
     
                $where .= " AND projectName like  '%".$projectName."%' ";
            }else{
     
                $where .= "  projectName like  '%".$projectName."%'";
            }
        }

        if($team_personname){
     
            if($point){
     
                 $where .= " AND team_personname like  '%".$team_personname."%'";
            }else{
     
                 $where .= "  team_personname like  '%".$team_personname."%'";
            }            
        }
        // 搜索条件
        $search = [
            'searchName' => $searchName,
            'projectName'   => $projectName,
            'team_personname'   => $team_personname,
        ];
        /***
         * ,$csvFileName,$columns,$accessNum = 0,$perSize=500000
        */ 
        
        $table  = "laval_record_workers";
        $count  = Db::connect('db_config_laval')->table($table)->where($where)->count();
        $csvFileName = "记工记录导出";
        
        $Explain = [
            'ID','姓名','用户手机号','负责人','工地名称','工地编号','班组名称','班组编号','类型','数量','记录时间','审核通过(1/0)','备注'
        ];

        $Field = [
         'id','parson_name','cell_phone','team_personname','projectName','projectCode','team_name','team_code','tit_id','num','current_time','is_state','content'
        ];

        /***
         * searchName  搜索条件
         * csvFileName 导出文件名称
         * columns   导出表头
         * Field     字段选中
         * accessNum 总条数
         * perSize   每个Excel 导出的数据
        */ 
        return  $this->ExcelFun($searchName,$csvFileName,$Explain,$Field,$accessNum=1,$perSize=500000);

    }

业务流程根据自己的项目需要

    public function getArticleAccessLog($searchName,$num,$lastId=0){
     
        $table  = "laval_record_workers";
        $list   = Db::connect('db_config_laval')->table($table)
                ->where('parson_name','like','%'.$searchName['searchName'].'%') 
                ->where('projectName','like','%'.$searchName['projectName'].'%')
                ->where('team_personname','like','%'.$searchName['team_personname'].'%')
                ->order('id','desc')
                ->limit($lastId,$num)
                ->select();
        foreach ($list as $key => &$value){
     
             $str = "workers_" . $value['tit_id'];
            if(Config($str)){
     
                $value['tit_id']   = COnfig($str);
            }else{
     
                $value['tit_id']   = '未配置';
            }
            $value['is_state'] = $value['is_state'] == 1 ? "是":(empty($value['is_state'])?"等待审核":"否");
            $table  = "laval_team";
            $team   = Db::connect('db_config_laval')->table($table)
                    ->where(['id'=>$value['team_id'],'team_code'=>$value['team_code']]) 
                    ->find();
            $value['team_name'] = $team['team_name'];
            $value['PersonPhone'] = $team['cell_phone'];
        }
        return $list;
    }