laravel php 几十万数据导出excel 分批分页导出

目录

需求

问题

解决

---

需求

导出几十万左右的数据导excel表

问题

1. 使用phpexcel等插件，碰到数据量大很慢，可能能花半个小时以上
2. 数据量大查询慢
3. 内存不足
4. 执行超时

解决

1. 使用原生csv导出
2. 设置脚本超时和内存，进行加大内存，不限制超时时间
3. 进行分页查询

    /**
     * 分批导出
     * @param Request $request
     */
    public function bpOut(Request $request){
        error_reporting(E_ALL);
        ini_set('display_errors', TRUE);
        ini_set('display_startup_errors', TRUE);
        ini_set("memory_limit", "2048M");
        set_time_limit(0);
        $lot  = $request->get('lot');
        $type = $request->get('type');
        if($type == 'increment'){
            $filename ="订单日增量表格_".$lot.".csv";
            $count = DB::table('orders_temporary')
                ->where('lot', $lot)
                ->Where(function ($query) {
                    $query->where(function ($query){
                        $query->where('status', 3);
                    })->orWhere(function ($query) {
                        $query->where('status', 2);
                    });
                })->distinct('order_sn')->orderBy('id','asc')->count();
        }else{
            $filename ="上传结果表格_".$lot.".csv";
            $count = $orders = DB::table('orders_temporary')->where('lot', $lot)->count();
        }
        $limit = 10000;//限制查询
        $page =ceil($count/$limit);//页数
        header('Content-Description: File Transfer');
        header('Content-Type: application/vnd.ms-excel');
        header('Content-Disposition: attachment; filename="'. $filename .'"');
        header('Expires: 0');
        header('Cache-Control: must-revalidate');
        header('Pragma: public');
        $fp = fopen('php://output', 'a');//打开output流
        ob_clean();
        $title = ['订单编号', '旺旺ID', '用户昵称', '手机号', 'id号', '信息', '原因'];
        mb_convert_variables('GBK', 'UTF-8', $title);
        fputcsv($fp, $title);
        for ($i=0; $i<$page; $i++){
            $offset = $i*$limit;
            if($type == 'increment'){
                $dbs = DB::table('orders_temporary')
                    ->where('lot', $lot)
                    ->Where(function ($query) {
                        $query->where(function ($query){
                            $query->where('status', 3);
                        })->orWhere(function ($query) {
                            $query->where('status', 2);
                        });
                    })->distinct('order_sn')->offset($offset)->orderBy('id','asc')->limit($limit)->get()->toArray();
            }else{
                $dbs = DB::table('orders_temporary')->where('lot', $lot)->offset($offset)->orderBy('id','asc')->limit($limit)->get()->toArray();
            }
            foreach ($dbs as $db){
                $db->order_sn= $this->numToString($db->order_sn);
                $db->wangwang_id = $db->wangwang_id;
                $db->username = $db->username;
                $db->phone = $db->phone;
                $db->id_number = $this->numToString($db->id_number);
                $db->status_messgae = $db->status_messgae;
                $db->message = $db->message;
                $data=[$db->order_sn,$db->wangwang_id,$db->username,$db->phone,$db->id_number,$db->status_messgae,$db->message];
                mb_convert_variables('GBK', 'UTF-8', $data);
                fputcsv($fp, $data);
            }
            unset($dbs);
        }
        ob_flush();
        flush();
        fclose($fp);
        exit();
    }