php-Curl扩展一个简单示例-爬取新闻网站数据

**

本文章只是给一个简单的思路作为参考，当前脚本仅适用于指定网站，因为每个网站的结构都不一样，不可能做到通用。大家可以结合自己要爬取的新闻网站结构做出调整。

**

// php最大执行时间设置为：半个小时，php原来默认为30秒，爬不完
ini_set('max_execution_time', '1800');
// 爬取页面全部数据
function curlGetData($url)
{
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt ($ch,  CURLOPT_HEADER,  false);
    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
    curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
    $response = curl_exec($ch);
    curl_close($ch);
    $response = str_replace('gb2312','utf-8',$response);
    $response =  iconv("gb2312","utf-8//IGNORE",$response);
    return $response;
}

// 依次循环新闻id读取新闻详情信息，需要栏目id,以及url
  // 编写一个函数实现新闻详情信息读取
function getNewsInfo($id, $cateid){
    $url = "网站url地址/news_view.asp?id={$id}";
    $result = curlGetData($url);
    $pattern = '/
[\s\S]*
/is';
    preg_match_all($pattern, $result, $result);
    $result = $result[0][0];  // 文章所有内容
    // 依次获取信息
    $newsInfo = [];
    $newsInfo['id'] = $id;
    // 获取新闻标题
    $pattern = '/
[\s\S*]+<\/h2>/is';
    preg_match($pattern, $result, $newsInfo['title']);
    $newsInfo['title'] = str_replace('
', '', $newsInfo['title']);
    $newsInfo['title'] = str_replace('
', '', $newsInfo['title']);
    $newsInfo['title'] = $newsInfo['title'][0];
    // 获取新闻来源
    $pattern = '/来源：[\w\x{4e00}-\x{9fa5}]+/ius';
    preg_match($pattern, $result, $newsInfo['nfrom']);
    $newsInfo['nfrom'] = str_replace('来源：', '', $newsInfo['nfrom']);
    $newsInfo['nfrom'] = $newsInfo['nfrom'][0];
    // 获取新闻作者
    $pattern = '/作者：[\w\x{4e00}-\x{9fa5}]+/ius';
    preg_match($pattern, $result, $newsInfo['author']);
    $newsInfo['author'] = str_replace('作者：', '', $newsInfo['author']);
    $newsInfo['author'] = $newsInfo['author'][0];
    // 获取新闻发布时间
    $pattern = '/发布：\d{2,4}\/\d{1,2}\/\d{1,2}/ius';
    preg_match($pattern, $result, $newsInfo['createtime']);
    $newsInfo['createtime'] = str_replace('发布：', '', $newsInfo['createtime']);
    $newsInfo['createtime'] = $newsInfo['createtime'][0];
    // 新闻栏目id
    $newsInfo['cateid'] = $cateid;
    // 点击量
    $pattern = '/点击：\d*/is';
    preg_match($pattern, $result, $newsInfo['click']);
    $newsInfo['click'] = intval(str_replace('点击：', '', $newsInfo['click'][0]));
    // 新闻内容
    $pattern = '/
[\s\S]*
/is';
    preg_match($pattern, $result, $newsInfo['content']);
    $newsInfo['content'] = str_replace('
', '', $newsInfo['content'][0]);
    $newsInfo['content'] = str_replace('
', '', $newsInfo['content']);
    $newsInfo['content'] = substr($newsInfo['content'], 0, strrpos($newsInfo['content'], '
'));
    // 图片url地址
    $pattern = '/src=\"\/uploadfile\/[\w.]+\"/is';
    preg_match_all($pattern, $newsInfo['content'], $newsimage);
    $newsimage = $newsimage[0];
    $newsimage = str_replace('src="', '该网站地址', $newsimage);
    $newsimage = str_replace('"', '', $newsimage);
    foreach($newsimage as $val)
    {
        $picname = str_replace('新闻网站地址/uploadfile/', '', $val);
        $im = file_get_contents($val);
        file_put_contents('./uploadfile/' . $picname, $im);
        // crabImage($val, './uploadfile/', $filename);
    }

    return $newsInfo;
}


// 输入一个栏目id实现全部爬取
function curlGetByCateId($cid){
    $url = "新闻网站地址/news_category.asp?id={$cid}";
    $response = curlGetData($url);
    $pattern = '/\d+/is';
    preg_match_all($pattern, $response, $response);
    $pageTotal = ceil(intval(substr($response[0][0], 18)) / 20);  // 该栏目下新闻总页数
    // 依次循环页码，获取该栏目所有新闻的id
    $newsId = [];  // 保存所有的新闻id
    for ($i = 1; $i <= $pageTotal; $i++)
    {
        $url = "新闻网站地址/news_category.asp?id={$cid}&page={$i}";
        $response = curlGetData($url);
        $pattern = '/href="news_view.asp\?id=\d+/is';
        preg_match_all($pattern, $response, $response);
        $response = $response[0];
        foreach ($response as $k => $v) {
            $response[$k] = intval(substr($v, 23));
        }
        $newsId = array_merge($response, $newsId);
    }


    // 连接数据库
    $pdo = new PDO('mysql:host=主机地址;dbname=curldemo;charset=utf8', '数据库用户名', '密码');
    $pdo->exec('set names utf8');

    $datas = [];  //存放所有的数据
     foreach($newsId as $val)
     {
         $datas = getNewsInfo($val, $cid);

             $sql = "insert into newsinfo(id,title,nfrom,content,createtime,author,click,cateid) values(:id,:title,:nfrom,:content,:createtime,:author,:click,:cateid)";
             $stmt = $pdo->prepare($sql);
             $stmt->execute($datas);
     }

}

// 根据栏目id爬出来该栏目总页数（因为数据量较大，一下子爬取整站数据可能需要的时间比较长，所以我做了调整，根据栏目id每次只爬取一个栏目的内容，想要一次性爬完的同学可以自行扩展，无非就是多一个循环而已）
// 入口函数
curlGetByCateId(7);
echo "爬完了！！！造作啊！" . '7';

原文链接：https://blog.csdn.net/qq_42195688/article/details/80578107