用curl抓取数据

<?php require_once('functions.php'); require_once('config.php'); /*json_decode*/ if(!function_exists('json_encode')) { include_once('JSON.php'); $GLOBALS['JSON_OBJECT'] = new Services_JSON(); function json_encode($value) { return $GLOBALS['JSON_OBJECT']->encode($value); } function json_decode($value) { return $GLOBALS['JSON_OBJECT']->decode($value); } } /*读取单词*/ $starttime = time(); $sql = "SELECT w_id,w_caption FROM ".DB_PRE."wtotal ORDER BY w_id ASC "; $query = mysql_query($sql); $wordids = array(); $wordsdata = array(); while($row=mysql_fetch_array($query,MYSQL_ASSOC)){ $wordids[] = $row['w_id']; $wordsdata[] = $row['w_caption']; } /*入库*/ $length = count($wordsdata); for($i=0;$i<$length;$i++){ $specialchars = array('à'=>'%C3%A0','â'=>'%C3%A2','é'=>'%C3%A9','è'=>'%C3%A8', 'ê'=>'%C3%AA','ë'=>'%C3%AB','î'=>'%C3%AE','ï'=>'%C3%AF','ô'=>'%C3%B4','û'=>'%C3% BB', 'ù'=>'%C3%B9','ü'=>'%C3%BC','ç'=>'%C3%A7','œ'=>'%C5%93'); //$wordurl = strtr($wordsdata[$i],$specialchars); /*抓取数据*/ $curl = curl_init(); $word_id = $wordids[$i]; $word = $wordsdata[$i]; /*抓取数据*/ $word = str_replace("/n","",$word); $word = trim($word); $wordurl = "http://www.frdic.com/dict/".$word."/"; curl_setopt($curl,CURLOPT_URL,$wordurl); curl_setopt($curl,CURLOPT_HTTPHEADER,array('Content-Type:application/json;charset=UTF-8','X-Requested-With: XMLHttpRequest','Referer:http://www.frdic.com/dict/')); curl_setopt($curl, CURLOPT_USERAGENT, 'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.13) Gecko/20100914 Firefox/3.5.13 ( .NET CLR 3.5.30729)'); curl_setopt($curl, CURLOPT_HEADER, false); curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); $data = curl_exec($curl); /*法法词典标签*/ $reg_ff = '/<div id="tab_-2">(?:.+?)<input type="hidden" id="status" value="(.+?)" //><//div>/i'; if(preg_match_all($reg_ff,$data,$ff)){ $ffdicttab = $ff[1][0]; }else { $ffdicttab = NULL; } //var_dump($ffdicttab); if($ffdicttab != ''){ $post2="{tabStatus:'".$ffdicttab."'}"; curl_setopt($curl,CURLOPT_URL,"http://www.frdic.com/Services/dicServices.asmx/GetTabExp"); curl_setopt($curl,CURLOPT_POST,true); curl_setopt($curl, CURLOPT_RETURNTRANSFER, true); curl_setopt($curl,CURLOPT_POSTFIELDS,$post2); curl_setopt($curl,CURLOPT_HTTPHEADER,array('Content-Type:application/json;charset=UTF-8','X-Requested-With:XMLHttpRequest','Referer:http://www.frdic.com/')); curl_setopt($curl,CURLOPT_USERAGENT,'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.13) Gecko/20100914 Firefox/3.5.13( .NET CLR 3.5.30729)'); curl_setopt($curl, CURLOPT_HEADER, false); $ffdicdata = curl_exec($curl); $ffdicdata = (array)json_decode($ffdicdata); $ffdicdata = $ffdicdata["d"]; $reg_fff1 = '/<DIV style="PADDING-BOTTOM: 11pt" mce_style="PADDING-BOTTOM: 11pt">(?:.+?)<//DIV>/is'; $reg_fff2 = '/<TBODY><TR style="PADDING-BOTTOM: 7.5pt" mce_style="PADDING-BOTTOM: 7.5pt" vAlign=top>(?:.+?)<//TR><//TBODY>/is'; if(preg_match($reg_fff1,$ffdicdata,$ffheader)){ $ffheader = striptext($ffheader); }else{ $ffheader = NULL; } $pattern = '/<TR style="PADDING-BOTTOM: 7.5pt" mce_style="PADDING-BOTTOM: 7.5pt" vAlign=top>(?:.+?)<//TR>/i'; if(preg_match_all($pattern,$ffdicdata,$ffdict)){ $frenchdict = striptext($ffdict[0]); }else{ $frenchdict = NULL; } if($ffheader != '' && $frenchdict != ''){ $frenchdict = array_merge($ffheader,$frenchdict); $frenchdictsen = implode($frenchdict,"."); $frenchdictsen = addslashes($frenchdictsen); if($frenchdictsen != ''){ $frenchdictsen = addslashes($frenchdictsen); $ckids = array(); $ckq = mysql_query($cksql); while($row=mysql_fetch_array($ckq,MYSQL_ASSOC)){ $ckids[] = $row['w_id']; } if(in_array($word_id,$ckids)){ $sql = "update uchome_wfrenchdict SET frdictsen='".$frenchdictsen."' where w_id=".$word_id; }else{ $sql = "INSERT INTO uchome_wfrenchdict(w_id,frdictsen)VALUES ('".$word_id."','".$frenchdictsen."')"; } } } //var_dump($frenchdictsen); } /*outputing some infomation...*/ if($i%100 == 0){ $currenttime = time(); $totaltime = ($length/($i+1)) * ($currenttime - $starttime); $lefttime = ($totaltime - ($currenttime - $starttime))/60.0; echo "outputing ".$i."item ".$lefttime."mins left!/n"; } /*run several times*/ if($i>=$length){ $i=0; for($j=0;$j<100;$j++) { if($j>=100) { break; } } } } mysql_close($conn); ?>