php抓取程序逻辑代码

• 带缓存的抓取逻辑

public static function crawlContent($url, $encode = true) {
    $file_name = '../cache/'.md5($url);
    if (!file_exists($file_name)) {
        @touch($file_name);
    }
    $content = file_get_contents($file_name);
    if (empty($content)) {
        $content = Request::curl($url);
        if (empty($content)) {
            sleep(1);
            $content = Request::curl($url);
        }
        $encode && $content = iconv("GBK", "UTF-8//IGNORE",$content);
        file_put_contents($file_name, $content);
    }
    return $content;
}

• 不带缓存，直接curl数据

    public static function curlContent($url, $encode = true) {  //不走缓存
            $content = Request::curl($url);
            if (empty($content)) {
                sleep(1);
                $content = Request::curl($url);
            }
            $encode && $content = iconv("GBK", "UTF-8//IGNORE",$content);
        return $content;
    }