实时同步脚本

前一阵由于公司的项目需要，需要将一个服务网站上的文件库实时同步过来，包括其目录架构和内容，于是我写了一个脚本去在Linux上执行。
现在贴一下我写的脚本代码。

 

// exit;

set_time_limit(0);

$conn=new PDO("mysql:host=xxx;dbname=xxx;","root","xxx");
$conn->exec("SET NAMES 'utf8';");

function writeLog($msg){//记录日志
    if(!$msg) return false;
    $dir = '/data/www/default/temp/';
    $log = 'updatelog';

    $filesize = filesize($dir.'/'.$log);
    if($filesize>3145728){//设置log文件大小为3mb
        rename($dir.'/'.$log,$dir.'/'.$log."".date('YmdH',time()));
    }
    $result = file_put_contents($dir.'/'.$log,(date('Y-m-d h:i:s',time())).'   '.$msg."\r\n",FILE_APPEND);
    return $result;
}
function get_transid(){//取目录id关联表数据
    global $conn;
    $result = $conn->query("select * from transid");
    $row = $result->fetchAll(PDO::FETCH_ASSOC);
    $row_n = array();
    foreach ($row as $key => $value) {
        $row_n[$value['oldid']] = $value['id'];
    }
    $row_n[''] = 0;
    return $row_n;
}
function addnavigation($pid,$arr){//更新category表中的navigation字段
    global $conn;
    if($pid!=0){
        $result = $conn->query("select * from category where cid=".$pid);
        $row = $result->fetch(PDO::FETCH_ASSOC);
        $arr[] = array('cid'=>$row['cid'],'name'=>$row['name']);
        $parentarr = addnavigation($row['pid'],$arr);
        return $parentarr;
    }else{
        return $arr;
    }
}
function writedata($dataArr,$flag=0,$table){//导入对比目录表
    global $conn;
    if(!in_array($table,array('tempdir2','file_compare'))){
        return "tablename is wrong!";
    }
    if($flag==0){//先做清空
        $res = $conn->exec("truncate table ".$table);
        $arr = $conn->query("select count(1) as flag from ".$table);
        $row = $arr->fetch(PDO::FETCH_ASSOC);
        $flag = $row['flag']==0?1:0;
        writedata($dataArr,$flag,$table);
    }else{
        //然后写入
        if($table=='tempdir2'){
            $stmt = $conn->prepare("insert into tempdir2 (id,catalogName,parentId,catalogRoutingNumber) values (:id,:name,:pid,:cnum)");
            foreach ($dataArr as $key => $value) {
                $arr = array(
                    ':name' => $value['catalogName'],
                    ':cnum' => $value['catalogRoutingNumber'],
                    ':id' => $value['id'],
                    ':pid' => $value['parentId']
                );
                $result = $stmt->execute($arr);
            }
        }elseif($table=='file_compare'){
            $stmt = $conn->prepare("insert into file_compare (oldid,CatalogId,title,uploadUserName,uploadDate,newUptime,summary,isvisiableForuser,isdeleted,url) values (:oldid,:pid,:title,:uploadUserName,:uploadDate,:newUptime,:summary,:isvisiable,:isdeleted,:url)");
            $id_arr = get_transid();//取目录id关联表
            foreach ($dataArr as $key => $value) {
                $time = strtotime(substr($value['uploadDate'],0,strlen($value['uploadDate'])-2));
                $newUptime = strtotime(substr($value['newVersionUploaddate'],0,strlen($value['newVersionUploaddate'])-2));
                if(in_array($value['knowledgeCatalogId'],array_keys($id_arr))){
                    $pid = intval($id_arr[$value['knowledgeCatalogId']]);
                }else{
                    $pid = 0;
                }
                $arr = array(
                    ':oldid' => $value['id'],
                    ':pid' => $pid,
                    ':title' => $value['title'],
                    ':uploadUserName' => $value['uploadUserName'],
                    ':uploadDate' => $time,
                    ':newUptime' => $newUptime,
                    ':summary' => $value['summary'],
                    ':isvisiable' => intval($value['isvisiableForuser']),
                    ':isdeleted' => intval($value['isdeleted']),
                    ':yxturl' => $value['knowledgeUrl']
                );
                $result = $stmt->execute($arr);
            }
        }
        return 1;
    }
}

echo date("Y-m-d H:i:s")."运行开始 ———————————————— ";
writeLog("【*****************检查更新开始********************】");

//**********************************************************************************
//获取接口数据
//
// CURL
function curl_post($url,$arr){
    $ch=curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
    curl_setopt($ch, CURLOPT_HTTPHEADER, array("Content-Type: application/json; charset=utf-8"));//important
    curl_setopt($ch, CURLOPT_POST, TRUE);
    curl_setopt($ch, CURLOPT_POSTFIELDS, json_encode($arr));
    $data=curl_exec($ch);
    curl_close($ch);
    return json_decode($data,true);
}
function curl_post_del($url,$arr){
    $ch=curl_init();
    curl_setopt($ch, CURLOPT_URL, $url);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, TRUE);
    curl_setopt($ch, CURLOPT_POST, TRUE);
    curl_setopt($ch, CURLOPT_POSTFIELDS, $arr);
    $data=curl_exec($ch);
    curl_close($ch);
    return json_decode($data,true);
}
function salt(){
    $salt=rand(10000,99999);
    return $salt;
}
$apikey="xxxxxxx";
$secretkey="xxxxxxx";
$salt1=salt();
$salt2=salt();
$signature1=hash('sha256',$secretkey.$salt1);
$signature2=hash('sha256',$secretkey.$salt2);

//获取文件数据
$url1='http://url';
$data1=array(
    'apikey' => $apikey,
    'salt' => $salt1,
    'signature' => $signature1,
    'kngCatalogId' => '',
    'updateDate' => '',
    'knowledgeTypes'=>''
);
$result1=curl_post($url1,$data1);
$new_filedata=$result1['data'];

// 获取目录数据
$url2='http://url';
$data2=array(
    'apikey' => $apikey,
    'salt' => $salt2,
    'signature' => $signature2
);
$result2=curl_post($url2,$data2);
$new_dirdata=$result2['data'];


//**********************************************************************************
//对比本地进行同步

// 写入目录对比表
writedata($new_dirdata,0,'tempdir2');

//对比新旧目录
$compare_sql = "SELECT t2.* from tempdir2 t2 LEFT JOIN tempdir t1 ON t1.id=t2.id && t1.catalogName=t2.catalogName && t1.parentId=t2.parentId && t1.catalogRoutingNumber=t2.catalogRoutingNumber WHERE t1.sid is null";
$compareArr = $conn->query($compare_sql);
$compareRow = $compareArr->fetchAll(PDO::FETCH_ASSOC);

$category_add_arr = array();
$category_update_arr = array();
$navigation_row = array();
if(count($compareRow)>0){
    writeLog("【目录存在差异数据".count($compareRow)."条】");
    //先对关联表进行全部更新
    foreach ($compareRow as $key => $value) {
        $id = $value['id'];
        $catalogName = $value['catalogName'];
        $parentId = $value['parentId'];
        $catalogRoutingNumber = $value['catalogRoutingNumber'];
        $result = $conn->query("select 1 from tempdir where id='".$id."'");
        $row = $result->fetch(PDO::FETCH_ASSOC); 
        if(empty($row)){
            $result_add = $conn->exec("insert into tempdir (id,catalogName,parentId,catalogRoutingNumber) values ('".$id."','".$catalogName."','".$parentId."','".$catalogRoutingNumber."')");//插入对比表
            $result_add2 = $conn->exec("insert into transid (oldid) values ('".$id."')");//插入ID关联表  该表ID就是分类表cid

            if($result_add>0 && $result_add2>0){
                $value["newid"] = $conn->lastInsertId();
                $category_add_arr[] = $value;
            }else{
                writeLog("【关联表或对比表插入失败】");
            }
        }else{
            $result_update = $conn->exec("update tempdir set catalogName='".$catalogName."',parentId='".$parentId."',catalogRoutingNumber='".$catalogRoutingNumber."' where id='".$id."'");//修改对比表
            if($result_update>0){
                $category_update_arr[] = $value;
            }else{
                writeLog("【对比表更新失败】");
            }
        }
    }
    //取关联表数据
    $dirid_arr = get_transid();
    //再更新使用表
    if(!empty($category_add_arr)){//插入目录树表
        writeLog("【新条目".count($category_add_arr)."条】");
        foreach ($category_add_arr as $key => $value) {
            $pid = $dirid_arr[$value['parentId']];
            $cid = $value['newid'];
            $displayorder = $value['catalogRoutingNumber'];
            $displayorder = substr($displayorder,strlen($displayorder)-3,strlen($displayorder));
            $navigation_row[] = array('cid'=>$cid,'pid'=>$pid,'name'=>$catalogName,);
            $result_add_c = $conn->exec("insert into category (cid,pid,name,displayorder,image,navigation,description) values ('".$cid."','".$pid."','".$catalogName."','".$displayorder."','','','')");
            // echo "【插入结果：".$result_add_c."】\r\n";
        }
    }
    if(!empty($category_update_arr)){//修改目录树表
        writeLog("【变动条目".count($category_update_arr)."条】");
        foreach ($category_update_arr as $key => $value) {
            $pid = $dirid_arr[$value['parentId']];
            $id = $dirid_arr[$value['id']];
            $displayorder = $value['catalogRoutingNumber'];
            $displayorder = substr($displayorder,strlen($displayorder)-3,strlen($displayorder));
            $result_update_c = $conn->exec("update category set pid='".$pid."',name='".$value['catalogName']."',displayorder='".$displayorder."' where cid='".$id."'");
            // echo "【修改结果：".$result_update_c."】\r\n";
        }
    }
    foreach ($navigation_row as $key => $value) {//更改navigation字段
        $arr = array();
        if($value['pid']!=0){
            $arr = addnavigation($value['cid'],$arr);
        }else{
            $arr[] = array('cid'=>$value['cid'],'name'=>$value['name']);
        }
        $navigation = addslashes(serialize($arr));
        $conn->exec("update category set navigation='".$navigation."' WHERE cid=".$value['cid']);
    }
}else{
    writeLog("【目录没有新变动】");
}

//**********************************************************************************
//对比是否有删除

//再次对比新旧目录
$delcompare_sql = "SELECT t1.* from tempdir t1 LEFT JOIN tempdir2 t2 ON t1.id=t2.id WHERE t2.sid is null";
$delcompareArr = $conn->query($delcompare_sql);
$delcompareRow = $delcompareArr->fetchAll(PDO::FETCH_ASSOC);
if(count($delcompareRow)>0){
    writeLog("【存在被删除项".count($delcompareRow)."条】");
    foreach ($delcompareRow as $key => $value) {
        $result_del_1 = $conn->exec("delete from tempdir where id='".$value['id']."'");
        $result_delid = $conn->query("select * from transid where oldid='".$value['id']."'");//查询预删除数据在目录树表中对应的id
        $row_delid = $result_delid->fetch(PDO::FETCH_ASSOC);
        if(!empty($row_delid)){
            $result_del_2 = $conn->exec("delete from transid where oldid='".$value['id']."'");
            $result_del_3 = $conn->exec("delete from category where cid=".$row_delid['id']);
        }
    }
}
//至此 目录树 同步完毕


//**********************************************************************************
//文件对比

//先写入临时对照表
writedata($new_filedata,0,'file_compare');

//对比新旧文件表
$compare_sql = "SELECT t2.*,t1.lastedit as lastedit  from file_compare t2 LEFT JOIN wiki_doc t1 ON t1.oldid=t2.oldid && t2.knowledgeCatalogId=t1.cid && t2.isdeleted=t1.isdeleted && t2.isvisiableForuser=t1.visible && t1.title=t2.title WHERE t1.did is null";
$compareArr = $conn->query($compare_sql);
$compareRow = $compareArr->fetchAll(PDO::FETCH_ASSOC);

if(count($compareRow)>0){
    writeLog("【文件表存在差异数据".count($compareRow)."条】");

    //********************************************************************************************************
    //查询被删除的文件，给领一关联网站发送通知
    $delid_arr = array();
    foreach ($compareRow as $key => $value) {
        if($value['isdeleted']==1){
            $delid_arr[] = "'".$value['oldid']."'";
        }
    }
    if(!empty($delid_arr)){
        $id_str = implode(",",$delid_arr);
        $sel_del_sql = "SELECT oldid from doc WHERE isdeleted=0 and oldid in (".$id_str.")";//对比找出此次被删除的文件
        $del_Arr = $conn->query($sel_del_sql);
        $del_Row = $del_Arr->fetchAll(PDO::FETCH_ASSOC);
        if(!empty($del_Row)){
            $report_del_str = '';//已被删除的ID字符串
            foreach ($del_Row as $key => $value) {
                $report_del_str .= $value['oldid'].",";
            }
            $report_del_str = substr($report_del_str,0,strlen($report_del_str)-1);
            $del_data=array('id' => $report_del_str);
            $url_repdel = "url";
            $post_del = curl_post_del($url_repdel,$del_data);//发送通知
            writeLog("【通知xxx已被删除文件ID，共".count($del_Row)."个，id=".$report_del_str."，".$post_del['reMsg']."】");
        }
    }
    //********************************************************************************************************

    //先对关联表进行全部更新
    $stmt_add = $conn->prepare("insert into doc (cid,title,summary,author,time,lastedit,lasteditor,visible,isdeleted,oldid) values (:cid,:title,:summary,:author,:time,:lastedit,:lasteditor,:visible,:isdeleted,:oldid)");//插入doc表
    $stmt_update = $conn->prepare("update doc set cid=:cid,title=:title,summary=:summary,lastedit=:lastedit,lasteditor=:lasteditor,visible=:visible,isdeleted=:isdeleted where oldid=:oldid");//更新doc表
    $stmt_down_url = $conn->prepare("insert into temp_downurl (oid,did,cid,time,filename) values (:oid,:did,:cid,:time,:filename)");

    foreach ($compareRow as $key => $value) {
        //判断插入还是更新
        $result = $conn->query("select did from doc where oldid='".$value['oldid']."'");
        $row = $result->fetch(PDO::FETCH_ASSOC);
        if(empty($row)){
            $arr = array(
                ':oldid' => $value['oldid'],
                ':cid' => intval($value['knowledgeCatalogId']),
                ':title' => $value['title'],
                ':summary' => $value['summary'],
                ':time' => intval($value['uploadDate']),
                ':lastedit' => intval($value['uploadDate']),
                ':author' => $value['uploadUserName'],
                ':lasteditor' => $value['uploadUserName'],
                ':isdeleted' => intval($value['isdeleted']),
                ':visible' => intval($value['isvisiableForuser'])
            );
            $result_add = $stmt_add->execute($arr);
            if($result_add>0){
                writeLog("【wiki_doc表成功插入oldid=".$value['oldid']."】");
            }else{
                writeLog("【wiki_doc表插入失败oldid=".$value['oldid']."】");
            }
            $did = $conn->lastInsertId();
            //向预下载表中添加数据
            $arr_url = array(
                ':oid' => $value['oldid'],
                ':did' => intval($did),
                ':cid' => intval($value['knowledgeCatalogId']),
                ':time' => intval($value['uploadDate']),
                ':filename' => $value['title']
            );
            $result_url = $stmt_down_url->execute($arr_url);
            if($result_url>0){
                writeLog("【预下载表写入成功did=".$did."】");
            }else{
                writeLog("【预下载表写入失败did=".$did."】");
            }
        }else{
            $arr = array(
                ':oldid' => $value['oldid'],
                ':cid' => intval($value['knowledgeCatalogId']),
                ':title' => $value['title'],
                ':summary' => $value['summary'],
                ':lastedit' => intval($value['newUptime']),
                ':lasteditor' => $value['uploadUserName'],
                ':isdeleted' => intval($value['isdeleted']),
                ':visible' => intval($value['isvisiableForuser'])
            );
            $result_update = $stmt_update->execute($arr);
            if($result_update>0){
                writeLog("【wiki_doc表修改成功oldid=".$value['oldid']."】");
            }else{
                writeLog("【wiki_doc表修改失败oldid=".$value['oldid']."】");
            }

            if($value['lastedit']!==$value['newUptime']){//确定文件重新被上传过
                //向预下载表中添加数据
                $arr_url = array(
                    ':oid' => $value['oldid'],
                    ':did' => intval($row['did']),
                    ':cid' => intval($value['knowledgeCatalogId']),
                    ':time' => intval($value['newUptime']),
                    ':filename' => $value['title']
                );
                $result_url = $stmt_down_url->execute($arr_url);
                if($result_url>0){
                    writeLog("【预下载表写入成功did=".$row['did']."】");
                }else{
                    writeLog("【预下载表写入失败did=".$row['did']."】");
                }
            }
        }
    }
}else{
    writeLog("【文件没有新变动】");
}

//查看link表是否正确对应doc表
$compare_sql_link = "SELECT t2.did,t2.cid from doc t2 LEFT JOIN categorylink t1 ON t1.did=t2.did && t2.cid=t1.cid WHERE t1.did is null";
$compareArr_link = $conn->query($compare_sql_link);
$compareRow_link= $compareArr_link->fetchAll(PDO::FETCH_ASSOC);

if(count($compareRow_link)>0){
    $stmt_add_link = $conn->prepare("insert into categorylink (did,cid) values (:did,:cid)");//插入link表
    $stmt_update_link = $conn->prepare("update categorylink set cid=:cid where did=:did");//插入link表
    foreach ($compareRow_link as $key => $value) {
        $result = $conn->query("select 1 from categorylink where did='".$value['did']."'");
        $row = $result->fetch(PDO::FETCH_ASSOC);
        if(empty($row)){
            $arr_link = array(
                ':did' => intval($value['did']),
                ':cid' => intval($value['cid'])
            );
            $result_add_link = $stmt_add_link->execute($arr_link);
            if($result_add_link>0){
                writeLog("【插入link表成功did=".$value['did']."】");
            }else{
                writeLog("【插入link表失败did=".$value['did']."】");
            }
        }else{
            $arr_link = array(
                ':did' => intval($value['did']),
                ':cid' => intval($value['cid'])
            );
            $result_update_link = $stmt_update_link->execute($arr_link);
            if($result_update_link>0){
                writeLog("【修改link表成功did=".$value['did']."】");
            }else{
                writeLog("【修改link表失败did=".$value['did']."】");
            }
        }
    }
}

writeLog("【**********************检查更新结束*************************】");
echo date("Y-m-d H:i:s")."运行结束\r\n";





















?>