php去重脚本

去重脚本

error_reporting(0);

@ini_set('memory_limit','-1');

set_time_limit(0);

echo" 去除文本重复工具"."\r\n\r\n";

echo"\n"."输入要整理的文件:"."\n";

$dic=trim(fgets(STDIN));

/*while (!feof($dic)){

        $file[]=stream_get_line($fp,65535,"\r\n");

}

*/



$file=file($dic);

$array=preg_replace('/($\s*$)|(^\s*^)/m','',$file); //消除空行

$new_array=array_values(array_unique($array));//消除重复行

$new_filename="new_". basename($dic);



if(file_put_contents("$new_filename",join("\r\n",$new_array))){



        $num=count($file);

        $new_num=count($new_array);

        $counts=$num-$new_num;

        $files=dirname(__FILE__).DIRECTORY_SEPARATOR.$new_filename;



echo<<