url 模拟http basic认证及批量抓取数据

url 模拟http basic认证

curl -e 'http://www.fnet.bosai.go.jp/REGS/dataget/' -H 'Authorization: Basic ZGF4dWV3dWhlbjo2MDU4Mzc5MDU=' -F 'format=MSEED&archive=tar%2Bgzip&station=ALL&component=BHX&component=LHZ&component=LHN&component=LHE&time=UT&s_year=2009&s_month=09&s_day=01&s_hour=00&s_min=00&s_sec=00&end=time&e_year=2011&e_month=09&e_day=01&e_hour=00&e_min=00&e_sec=00&sec=300&commands=&lang=ja' --output data1.txt http://www.fnet.bosai.go.jp/REGS/dataget/cgi-bin/dataget.cgi

使用PHP模拟表单批量抓取数据

<?php

$start = '20090901';
while(1) {
if($start < '20110901') {
echo "start: " . date('Y-m-d H:i:s') . PHP_EOL;

$next = date('Ymd', strtotime("+1 day", strtotime($start))); // 在start基础上加1天，当作查询的终止时间
$start_y = date("Y", strtotime($start));
$start_m = date("m", strtotime($start));
$start_d = date("d", strtotime($start));


$end_y = date("Y", strtotime($next));
$end_m = date("m", strtotime($next));
$end_d = date("d", strtotime($next));

// 模拟http表单提交动作，将结果文本保在以$start-$end命名的文件中
exec("curl -e 'http://www.fnet.bosai.go.jp/REGS/dataget/' -H 'Authorization: Basic ZGF4dWV3dWhlbjo2MDU4Mzc5MDU=' -F 'format=MSEED&archive=tar%2Bgzip&station=ALL&component=BHX&component=LHZ&component=LHN&component=LHE&time=UT&s_year={$start_y}&s_month={$start_m}&s_day={$start_d}&s_hour=00&s_min=00&s_sec=00&end=time&e_year={$end_y}&e_month={$end_m}&e_day={$end_d}&e_hour=00&e_min=00&e_sec=00&sec=300&commands=&lang=ja' --output {$start}-{$next}.txt http://www.fnet.bosai.go.jp/REGS/dataget/cgi-bin/dataget.cgi");

$start = $next; //< 将$start初始化$next,重新开始循环
echo "end: " . date('Y-m-d H:i:s') . PHP_EOL;
} else {
break;
}
}

将得到的*.txt处理下

cat *.txt|grep -E '^[A-Z]{3} [A-Z]{3}'|cut -c1-47 > /tmp/data.txt