python爬51cto某一课程表
因为看见老男孩的学员做过这一期，也很有兴趣，并且刚在学python，所以用python实现了下。由于刚学习python，肯定还有很多地方可以优化。
#!/bin/bash
# oldboy linux training
# 2015-06-01
# Happy Children's Day
# 说明：本脚本来自老男孩linux21期学员张耀开发！
EduFile=/tmp/edu.html
EduFile2=/tmp/edu2.html
Url="$*"
# Check for given parameters
[ $# -eq 0 ] && {
echo "USAGE: /bin/sh $0 http://...."
exit 1
}
# Judge url is ok?
curl -I $Url &>/dev/null
[ $? -ne 0 ] &&{
echo "Bad url,Please check it"
exit 1
}
# Defined get pagenum and CourseId Functions
function getnum(){
curl -s $Url>$EduFile
grep '"pagesGoEnd"' $EduFile &>/dev/null
if [ $? -eq 0 ]
then
num=`sed -rn 's#.*page=([0-9].*)" class="pagesGoEnd".*$#\1#gp' $EduFile`
else
num=`sed -rn 's|.*page=([0-9].*)#" class="pagesNum".*$|\1|gp' $EduFile`
fi
pagenum=${num:-1}
CourseId=`echo $Url|awk -F "[-.]" '{print $4}'`
}
# Defined curl html Functions
function Curl(){
getnum
for i in `seq $pagenum`
do
curl "http://edu.51cto.com/index.php?do=course&m=lessions&course_id=$CourseId&page=$i" 1>>$EduFile 2>/dev/null
done
}
# Defined Create table Functions
function table(){
sum=""
index=1
sed -rn '/do=lesson/ s#<.*(<a href=")(.*)</h4>$#\1http://edu.51cto.com\2#gp' $EduFile > $EduFile2
while read line
do
sum=$sum"<tr><th width="40" scope="row">$index</th><td width="520">$line</td>"
((index++))
done <$EduFile2
}
# Defined Create html Functions
function html(){
cat >/tmp/oldboy.html<<-END
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>test</title>
</head>
<body>
<table width="560" border="1">
$sum
</table>
</body>
</html>
END
}
function main(){
Curl
table
html
}
main
#!/usr/bin/env python
#coding:utf-8
import urllib,urllib2,sys,os,re
reload(sys)
sys.setdefaultencoding('utf-8')
def get_course_id_page_num(URI):
orgin = urllib2.urlopen(URI).read().decode('utf-8').encode('utf-8')
try:
page_pattern = re.compile(r'<a href="/index.php?.*course_id=(.*)&page=(.*)" class="pagesGoEnd"')
course_id = page_pattern.search(orgin).group(1)
pagenum = page_pattern.search(orgin).group(2)
except AttributeError:
page_pattern = re.compile(r'<a href="/index.php?.*course_id=(.*)&page=(.*)#" class="pagesNum"')
course_id = page_pattern.search(orgin).group(1)
pagenum = page_pattern.search(orgin).group(2)
return (course_id,pagenum)
def get_url_title(course,page):
#cto = file('/tmp/edu.51cto.html','a+')
url= "http://edu.51cto.com/index.php?do=course&m=lessions&course_id=" + str(course) + "&page=" + str(page)
request = urllib2.urlopen(url).read().decode('utf-8').encode('utf-8')
url_title_pattern = re.compile(r'<a href="(.*)" target="_blank">(.*)</a></h4>')
ut=url_title_pattern.findall(request)
cto_table_1 = '<tr><th scope=row>+_+</th><td><a href="'
cto_table_2 = 'http://edu.51cto.com'
cto_table_3 = '" target="_blank">'
cto_table_4 = '</a></td>'
for k,v in ut:
if 'lesson' in k:
line = cto_table_1 + cto_table_2 + k + cto_table_3 + v + cto_table_4
print >> cto, line
#cto.close()
if __name__ == "__main__":
URI = "http://edu.51cto.com/course/course_id-839.html"
#URI = sys.argv[1]
cto = file('/tmp/edu.51cto.html','a+')
cto_head = '<head>\n<meta http-equiv="Content-Type" content="text/html;charset=utf-8"/>\n<title>51cto</title>\n</head>\n<body><table width="560" border="1">\n'
print >> cto, cto_head
course_id,pagenum = get_course_id_page_num(URI)
for i in range(1,int(pagenum) + 1):
get_url_title(course_id,i)
cto_tail = '</table>\n</body>\n</html>'
print >> cto, cto_tail
cto.close()
以下就是程序运行之后的效果图了