Tornado入门
文章目录
- 简介
- 安装
- 初试
- 获取get请求参数
- 从命令行读取配置
- 正则表达式指定路径
- HTTP状态码
- 表单和模板
- 模板语法
- 模板综合示例
- 模板继承
- 异步和非阻塞
- RequestHandler常用方法
- RequestHandler子类
- 协程等待 Condition
- 长轮询
- WebSocket
- 总结
- 遇到的坑
- 封装
- 参考文献
简介
Tornado是一个Python web框架和异步网络库，使用非阻塞网络I/O，Tornado可以支撑上万级的连接，这使它成为长轮询、WebSockets和其他需要与每个用户建立长时间连接的应用程序的理想选择。
安装
pip install tornado
初试
import tornado.web
import tornado.ioloop
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.write("Hello World!")
if __name__ == "__main__":
print("http://localhost:8888/")
app = tornado.web.Application([
(r"/", MainHandler),
])
app.listen(8888)
tornado.ioloop.IOLoop.current().start()
访问http://localhost:8888/
获取get请求参数
import tornado.web
import tornado.ioloop
class MainHandler(tornado.web.RequestHandler):
def get(self):
name = self.get_argument("name", default="") # 获取get请求的name参数，默认值为空
self.write("Hello {}!".format(name))
if __name__ == "__main__":
print("http://localhost:8888/?name=XerCis")
app = tornado.web.Application([
(r"/", MainHandler),
])
app.listen(8888)
tornado.ioloop.IOLoop.current().start()
访问http://localhost:8888/?name=XerCis
从命令行读取配置
调用tornado.options模块的函数，步骤如下：
define()定义参数parse_command_line()读取命令行参数- 调用属性
options.xxx提取参数
从命令行读取配置.py
import tornado.web
import tornado.ioloop
from tornado.options import define, options, parse_command_line
define("port", default=8888, help="运行端口", type=int)
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.write("Hello World!")
if __name__ == "__main__":
parse_command_line()
print("http://localhost:{}/".format(options.port))
app = tornado.web.Application([
(r"/", MainHandler),
])
app.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
运行
python "03 从命令行读取配置.py" --port=8000
正则表达式指定路径
/reverse/(\w+)：正则表达式匹配任何以字符串/reverse/开始并紧跟着一个或多个字母的路径/wrap：默认每40个字符换行
import textwrap
import tornado.web
import tornado.ioloop
import tornado.httpserver
from tornado.options import define, options, parse_command_line
define("port", default=8888, help="运行端口", type=int)
class ReverseHandler(tornado.web.RequestHandler):
def get(self, input):
self.write(input[::-1]) # 逆转
class WrapHandler(tornado.web.RequestHandler):
def post(self):
text = self.get_argument("text")
width = self.get_argument("width", 40) # 40个字符
self.write(textwrap.fill(text, int(width)))
if __name__ == "__main__":
parse_command_line()
print("http://localhost:{}/reverse/slipup".format(options.port)) # 返回pupils
app = tornado.web.Application(
handlers=[
(r"/reverse/(\w+)", ReverseHandler), # 正则表达式匹配任何以字符串/reverse/开始并紧跟着一个或多个字母的路径
(r"/wrap", WrapHandler)
]
)
http_server = tornado.httpserver.HTTPServer(app)
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
使用 requests 库发送 GET 和 POST 请求
import requests
print(requests.get("http://localhost:8888/reverse/slipup").text)
print(requests.post("http://localhost:8888/wrap", params={
"text": "Lorem ipsum dolor sit amet, consectetuer adipiscing elit."}).text)
# pupils
# Lorem ipsum dolor sit amet, consectetuer
# adipiscing elit.
url中获取多个参数
import textwrap
import tornado.web
import tornado.ioloop
import tornado.httpserver
from tornado.options import define, options, parse_command_line
define("port", default=8888, help="运行端口", type=int)
class InfoHandler(tornado.web.RequestHandler):
def get(self, name, age, gender):
self.write("{} {} {}".format(name, age, gender))
if __name__ == "__main__":
parse_command_line()
print("http://localhost:{}/XerCis/23/male".format(options.port))
print("http://localhost:{}/XerCis/23/male/".format(options.port))
app = tornado.web.Application(
handlers=[
(r"/(\w+)/(\d+)/(\w+)/?", InfoHandler), # 加了?后，Tornado解析url会自动加上/
# (r"/(?P\w+)/(?P\d+)/(?P\w+)/?", InfoHandler), # 可读性更强的写法，?P<对应变量名>
]
)
http_server = tornado.httpserver.HTTPServer(app)
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
HTTP状态码
| 状态码 | 含义 | 情况 |
|---|---|---|
| 404 | Not Found | 请求路径不匹配 |
| 400 | Bad Request | 调用了一个没有默认值的 get_argument() |
| 405 | Method Not Allowed | 调用了没有定义的HTTP方法 |
| 500 | Internal Server Error | 程序遇到不能让其退出的错误 |
| 200 | OK | 响应成功 |
重写 write_error() 添加常规的错误消息
import tornado.web
import tornado.ioloop
import tornado.httpserver
from tornado.options import define, options, parse_command_line
define("port", default=8888, help="运行端口", type=int)
class IndexHandler(tornado.web.RequestHandler):
def get(self):
user = self.get_argument("user")
if user == "500": # 手动引发错误
raise Exception
self.write("Hello, " + user)
def write_error(self, status_code, **kwargs):
self.write("You caused a %d error." % status_code)
if __name__ == "__main__":
parse_command_line()
print("http://localhost:{}/?user=XerCis".format(options.port))
app = tornado.web.Application(handlers=[(r"/", IndexHandler)])
http_server = tornado.httpserver.HTTPServer(app)
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
使用 requests 库发送 GET 和 POST 请求
import requests
response = requests.get("http://localhost:8888/foo")
print(response, response.text) # 状态码404，请求路径不匹配
response = requests.get("http://localhost:8888/")
print(response, response.text) # 状态码400，调用了一个没有默认值的 get_argument()
response = requests.post("http://localhost:8888/")
print(response, response.text) # 状态码405，调用了没有定义的HTTP方法
response = requests.get("http://localhost:8888/", params={
"user": "500"})
print(response, response.text) # 状态码500，程序遇到不能让其退出的错误
response = requests.get("http://localhost:8888/", params={
"user": "XerCis"})
print(response, response.text) # 状态码200，响应成功
# 404: Not Found 404: Not Found
# You caused a 400 error.
# You caused a 405 error.
# You caused a 500 error.
# Hello, XerCis
表单和模板
创建文件夹 templates，存放以下两个HTML
index.html
<html>
<head><title>对对联title>head>
<body>
<h1>输入下联h1>
<form method="post" action="/couplet">
<p>上联<br><input type="text" name="first" value="天行健，君子以自强不息。">p>
<p>下联<br><input type="text" name="second">p>
<input type="submit">
form>
body>
html>
couplet.html
<html>
<head><title>对对联title>head>
<body>
<h1>你的对联h1>
<p>{
{first}}<br><br>{
{second}}p>
body>
html>
双大括号中的单词是占位符，当渲染模板时调用 render() 传递关键字参数，以实际值代替占位符。
代码
import tornado.web
import tornado.ioloop
from pathlib import Path
import tornado.httpserver
from tornado.options import define, options, parse_command_line
define("port", default=8888, help="运行端口", type=int)
class IndexHandler(tornado.web.RequestHandler):
def get(self):
self.render("index.html")
class CoupletHandler(tornado.web.RequestHandler):
def post(self):
first = self.get_argument("first") # 上联
second = self.get_argument("second") # 下联
if second == "":
second = "地势坤，君子以厚德载物。"
self.render("couplet.html", first=first, second=second) # 传递关键字参数，指定什么值填充到HTML文件中占位符中{
{}}
if __name__ == "__main__":
parse_command_line()
print("http://localhost:{}/".format(options.port))
app = tornado.web.Application(
handlers=[
(r"/", IndexHandler),
(r"/couplet", CoupletHandler)
],
template_path=Path(__file__).parent / "templates" # 模板文件
)
http_server = tornado.httpserver.HTTPServer(app)
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
效果
模板语法
1. 模板填充表达式 { { 表达式 }}
from tornado.template import Template
content = Template("{
{ header }}
")
print(content.generate(header="Welcome!"))
# b'Welcome!
'
print(Template("{
{ 1+1 }}").generate())
print(Template("{
{ 'scrambled eggs'[-4:] }}").generate())
print(Template("{
{ ', '.join([str(x*x) for x in range(10)]) }}").generate())
# b'2'
# b'eggs'
# b'0, 1, 4, 9, 16, 25, 36, 49, 64, 81'
2. 控制语句 { {% 控制语句 %}}，支持 if、for、while、try，如：
{% if page is None %}{% if len(entries) == 3 %}{% for book in books %}{% end %}
book.html，放置于templates文件夹下
<html>
<head>
<title>{
{ title }}title>
head>
<body>
<h1>{
{ header }}h1>
<ul>
{% for book in books %}
<li>{
{ book }}li>
{% end %}
ul>
body>
html>
代码
import tornado.web
import tornado.ioloop
from pathlib import Path
import tornado.httpserver
from tornado.options import define, options, parse_command_line
define("port", default=8888, help="运行端口", type=int)
class BookHandler(tornado.web.RequestHandler):
def get(self):
self.render(
"book.html",
title="首页",
header="推荐书籍",
books=[
"Python简明手册",
"流畅的Python",
"Python深度学习"
]
)
if __name__ == "__main__":
parse_command_line()
print("http://localhost:{}/book".format(options.port))
app = tornado.web.Application(
handlers=[
(r"/book", BookHandler)
],
template_path=Path(__file__).parent / "templates" # 模板文件
)
http_server = tornado.httpserver.HTTPServer(app)
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
效果
3. 模板中使用函数
| 函数 | 功能 |
|---|---|
| escape(s) | 替换字符串中的&、<、>为HTML字符 |
| url_escape(s) | 使用urllib.quote_plus()替换字符串中的字符为URL编码 |
| json_encode(val) | 使用json.dumps()将val编码为JSON |
| squeeze(s) | 将连续多个空白字符替换为一个空格 |
| 自定义函数 | 函数名作为参数传递 |
代码
from tornado.template import Template
def foo():
return 'World!'
print(Template("{
{ escape('&<>') }}").generate())
print(Template("{
{ url_escape('你好') }}").generate())
print(Template("{
{ json_encode({'a': 1}) }}").generate())
print(Template("{
{ squeeze(' a b c ') }}").generate())
print(Template("Hello {
{ fun() }}").generate(fun=lambda: "World!"))
print(Template("Hello {
{ fun() }}").generate(fun=foo))
# b'&<>'
# b'%E4%BD%A0%E5%A5%BD'
# b'{"a": 1}'
# b'a b c'
# b'Hello World!'
# b'Hello World!'
推荐阅读：
- tornado.template — 产生灵活的输出
- tornado.template — Flexible output generation
模板综合示例
模板综合示例
目录结构
style.css
body {
font-family: Helvetica, Arial, sans-serif;
width: 600px;
margin: 0 auto;
}
.replaced:hover {
color: #00f;
}
index.html
<html>
<head>
<title>文本替换title>
head>
<body>
<h1>文本替换h1>
<p>替换文本中的单词将被源文本中的相同字母开头的单词替换。p>
<form method="post" action="/poem">
<p>源文本<br>
<textarea rows=5 cols=70 name="source">
Twas brillig, and the slithy toves
Did gyre and gimble in the wabe:
All mimsy were the borogoves,
And the mome raths outgrabe.
textarea>p>
<p>替换文本<br>
<textarea rows=5 cols=70 name="change">
When in the course of human events it becomes necessary for one people to dissolve the political bands which have connected them with another and to assume among the powers of the earth, the separate and equal station to which the Laws of Nature and of Nature's God.
textarea>p>
<input type="submit">
form>
body>
html>
munged.html
<html>
<head>
<title>文本替换title>
head>
<body>
<h1>结果h1>
<p>
{% for line in change_lines %}
{% for word in line.split(' ') %}
{% if len(word) > 0 and word[0] in source_map %}
<span class="replaced" title="{
{word}}">{
{ choice(source_map[word[0]]) }}span>
{% else %}
<span class="unchanged" title="unchanged">{
{word}}span>
{% end %}
{% end %}
<br>
{% end %}
p>
body>
html>
main.py
import random
import tornado.web
import tornado.ioloop
from pathlib import Path
import tornado.httpserver
from tornado.options import define, options, parse_command_line
define("port", default=8888, help="运行端口", type=int)
class IndexHandler(tornado.web.RequestHandler):
def get(self):
self.render('index.html')
class MungedPageHandler(tornado.web.RequestHandler):
def map_by_first_letter(self, text):
mapped = dict()
for line in text.split('\r\n'):
for word in [x for x in line.split(' ') if len(x) > 0]:
if word[0] not in mapped:
mapped[word[0]] = []
mapped[word[0]].append(word)
return mapped
def post(self):
source_text = self.get_argument('source')
text_to_change = self.get_argument('change')
source_map = self.map_by_first_letter(source_text)
change_lines = text_to_change.split('\r\n')
self.render('munged.html', source_map=source_map, change_lines=change_lines, choice=random.choice)
if __name__ == "__main__":
parse_command_line()
print("http://localhost:{}/".format(options.port))
app = tornado.web.Application(
handlers=[
(r"/", IndexHandler),
(r"/poem", MungedPageHandler)
],
template_path=Path(__file__).parent / "templates", # 模板文件
static_path=Path(__file__).parent / "static", # 静态文件
)
http_server = tornado.httpserver.HTTPServer(app)
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
效果
CSS、JavaScript、图像等无需单独编写处理函数的静态内容，可指定 static_path 实现。
生成静态URL，而不使用硬编码的原因：
static_url会创建了一个基于文件内容的hash值，并将其添加到URL末尾（查询字符串的参数v）。这个hash值确保浏览器总是加载文件的最新版而不是缓存版本。- 改变应用URL结构，而不需要改变模板中的代码。
模板继承
项目结构
recommended.css
.book {
width: 500px;
border: 1px solid #555;
padding: 0 15px 0 15px;
margin: 15px;
background-color: #F5F5F5;
}
.book img.book_image {
margin: 5px 15px 5px 0;
float: left;
}
.book .book_details {
margin: 5px 10px 5px 0;
}
.book .book_title {
color: #777;
}
style.css
body {
font-family: Georgia;
}
.small {
font-size: .7em;
}
#discussion {
width: 700px;
margin: 10px 0 10px 0;
padding: 10px;
color: #111;
}
#discussion h3 {
text-decoration: underline;
}
#discussion .comment {
width: 500px;
margin: 10px 0 10px 0;
padding: 10px;
border: 1px solid #555;
background-color: #FAFAFA;
}
#discussion .comment .comment_user {
text-decoration: underline;
}
#discussion .comment .comment_user .bold {
font-weight: bold;
}
#discussion .comment .comment_text {
font-style: italic;
}
collective_intelligence.gif
head_first_python.gif
restful_web_services.gif
recommended.js
// document.write("script loaded!");
script.js
模板继承 使站点能够复用像header、footer和布局网格这样的内容。
使用 extends 和 block 语句进行模板继承
在新模板上继承模板只需在顶部放上一句 {% extends "filename.html" %}即可
需要复用的内容用 block包围，如
{% block header %}
<h1>Hello world!h1>
{% end %}
main.html 基础模板，是整个网站的通用架构
<html>
<head>
<title>{
{ page_title }}title>
head>
<body>
<div id="container">
<header>
{% block header %}<h1>Burt's Booksh1>{% end %}
header>
<div id="main">
<div id="content">
{% block body %}{% end %}
div>
div>
<footer>
{% block footer %}
<p>更多信息联系： <a href="mailto:[email protected]">[email protected]a>.p>
<p class="small">在Facebook上关注我们： {% raw linkify("https://facebook.com/burtsbooks", extra_params='ref=website') %}.p>
{% end %}
footer>
div>
"
}
]
)
class RecommendedHandler(tornado.web.RequestHandler):
def get(self):
self.render(
"recommended.html",
page_title="Burt's Books | 推荐书籍",
header_text="推荐书籍",
books=[
{
"title": "集体智慧编程",
"subtitle": "构建智能Web 2.0应用程序",
"image": "/static/images/collective_intelligence.gif",
"author": "Toby Segaran",
"date_added": 1310248056,
"date_released": "2007年08月",
"description": "本书演示了如何构建Web应用程序来挖掘Internet上的大量数据。通过算法从Web网站获取、收集并分析用户的数据和反馈信息，以便创造新的用户价值和商业价值。包括协同过滤技术、集群数据分析、搜索引擎技术、海量信息搜索与分析、贝叶斯过滤技术、决策树技术实现预测和决策建模、社交网络信息匹配、机器学习和人工智能应用等。
"
},
{
"title": "RESTful Web Services",
"subtitle": "面向现实世界的Web服务",
"image": "/static/images/restful_web_services.gif",
"author": "Leonard Richardson, Sam Ruby",
"date_added": 1311148056,
"date_released": "2007年05月",
"description": "除了构建人类使用的Web站点，未来更在于构建机器使用的Web站点。本书解释了如何构建网络，充分利用REST，HTTP和Web基础设施。详细讨论了统一口、资源、表述、URI、请求、缓存、安全等诸多内容。
"
},
{
"title": "深入浅出Python",
"subtitle": "",
"image": "/static/images/head_first_python.gif",
"author": "Paul Barry",
"date_added": 1311348056,
"date_released": "2010年11月",
"description": "你是否曾希望从一本书中学习Python？本书提供一个完整的Python学习体验，它以一种独特的方法学习该语言，帮助您成为一个伟大的Python程序员。您将快速学习该语言的基础知识，然后学习持久性、异常处理、Web开发、SQLite等。
"
}
]
)
class BookModule(tornado.web.UIModule):
def render(self, book):
return self.render_string(
"modules/book.html",
book=book,
)
def css_files(self):
return "css/recommended.css"
def javascript_files(self):
return "js/recommended.js"
if __name__ == "__main__":
parse_command_line() # 解析命令行参数
print("http://localhost:{}/".format(options.port)) # 首页
print("http://localhost:{}/discussion/".format(options.port)) # 聊天室
print("http://localhost:{}/recommended/".format(options.port)) # 推荐书籍
app = tornado.web.Application(
handlers=[
(r"/", IndexHandler),
(r"/discussion/", DiscussionHandler),
(r"/recommended/", RecommendedHandler),
],
template_path=Path(__file__).parent / "templates", # 模板文件
static_path=Path(__file__).parent / "static", # 静态文件
ui_modules={
"Book": BookModule},
)
http_server = tornado.httpserver.HTTPServer(app)
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
效果
异步和非阻塞
官方文档建议 Python 3.5+ 用 async def 替代 tornado.gen装饰器
尽量使用 async 而不是 coroutine 装饰器的原因：
- 基于coroutine是一个从生成器过渡到协程的方案
- yield和await的混合使用造成代码可读性很差
- 生成器可以模拟协程，但生成器应该做自己
- 原生协程总体来说比基于装饰器的协程快
- 原生协程可以使用async for和async with更符合Python风格
- 原生协程返回的是awaitable对象，装饰器协程返回的是future
为什么使用异步：
- CPU速度远高于IO速度
- IO包括网络访问和本地文件访问，比如requests，urllib等传统网络库都是同步IO
- 网络IO大部分时间处于等待状态，在等待的时候CPU是空闲的，但又不能执行其他操作
阻塞：调用函数时当前线程被挂起。
非阻塞：调用函数时当前线程不会被挂起，而是立即返回。
socket阻塞IO
import socket
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
host = 'www.baidu.com'
client.connect((host, 80))
client.send('GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n'.format('/', host).encode('utf-8'))
data = b''
while True:
d = client.recv(1024) # 阻塞直到有数据
if d:
data += d
else:
break
data = data.decode('utf-8')
print(data)
socket非阻塞IO
import socket
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.setblocking(False) # 非阻塞
host = 'www.baidu.com'
try:
client.connect((host, 80))
except BlockingIOError:
# 做其他事
pass
while True:
try:
client.send('GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n'.format('/', host).encode('utf-8'))
print('连接成功')
break
except OSError:
pass
data = b''
while True:
try:
d = client.recv(1024)
except BlockingIOError:
continue
if d:
data += d
else:
break
data = data.decode('utf-8')
print(data)
select非阻塞IO
import socket
from selectors import DefaultSelector, EVENT_READ, EVENT_WRITE
selector = DefaultSelector()
class Fetcher:
def connected(self, key):
selector.unregister(key.fd)
self.client.send(
'GET {} HTTP/1.1\r\nHost:{}\r\nConnection:close\r\n\r\n'.format('/', self.host).encode('utf-8'))
selector.register(self.client.fileno(), EVENT_READ, self.readble)
def readble(self, key):
d = self.client.recv(1024)
if d:
self.data += d
else:
selector.unregister(key.fd)
data = self.data.decode('utf-8')
print(data)
def get_url(self, url):
self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.client.setblocking(False)
self.data = b''
self.host = 'www.baidu.com'
try:
self.client.connect((self.host, 80))
except BlockingIOError:
pass
selector.register(self.client.fileno(), EVENT_WRITE, self.connected)
def loop_forever():
while True:
ready = selector.select()
for key, mask in ready:
call_back = key.data
call_back(key)
if __name__ == '__main__':
fetcher = Fetcher()
url = 'http://www.baidu.com'
fetcher.get_url(url)
loop_forever()
这样的回调+事件循环实现的异步导致：
- 回调过深代码很难维护
- 栈撕裂造成异常无法向上抛出
协程，又称微线程，Coroutine。协程是可以被暂停和切换到其他协程运行的函数。协程运行时必须有事件循环。
旧代码改写为新代码
from tornado.gen import coroutine
@coroutine
def yield_test():
yield 1
yield 2
yield 3
@coroutine
def main():
yield from yield_test()
新代码
async def yield_test():
yield 1
yield 2
yield 3
async def main():
await yield_test()
HTTPClient同步请求，类似requests
from tornado import httpclient
http_client = httpclient.HTTPClient()
try:
response = http_client.fetch('http://www.baidu.com')
print(response.body)
except Exception as e:
print(e)
http_client.close()
AsyncHTTPClient异步请求
from tornado import ioloop
from tornado import httpclient
http_client = httpclient.AsyncHTTPClient()
async def f():
try:
response = await http_client.fetch('http://www.baidu.com')
except Exception as e:
print(e)
else:
print(response.body)
if __name__ == '__main__':
io_loop = ioloop.IOLoop.current()
io_loop.run_sync(f) # 协程运行完后自动停止事件循环
直接使用Python3的事件循环（Tornado的IOLoop底层是asyncio）
import asyncio
from tornado import httpclient
http_client = httpclient.AsyncHTTPClient()
async def f():
try:
response = await http_client.fetch('http://www.baidu.com')
except Exception as e:
print(e)
else:
print(response.body)
if __name__ == '__main__':
# asyncio.ensure_future(f())
# asyncio.get_event_loop().run_forever()
asyncio.get_event_loop().run_until_complete(f())
高并发爬虫
from bs4 import BeautifulSoup
from urllib.parse import urljoin
from tornado import gen, httpclient, ioloop, queues
base_url = 'http://www.tornadoweb.org/en/stable/'
concurrency = 3 # 并发数
async def get_url_links(url):
response = await httpclient.AsyncHTTPClient().fetch(url) # 爬取
html = response.body.decode() # 解码
soup = BeautifulSoup(html) # 解析
links = [urljoin(base_url, a.get('href')) for a in soup.find_all('a', href=True)] # 当前页面所有url
return links
async def main():
seen = set() # 访问过的页面
q = queues.Queue() # 不使用Python自带的Queue因其同步而非异步
async def fetch_url(current_url):
# 生产者
if current_url in seen: # 访问过直接返回
return
print(current_url) # 输出爬取到的页面
seen.add(current_url) # 已访问
next_urls = await get_url_links(current_url) # 当前页面所有url
for new_url in next_urls:
if new_url.startswith(base_url): # 属于该站点
await q.put(new_url) # 放入队列
async def worker():
# 消费者
async for url in q: # 异步取数据
if url is None:
return
try:
await fetch_url(url)
except Exception as e:
print(e)
finally:
q.task_done()
# 1. 放入初试url到队列
await q.put(base_url)
# 2. 启动协程
workers = gen.multi([worker() for _ in range(concurrency)]) # 多并发消费者
await q.join()
for _ in range(concurrency):
await q.put(None)
await workers
if __name__ == '__main__':
io_loop = ioloop.IOLoop.current()
io_loop.run_sync(main) # 协程运行完后自动停止事件循环
在Tornado中不要写同步IO，不然会阻塞
import time
import tornado.web
import tornado.ioloop
class FirstHandler(tornado.web.RequestHandler):
async def get(self):
time.sleep(5) # 模拟阻塞
self.write("1")
class SecondHandler(tornado.web.RequestHandler):
async def get(self):
self.write("2")
if __name__ == "__main__":
print("http://localhost:8888/")
print("http://localhost:8888/2")
app = tornado.web.Application([
(r"/", FirstHandler),
(r"/2", SecondHandler),
])
app.listen(8888)
tornado.ioloop.IOLoop.current().start()
同时请求两个接口，即使第二个接口没有IO操作，但仍然要等待上一个请求的阻塞IO完成。
RequestHandler常用方法
RequestHandler
import tornado.web
import tornado.ioloop
class IndexHandler(tornado.web.RequestHandler):
def get(self):
self.redirect("/") # 重定向，默认状态码为301永久重定向，若参数permanent=True则为302临时重定向
class ErrorHandler(tornado.web.RequestHandler):
def get(self):
self.send_error() # 直接引发错误
def write_error(self, status_code, **kwargs):
print(status_code)
self.write('error') # 跳转错误页
class MainHandler(tornado.web.RequestHandler):
def initialize(self, db):
# 用于初始化
self.db = db
print('1. initialize')
def prepare(self):
# 用于真正调用请求之前的初始化方法
# 打印日志， 打开文件
print('2. prepare')
def on_finish(self):
# 请求完成后调用
# 关闭句柄，清理内存
print('3. on_finish')
def get(self):
self.write("Hello World!")
def post(self):
pass
def head(self):
pass
def delete(self):
pass
def patch(self):
pass
def put(self):
pass
def options(self):
pass
if __name__ == "__main__":
print("http://localhost:8888/")
print("http://localhost:8888/index/")
print("http://localhost:8888/error/")
app = tornado.web.Application([
(r"/", MainHandler, dict(db='db')),
(r"/index/?", IndexHandler),
(r"/error/?", ErrorHandler),
])
app.listen(8888)
tornado.ioloop.IOLoop.current().start()
RequestHandler子类
RedirectHandler：重定向
StaticFileHandler：静态文件
static/img.jpg
import tornado.web
import tornado.ioloop
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.write("Hello World!")
if __name__ == "__main__":
print("http://localhost:8888/index/")
print("http://localhost:8888/static/img.jpg")
app = tornado.web.Application([
(r"/", MainHandler),
(r"/index/?", tornado.web.RedirectHandler, {
"url": "/"}),
(r"/static/(.*)", tornado.web.StaticFileHandler, {
"path": "static"}),
# (r"/(.*)", tornado.web.StaticFileHandler, {"path": ""}),
])
app.listen(8888)
tornado.ioloop.IOLoop.current().start()
协程等待 Condition
同步原语
from tornado import gen
from tornado.ioloop import IOLoop
from tornado.locks import Condition
condition = Condition()
async def waiter():
print("I'll wait right here") # 1.等待
await condition.wait()
print("I'm done waiting") # 4.继续
async def notifier():
print("About to notify") # 2.通知可以继续
condition.notify()
print("Done notifying") # 3.通知完成
async def runner():
await gen.multi([waiter(), notifier()]) # 同时等待waiter()和notifier()
IOLoop.current().run_sync(runner) # 事件循环
# import asyncio
# asyncio.get_event_loop().run_until_complete(runner()) # 事件循环
长轮询
项目结构
chat.css
body {
background: white;
margin: 10px;
}
body,
input {
font-family: sans-serif;
font-size: 10pt;
color: black;
}
table {
border-collapse: collapse;
border: 0;
}
td {
border: 0;
padding: 0;
}
#body {
position: absolute;
bottom: 10px;
left: 10px;
}
#input {
margin-top: 0.5em;
}
#inbox .message {
padding-top: 0.25em;
}
#nav {
float: right;
z-index: 99;
}
chat.js
$(function () {
$("#messageform").on("submit", function () {
newMessage($(this));
return false;
});
$("#messageform").on("keypress", function (e) {
if (e.keyCode == 13) {
newMessage($(this));
return false;
}
return true;
});
$("#message").select();
updater.poll();
});
function newMessage(form) {
var message = form.formToDict();
var disabled = form.find("input[type=submit]");
disabled.disable();
$.postJSON("/a/message/new", message, function (response) {
updater.showMessage(response);
if (message.id) {
form.parent().remove();
} else {
form.find("input[type=text]").val("").select();
disabled.enable();
}
});
}
function getCookie(name) {
var r = document.cookie.match("\\b" + name + "=([^;]*)\\b");
return r ? r[1] : undefined;
}
jQuery.postJSON = function (url, args, callback) {
args._xsrf = getCookie("_xsrf");
$.ajax({
url: url, data: $.param(args), dataType: "text", type: "POST",
success: function (response) {
if (callback) callback(eval("(" + response + ")"));
}, error: function (response) {
console.log("ERROR:", response);
}
});
};
jQuery.fn.formToDict = function () {
var fields = this.serializeArray();
var json = {
};
for (var i = 0; i < fields.length; i++) {
json[fields[i].name] = fields[i].value;
}
if (json.next) delete json.next;
return json;
};
jQuery.fn.disable = function () {
this.enable(false);
return this;
};
jQuery.fn.enable = function (opt_enable) {
if (arguments.length && !opt_enable) {
this.attr("disabled", "disabled");
} else {
this.removeAttr("disabled");
}
return this;
};
var updater = {
errorSleepTime: 500,
cursor: null,
poll: function () {
var args = {
"_xsrf": getCookie("_xsrf")};
if (updater.cursor) args.cursor = updater.cursor;
$.ajax({
url: "/a/message/updates", type: "POST", dataType: "text",
data: $.param(args), success: updater.onSuccess,
error: updater.onError
});
},
onSuccess: function (response) {
try {
updater.newMessages(eval("(" + response + ")"));
} catch (e) {
updater.onError();
return;
}
updater.errorSleepTime = 500;
window.setTimeout(updater.poll, 0);
},
onError: function (response) {
updater.errorSleepTime *= 2;
console.log("Poll error; sleeping for", updater.errorSleepTime, "ms");
window.setTimeout(updater.poll, updater.errorSleepTime);
},
newMessages: function (response) {
if (!response.messages) return;
var messages = response.messages;
updater.cursor = messages[messages.length - 1].id;
console.log(messages.length, "new messages, cursor:", updater.cursor);
for (var i = 0; i < messages.length; i++) {
updater.showMessage(messages[i]);
}
},
showMessage: function (message) {
var existing = $("#m" + message.id);
if (existing.length > 0) return;
var node = $(message.html);
node.hide();
$("#inbox").append(node);
node.slideDown();
}
};
index.html
<html>
<head>
<meta charset="UTF-8">
<title>聊天室title>
head>
<body>
<div id="body">
<div id="inbox">
{% for message in messages %}
{% module Template("message.html", message=message) %}
{% end %}
div>
<div id="input">
<form action="/a/message/new" method="post" id="messageform">
<table>
<tr>
<td><input type="text" name="body" id="message" style="width:500px">td>
<td style="padding-left:5px">
<input type="hidden" name="next" value="{
{ request.path }}">
{% module xsrf_form_html() %}
td>
tr>
table>
form>
div>
div>
<script src="https://cdn.bootcss.com/jquery/3.0.0/jquery.min.js">script>