python multiprocessing.manage - 不知原因的IOError
IOError: [Errno 32] Broken pipe
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python2.6/threading.py", line 532, in __bootstrap_inner
self.run()
File "/home/mii/dianshang_test/dianshang_new_test/dianshang_current.py", line 77, in run
self.c.crawl_step_1(str(item[0]),str(item[2]),str(item[4]),str(item[3]), str(item[5]), str(item[6]), self.sites[int(item[1])])
File "/home/mii/dianshang_test/dianshang_new_test/dianshang_current.py", line 108, in crawl_step_1
if not self.crawl_step_2(url, site_name, keyword_id, site_type, id, Site):
File "/home/mii/dianshang_test/dianshang_new_test/dianshang_current.py", line 136, in crawl_step_2
self.sql.in2db(url, site_name, keyword_id, site_type, author, content, pubtime, title, id)
File "/home/mii/dianshang_test/dianshang_new_test/dianshang_current.py", line 266, in in2db
self.db.write_sql('insert', 'data_db', sql_raw_data, sql_keyword_rawdata_map)
File "tools/HandleSQL.py", line 26, in write_sql
self.queue.put(sql_data)
File "<string>", line 2, in put
File "/usr/lib/python2.6/multiprocessing/managers.py", line 725, in _callmethod
conn.send((self._id, methodname, args, kwds))
IOError: [Errno 32] Broken pipe
IOError: [Errno 104] Connection reset by peer
Exception in thread Thread-250:
Traceback (most recent call last):
File "/usr/lib/python2.7/threading.py", line 552, in __bootstrap_inner
self.run()
File "/laiseek/congbo/all_crawler/tools/mythreading.py", line 40, in run
self.thread_func(thread_task)
File "/laiseek/congbo/all_crawler/dianshang/current/dianshang.py", line 67, in crawl_by_thread
crawl.crawl_by_source(site_search, keyword, max_pages_search, handle_product_results, args_for_handle_results)
File "/laiseek/congbo/all_crawler/tools/crawl.py", line 39, in crawl_by_source
stop_next_page = handle_results(site, url, results, args_for_handle_results)
File "/laiseek/congbo/all_crawler/dianshang/current/dianshang.py", line 94, in handle_product_results
crawl.crawl_by_source(site_comment, keyword, max_pages_comment, handle_comment_results, args_for_handle_results)
File "/laiseek/congbo/all_crawler/tools/crawl.py", line 39, in crawl_by_source
stop_next_page = handle_results(site, url, results, args_for_handle_results)
File "/laiseek/congbo/all_crawler/dianshang/current/dianshang.py", line 117, in handle_comment_results
queue_sql.put_queue('insert', 'data_db', 'raw_data', sql_values_raw_data)
File "/laiseek/congbo/all_crawler/tools/queue_for_sql.py", line 24, in put_queue
self.queue.put(sql_data)
File "<string>", line 2, in put
File "/usr/lib/python2.7/multiprocessing/managers.py", line 759, in _callmethod
kind, result = conn.recv()
IOError: [Errno 104] Connection reset by peer
错误没有复现，不知原因为何，记录一下。
参考：