2018-07-16-spider2

coding=utf-8

from urllib import request
import uuid
import pyamf
import json,datetime
import socket
import os
from pyamf import remoting
from pyamf.flex import messaging,ArrayCollection
import xlwt

注册body类

class HqPara():
def init(self,breedInfo,provice,breedInfoDl,marketInfo):
self.breedInfo = breedInfo # 对象品种名称
self.provice = provice
self.breedInfoDl = breedInfoDl # 对象品种大类
self.marketInfo = marketInfo
class BreedInfoPo():
def init(self,children0,item_code,item_name,children1):
self.parentcode = None
self._children = children0
self.itemcode = item_code #String 000000
self.itemname = item_name #String 全部
self.children = children1
class ProvincePo():
def init(self,item_code,item_name):
self._children = None
self.itemcode = item_code #String 000000
self.itemname = item_name #String 全国
self.children = None
class BreedInfo():
def init(self,item_code,item_name):
self.parentcode = None
self._children = None
self.itemcode = item_code #String AM
self.itemname = item_name #String 水产品
self.children = None
class PMarketInfo():
def init(self,market_code,market_name):
self.marketCode = market_code
self.marketName = market_name #String 000000/AM

pyamf.register_class(HqPara, alias='com.itown.kas.pfsc.report.po.HqPara')
pyamf.register_class(BreedInfoPo, alias='com.itown.kas.pfsc.report.po.BreedInfoPo')
pyamf.register_class(ProvincePo, alias='com.itown.kas.pfsc.report.po.ProvincePo')
pyamf.register_class(BreedInfo, alias='com.itown.kas.pfsc.report.po.BreedInfoPo')
pyamf.register_class(PMarketInfo, alias='com.itown.kas.pfsc.report.po.PMarketInfo')
def construct_request(product_code,product_name,provice_code,provice_name,breed_code,breed_name,market_code,market_name,page_num,total_num):
breedInfo = BreedInfoPo(None,product_code,product_name,None)
provice = ProvincePo(provice_code,provice_name)
breedInfoDl = BreedInfo(breed_code,breed_name)
marketInfo = PMarketInfo(market_code,market_name)
msg = messaging.RemotingMessage(messageId=str(uuid.uuid1()).upper(),clientId=str(uuid.uuid1()).upper(),operation='getHqSearchData',destination='reportStatService',timeToLive=0,timestamp=0)
msg.body = [HqPara(breedInfo,provice,breedInfoDl,marketInfo),str(page_num), str(total_num)]
msg.headers['DSEndpoint'] = None
msg.headers['DSId'] = str(uuid.uuid1()).upper()
req = remoting.Request('null',body = (msg,))
env = remoting.Envelope(amfVersion=pyamf.AMF3)
env.bodies = [('/1',req)]
data = bytes(remoting.encode(env).read())
return data

data即为构造出amf编码后的请求结构

def getResponse(data):
url = 'http://jgsb.agri.cn/messagebroker/amf'
headers={'Content-Type': 'application/x-amf'}
#response = request.urlopen(url,headers=headers,data=data).read()
req = request.Request(url, data, headers=headers)
opener = request.build_opener()
response = opener.open(req).read()
return response

def amfParse(response):
amf_parse_info = remoting.decode(response)
total_page = amf_parse_info.bodies[0][1].body.body[2]
total_num = amf_parse_info.bodies[0][1].body.body[3]
info = amf_parse_info.bodies[0][1].body.body[0]
return total_page,total_num,info

def getAllInfo(total_page):
records = []
for i in range(int(total_page)+1):
data = construct_request('000000','全部','320000','江苏省','AM','水产品','000000','全部',i,15)
resp = getResponse(data)
total_page,total_num,info = amfParse(resp)
for record in info:
#print('record----------------:'+str(record))
if not record['reportDate'] == None:
record['reportDate'] = record['reportDate'].strftime('%Y-%m-%d %H:%M:%S')
if not record['auditDate'] == None:
record['auditDate'] = record['auditDate'].strftime('%Y-%m-%d %H:%M:%S')
if not record['snatchDate'] == None:
record['snatchDate'] = record['snatchDate'].strftime('%Y-%m-%d %H:%M:%S')
records.append(record)
return records

def store2json(records):
isExists=os.path.exists('D:/json')
if not isExists:

如果不存在则创建目录

    os.makedirs('D:/json') 
    print ('创建D:/json文件夹成功')
else:
    jsonfile = 'D:/json/info.json'
    with open(jsonfile, 'w') as file_obj:

将”obj“转换为JSON格式的字符串

        json.dump(records,file_obj,indent=4)
        print('--------------------写入json文件成功------------------')

def json2xlsx():
jsonfile = 'D:/json/info.json'
with open(jsonfile,'r',encoding='utf-8') as file_obj: # 将json文件转化为字典
data_all = json.load(file_obj)

book = xlwt.Workbook()  # 创建excel文件
sheet = book.add_sheet('sheet1')  # 创建一个表
title = ['品名', '批发市场', '最高价', '最低价','平均价', '产地', '信息员', '时间']
for col in range(len(title)):  # 存入第一行标题
    sheet.write(0, col, title[col])
row = 1  # 定义行
for k in data_all:
    print(len(data_all))
    data = []
    data.append(k['farmProduceName'])
    data.append(k['marketName'])
    data.append(k['maxPrice'])
    data.append(k['minPrice'])
    data.append(k['averagePrice'])
    data.append(k['producAdd'])
    data.append(k['reportMan'])
    data.append(k['auditDate'])
    print(data)

data.insert(0, k) # 第一列加入序号

    for index in range(len(data)):  # 依次写入每一行
        sheet.write(row, index, data[index])
    row += 1
book.save('D:/xlsx/score.xls')

if name=="main":
data = construct_request('000000','全部','000000','全国','AM','水产品','000000','全部',1,15)
#print('data----------------:'+str(data))
response = getResponse(data)
print(response)
total_page,total_num,info = amfParse(response)
print(total_num)
records = getAllInfo(total_page) #res为数组
#print(records)
store2json(records)
json2xlsx()