python 爬取steam/csgo市场数据

在不登录的情况下频繁访问steam市场很容易被禁止访问，由于steam社区市场被墙了，所以需要挂才能正常爬数据

首先挂好VPN，登录网页版steam，得到cookies和user-agent的内容，然后复制到headers中的cookies和user-agent位置上

设置好相关的路径后就可以了
得到的数据会保存到csv文件中

csgo的每一种箱子都会有一个对应的id，得到相应的ID然后保存到列表中就可以了，只写了得到箱子在售量和起价数据的程序，其他的东西原理上应该都差不多

初学python，有什么错误请多多指教

import sys
sys.path.append(r'E:\anaconda\Lib\site-packages')

from pyquery import PyQuery as pq
import requests
from urllib.parse import urlencode
from lxml import etree
import time
import random
import csv

base_url='https://steamcommunity.com/market/itemordershistogram?'

headers={   'Host': 'steamcommunity.com',
            'Referer': 'https://steamcommunity.com/market',
            'X-Requested-With': 'XMLHttpRequest',
            'Connection': 'keep-alive',
            'Cookie':' ',
            'user-agent':'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Mobile Safari/537.36',
            }

retry_id_list = []
def get_page(case_id):
    params = {'country': 'CN',
              'language': 'schinese',
              'currency': '23',   #货币
              'item_nameid': case_id,
              'two_factor': 0              
            }

    url = base_url + urlencode(params)
    
    try:
        response = requests.Session().get(url,headers=headers)
        if response.status_code==200:
            print('successully get page')
            return response.json()
    except requests.ConnectionError as e:
        retry_id_list.append(case_id)
        print('error: ',e.args)
        return 0

def parse_page(json):
    if json:
        sell_items = json.get('sell_order_summary')
        html = etree.HTML(sell_items)
        num_and_price = html.xpath('//span/text()')
        price = num_and_price[1].split(' ')[1]
        num = num_and_price[0]
#        print(price)
#        print(num)
        return num,price

case_id_list = ['1275323','1269049','1546282','1913364','15490345',##CSGO	电竞2013	英勇大	CSGO2	电竞2013冬	
                '3438414','6820494','7177182','8987853',#冬季攻势	CSGO3	凤凰	猎杀者
                '14962905','15490346','23853214','29205213',#突围	电竞2014夏	先锋	幻彩
                '40091990','49359031','67060949','84444464',#幻彩2	弯曲	暗影	左轮
                '139654771','149865785','156110183','165027636',#野火	幻彩3	伽玛	伽玛2	
                '175854202','175880240','175896275','175917239',#手套	光谱	九头蛇	光谱2
                '175966708','175999886','176024744','176042493'] #命悬一线，地平线，头号，棱彩    

num_lst = []
price_lst = []
_time_ = time.strftime("%Y.%m.%d",time.localtime())  #获取当前时间
num_lst.append(_time_)     
price_lst.append(' ')

index = 0
retry_index_list = []

if __name__ =='__main__':
    for case_id in case_id_list:
        index = index + 1
        json = get_page(case_id)
        if json==0:
            num_lst.append(' ')
            price_lst.append(' ')
            retry_index_list.append(index)
        else:           
            data = parse_page(json)
            if int(data[0])>1000000:                               #调整箱子在售数量单位和精确度
                num_w = round(int(data[0])*0.0001,1)
            elif int(data[0])>10000:
                num_w = round(int(data[0])*0.0001,2)
            elif int(data[0])>1000:
                num_w = round(int(data[0])*0.0001,3)
            else:
                num_w = int(data[0])*0.0001
                
            num_lst.append(str(num_w))
            price_lst.append(data[1])
        time.sleep(random.randint(2,6))
        
    k_ = 0
    for retry_id in retry_id_list:
        print('retrying')
        json = get_page(retry_id)
        data = parse_page(json)
        if int(data[0])>1000000:                               #调整箱子在售数量单位和精确度
            num_w = round(int(data[0])*0.0001,1)
        elif int(data[0])>10000:
            num_w = round(int(data[0])*0.0001,2)
        elif int(data[0])>1000:
            num_w = round(int(data[0])*0.0001,3)
        else:
            num_w = int(data[0])*0.0001        
        
        num_lst[retry_index_list[k_]] = str(num_w)
        price_lst[retry_index_list[k_]] = data[1]
        k_=k_+1
        time.sleep(random.randint(2,6))
            
    with open('case_data.csv','a+') as csvfile:   #写入csv文件
        writer=csv.writer(csvfile)
        writer.writerow(num_lst)
        writer.writerow(price_lst)