GitHub 自动下载 Release 固件
一般在 GitHub 上成熟的仓库，都会在 Releases 页面上发布最新稳定的版本。
作为一名嵌入式程序员，就以 Espressif 下的 esp-idf 仓库为例，截至到作者写这篇文档前，最新发布的版本为 ESP-IDF Pre-release v5.0-beta1。
同样作为一名嵌入式程序员，与互联网行业的程序员不同，要经常和 release 的固件打交道。因为这些固件都是经过严格测试的稳定版本，修复了很多 bug 和增加了新的 feature，所以始终保持设备运行最新的 release 固件是很有必要的。但是对于一些没有订阅功能的 release 固件，每次都要手动去 check 下，如果有最新的 release 固件，则首先要通过浏览器下载到本地，在将本地的固件下载到开发板中。于是就萌生了写一个自动化的 Python 脚本来代替这些无意义的重复劳动的想法。
本文就以 Espressif 下的 esp-at 仓库为例，利用 GitHub 的 rest API 写一个自动从 Releases 上下载最新的固件并下载到开发板中的自动化 Python 脚本。省去自己通过浏览器下载固件到本地，在将本地固件下载到开发板中的过程。
源码可以参考作者的 GitHub 仓库。
首先就是将所有与 GitHub 交互的操作封装成 GitDownload 类。主要是用到了以下 rest API:
- Releases
"""github download class"""
import requests
import json
import string
import re
class GitDownload:
"""
github download class
"""
pass
def __init__(self, owner, repository, user_name, token):
self.owner = owner
self.repository = repository
self.rest_url = "https://api.github.com/repos"
self.separator = "/"
self.username = user_name
self.token = token
self.branch = {}
self.release_info = {}
self.release_ver = {}
self.release_modules = {}
self.release_module_download_url = {}
self.session = requests.session()
self.session.auth = (self.username, self.token)
def get_branch(self):
url = self.rest_url + self.separator + self.owner + self.separator + self.repository + self.separator + "branches"
response = self.session.get(url)
if response.status_code == 200:
branch_info = json.loads(response.text)
branch_num = len(branch_info)
if branch_num:
self.branch = {}
for i in range(0, branch_num):
self.branch[i] = branch_info[i].get("name")
return self.branch
def get_release_info(self):
self.release_info.clear()
url = "{0}/{1}/{2}/releases".format(self.rest_url, self.owner, self.repository)
response = self.session.get(url)
status_code = response.status_code
if status_code == 200:
# convert JSON data to Python object, here is list
release_info = json.loads(response.text)
# get each firmware information corresponding to each release
release_num = len(release_info)
if release_num:
for i in range(0, release_num):
firmware_info = re.findall(r'\[ESP.*?zip\)', release_info[i].get("body"))
self.release_info[release_info[i].get("name")] = firmware_info
else:
print(f"get release info failed, error:{status_code}")
return self.release_info
def get_release_version(self):
self.release_ver.clear()
# first check self.release_info
if len(self.release_info):
index = 0
for key, value in self.release_info.items(): # key is release version here
self.release_ver[index] = key
index = index + 1
else:
url = "{0}/{1}/{2}/releases".format(self.rest_url, self.owner, self.repository)
response = self.session.get(url)
status_code = response.status_code
if status_code == 200:
# convert JSON data to Python list, here is list
release_info = json.loads(response.text)
release_num = len(release_info)
if release_num:
for i in range(0, release_num):
# each element in the list is a dictionary
self.release_ver[i] = release_info[i].get("name")
else:
print(f"get release version failed, error:{status_code}")
return self.release_ver
def get_release_modules(self, version):
self.release_modules.clear()
for release_version in self.release_ver.values():
if release_version == version:
release_modules_info = self.release_info.get(version)
# resolves supported modules from a list of specified version information
# the content of the list are as follows:
# ['[ESP32-C3-MINI-1_AT_Bin_V2.4.1.0.zip](https://github.com/espressif/esp-at/files/9289473/ESP32-C3-MINI-1_AT_Bin_V2.4.1.0.zip)']
for i in range(0, len(release_modules_info)):
module = re.findall(r'(?<=\[).*?(?=_AT)', release_modules_info[i])
self.release_modules[i] = module[0]
return self.release_modules
def get_release_module_download_url(self, version):
self.release_module_download_url.clear()
for release_version in self.release_ver.values():
if release_version == version:
release_modules_info = self.release_info.get(version)
# resolves supported modules from a list of specified version information
# the content of the list are as follows:
# ['[ESP32-C3-MINI-1_AT_Bin_V2.4.1.0.zip](https://github.com/espressif/esp-at/files/9289473/ESP32-C3-MINI-1_AT_Bin_V2.4.1.0.zip)']
# in the dictionary, the key is module and the value is URL, the content of the dictionary are as follows:
# {'ESP32-C3-MINI-1': 'https://github.com/espressif/esp-at/files/9289473/ESP32-C3-MINI-1_AT_Bin_V2.4.1.0.zip'}
for i in range(0, len(release_modules_info)):
name = re.findall(r'(?<=\[).*?(?=_AT)', release_modules_info[i])
url = re.findall(r'https.*?(?=\))', release_modules_info[i])
self.release_module_download_url[name[0]] = url[0]
return self.release_module_download_url
def get_spec_release_module_download_url(self, version, module_name):
self.get_release_module_download_url(version)
return self.release_module_download_url.get(module_name)
其次就是将剩余的一些操作（包括指定下载版本、指定下载模块、下载固件并解压、将固件下载到对应开发板中），这部分操作放到了 download.py 中。
#!/usr/bin/env python3
#
# Copyright (C) 2021 alson <tx_danyang@163.com>
# This file is subject to the terms and conditions defined in
# file 'LICENSE', which is part of this source code package.
from distutils.log import debug
import sys
import os
import getopt
import logging
import requests
import zipfile
import serial
import serial.tools.list_ports
from tqdm import tqdm
from gitdownload import GitDownload
def get_bin_path(file_name):
file_path = ""
if not os.path.exists(file_name):
logging.error(f"no specified file found")
return file_path
ret = zipfile.is_zipfile(file_name)
if not ret:
logging.error(f"no specified format (zip) found")
return file_path
else:
if not os.path.exists(file_name[0:-4]):
fz = zipfile.ZipFile(file_name, 'r')
for file in fz.namelist():
fz.extract(file, file_name[0:-4])
for root, dirs, files in os.walk(file_name[0:-4]):
if root.split('/')[-1] == "factory":
for file in files:
if file.split('.')[-1] == "bin":
file_path = os.path.join(root, file)
return file_path
def get_serial_ports_list():
serial_ports_dict = {}
serial_ports_list = list(serial.tools.list_ports.comports())
index = 0
if len(serial_ports_list):
for port_name in list(serial_ports_list):
serial_ports_dict[index] = port_name[0]
index += 1
return serial_ports_dict
def download_firmware(url):
download = requests.head(url, allow_redirects=True)
header = download.headers
file_size = int(header.get('Content-Length'))
logging.info(f"file_size is {file_size}")
file_name = (header.get('Content-Disposition')).split('=')[-1]
logging.info(f"file_name is {file_name}")
if os.path.exists(file_name):
logging.info(f"{file_name} already exists")
return file_name
pbar = tqdm(desc = "downloaded: ", total=file_size, unit='B', unit_scale=True)
download = requests.get(url, stream=True)
downloaded_size = 0
with open(file_name, "wb") as fb:
for size in download.iter_content(1024):
if size:
fb.write(size)
downloaded_size += len(size)
percent = int((downloaded_size / file_size) * 100)
pbar.update(len(size))
pbar.close()
return file_name
if __name__ == '__main__':
opts,args = getopt.getopt(sys.argv[1:], '-h -d:',['help', 'debug='])
for arg_name, arg_value in opts:
if arg_name in ('-d', '--debug'):
if arg_value in ("debug", "info", "warning", "error", "critical"):
if arg_value == "debug":
logging.basicConfig(level=logging.DEBUG)
elif arg_value == "info":
logging.basicConfig(level=logging.INFO)
elif arg_value == "warning":
logging.basicConfig(level=logging.WARNING)
elif arg_value == "error":
logging.basicConfig(level=logging.ERROR)
else:
logging.basicConfig(level=logging.CRITICAL)
else:
logging.error("debug arg must be in {\"debug\", \"info\", \"warning\", \"error\", \"critical\"}")
sys.exit(-1)
if arg_name in ('-h', '--help'):
print(f"usage: download.py [-h][-d {{debug, info, warning, error, critical}}]")
sys.exit(0)
# get release information
# token please refer to you GitHuh account Setting -> Developer settings -> Personal access tokens
git_download = GitDownload("espressif", "esp-at", "Alson-tang", "ghp_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx")
release_info = git_download.get_release_info()
if not len(release_info):
logging.error(f"no release info found")
sys.exit(-1)
logging.debug(f"{release_info}")
# get release version
release_ver = git_download.get_release_version()
if len(release_ver):
for key, value in release_ver.items():
print(f"{key}: {value}")
version_index = input("please enter the version index:")
version = release_ver.get(int(version_index))
else:
logging.error(f"no release version found")
sys.exit(-1)
# get modules under the specified version
release_modules = git_download.get_release_modules(version)
if len(release_modules):
for key, value in release_modules.items():
print(f"{key}: {value}")
module_index = input("please enter the module index: ")
module = release_modules.get(int(module_index))
else:
logging.error(f"no release module found")
sys.exit(-1)
# get the firmware download address of the specified version and the specified module
url = git_download.get_spec_release_module_download_url(version, module)
logging.info(f"url is {url}")
download_file_name = download_firmware(url)
logging.info(f"download file name is {download_file_name}")
# traverse the directory to find the bin file
bin_file_path = get_bin_path(download_file_name)
if bin_file_path == "":
logging.error(f"no bin file found")
sys.exit(-1)
logging.info(f"bin path is {bin_file_path}")
# get available serial ports
serial_ports = get_serial_ports_list()
if len(serial_ports):
for key, value in serial_ports.items():
print(f"{key}: {value}")
serial_port_index = input("please enter the serial port index: ")
serial_port = serial_ports.get(int(serial_port_index))
else:
logging.error(f"no available serial port")
sys.exit(-1)
# call esptool.py to download firmware
if module.split('-')[1] == "C3":
chip_module = "esp32c3"
else:
chip_module = "esp32"
command = "esptool.py -p {0} -b 921600 --before default_reset --after hard_reset --chip {1} write_flash --flash_mode dio --flash_size detect --flash_freq 40m 0x0 {2}".format(serial_port, chip_module, bin_file_path)
logging.info(f"{command}")
os.system(command)
sys.exit(0)
整个代码的逻辑其实没那么复杂，总结起来可以分为以下几步：
- 获取所有发布的版本
- 指定要下载的版本，获取该版本支持的模块
- 指定模块后下载固件到当前目录（如果当前目录下已有对应的固件版本，则跳过）
- 获取所有可用串口
- 指定串口后将下载的固件自动下载到开发板中
可以先阅读 README 了解更多详细的信息。
以下是运行脚本的 LOG
-
获取所有发布的版本（esp-at 发布的所有版本都将在此处列出。 假设你输入
0，你要下载的版本是v2.4.1.0）0: v2.4.1.0 1: v2.4.0.0 2: v2.3.0.0_esp32c3 3: v2.2.1.0_esp8266 4: v2.2.0.0_esp32 5: v2.2.0.0_esp8266 6: v2.2.0.0_esp32c3 7: v2.1.0.0_esp32s2 8: v2.1.0.0_esp8266 9: v2.1.0.0_esp32 10: v2.1.0.0-rc2_esp32 11: v2.1.0.0-rc1_esp8266 12: v2.1.0.0-rc1_esp32 13: v2.0.0.0_esp32 14: v2.0.0.0_esp8266 15: v1.2.0.0 16: v1.1.3.0 17: v1.1.2.0 18: v1.1.1.0 19: v1.1.0.0 20: v1.0.0.0 21: v0.10.0.0 please enter the version index: -
指定要下载的版本，获取该版本支持的模块（
v2.4.1.0下的所有模块都将在此处列出。 这个版本只对应一个模块，这里输入0）0: ESP32-C3-MINI-1 please enter the module index: -
指定模块后下载固件到当前目录（如果当前目录下已有对应的固件版本，则跳过）
-
获取所有可用串口
0: /dev/ttyS0 1: /dev/ttyUSB1 2: /dev/ttyUSB0 please enter the serial port index: -
指定串口后将下载的固件自动下载到开发板
esptool.py v3.1-dev Serial port /dev/ttyUSB0 Connecting.... WARNING: This chip doesn't appear to be a ESP32-C3 (chip magic value 0x1b31506f). Probably it is unsupported by this version of esptool. Chip is unknown ESP32-C3 (revision 3) Features: Wi-Fi Crystal is 40MHz MAC: 84:f7:03:09:17:f4 Uploading stub... Running stub... Stub running... Changing baud rate to 921600 Changed. Configuring flash size... Auto-detected Flash size: 4MB Compressed 4194304 bytes to 878431... Writing at 0x000fa2d8... (40 %)