一、简介
OpenWrt路由操作系统的框架基础软件有很多,大部分是通用的软件模块,如 dhcp 、dnsmasq、iproute、cmwp、、ipsec等等;OpenWrt还集成部分具有专属特征软件模块,也是OpenWRT系统核心框架软件组件,从此篇开始分析 《OpenWrt系统框架基础软件模块》系列文章。
OpenWrt 核心软件:procd、uci、libubox、ubus、ubox、luci、netifd 软件组件内容,此部分软件模块是构成OpenWrt框架基础软件。
procd 部分源码内容涉及内容较多,笔者前面几篇文章《详解 OpenWRT系统框架基础软件模块之libubox》等,都是为分析 procd、netifd 两部分内容准备铺垫,请同学们回顾一下前面的内容,在看此篇文章。
OpenWRT 系统中各软件组件之间的关系,如下图:
此图仅是把部分软件模块标识出来,其中还有很多软件模块如:uhttpd、rpcd、mwan3、hotplug、coldplug等等模块,都是依托openWRT的系统ubus总线来构建。此框图可以快速建立系统软件组件之间关系。
我们先看看procd 文件夹中的源码结构,CmakeList.txt 文件是项目生产依据。关键内容如下:
cmake_minimum_required(VERSION 2.6)
PROJECT(procd C)
INCLUDE(GNUInstallDirs)
ADD_DEFINITIONS(-Os -ggdb -Wall -Werror --std=gnu99 -Wmissing-declarations)
SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "")
IF(APPLE)
INCLUDE_DIRECTORIES(/opt/local/include)
LINK_DIRECTORIES(/opt/local/lib)
ENDIF()
ADD_LIBRARY(setlbf SHARED service/setlbf.c)
INSTALL(TARGETS setlbf
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
)
SET(SOURCES procd.c signal.c state.c inittab.c rcS.c ubus.c system.c sysupgrade.c
service/service.c service/instance.c service/validate.c service/trigger.c service/watch.c
utils/utils.c)
IF(NOT DISABLE_INIT)
SET(SOURCES ${SOURCES} watchdog.c plug/coldplug.c plug/hotplug.c)
ENDIF()
IF(ZRAM_TMPFS)
ADD_DEFINITIONS(-DZRAM_TMPFS)
SET(SOURCES_ZRAM initd/zram.c)
ENDIF()
add_subdirectory(upgraded)
// 第一 系统初始化 /sbin/init 线程
// 内核检索 evn arg 环境变量、执行用户态初始化线程 /sbin/init;
IF(DISABLE_INIT)
ADD_DEFINITIONS(-DDISABLE_INIT)
ELSE()
ADD_EXECUTABLE(init initd/init.c initd/early.c initd/preinit.c initd/mkdev.c sysupgrade.c watchdog.c
utils/utils.c ${SOURCES_ZRAM})
TARGET_LINK_LIBRARIES(init ${LIBS})
INSTALL(TARGETS init
RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
)
// 第二 procd 线程
ADD_EXECUTABLE(procd ${SOURCES})
TARGET_LINK_LIBRARIES(procd ${LIBS})
INSTALL(TARGETS procd
RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
)
// 第三 udevtrigger 设备守护线程(udev)
ADD_EXECUTABLE(udevtrigger plug/udevtrigger.c)
INSTALL(TARGETS udevtrigger
RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
)
ENDIF()
//第四 utrace 用户空间 trace 工具
IF(UTRACE_SUPPORT)
ADD_EXECUTABLE(utrace trace/trace.c)
TARGET_LINK_LIBRARIES(utrace ${ubox} ${json} ${blobmsg_json})
INSTALL(TARGETS utrace
RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
)
ADD_DEPENDENCIES(utrace syscall-names-h)
此项目中涉及内容较多,请参考文件中注释。此部是OpenWrt系统核心基础功能实现,通过此部分内容可以清晰看到 系统启动流程。由于内容涉及面较宽,把分析源码过程文件贴出来,供大家参考作为源码关系导图使用。
内核源码 init/main.c 文件中
static int __ref kernel_init(void *unused)
{
int ret;
kernel_init_freeable();
/* need to finish all async __init code before freeing the memory */
async_synchronize_full();
ftrace_free_init_mem();
free_initmem();
mark_readonly();
system_state = SYSTEM_RUNNING;
numa_default_policy();
rcu_end_inkernel_boot();
if (ramdisk_execute_command) {
ret = run_init_process(ramdisk_execute_command);
if (!ret)
return 0;
pr_err("Failed to execute %s (error %d)\n",
ramdisk_execute_command, ret);
}
/*
* We try each of these until one succeeds.
*
* The Bourne shell can be used instead of init if we are
* trying to recover a really broken machine.
*/
if (execute_command) {
ret = run_init_process(execute_command);
if (!ret)
return 0;
panic("Requested init %s failed (error %d).",
execute_command, ret);
}
if (!try_to_run_init_process("/sbin/init") ||
!try_to_run_init_process("/etc/init") ||
!try_to_run_init_process("/bin/init") ||
!try_to_run_init_process("/bin/sh"))
return 0;
panic("No working init found. Try passing init= option to kernel. "
"See Linux Documentation/admin-guide/init.rst for guidance.");
}
该函数检查 execute_command 环境参数(env arg)为空时、执行用户态初始化线程 /sbin/init,自此进入到用户态空间;
int main(int argc, char **argv)
(1). 初始化信号量
ulog_open(ULOG_KMSG, LOG_DAEMON, "init");
sigaction(SIGTERM, &sa_shutdown, NULL);
sigaction(SIGUSR1, &sa_shutdown, NULL);
sigaction(SIGUSR2, &sa_shutdown, NULL);
sigaction(SIGPWR, &sa_shutdown, NULL);
(2).
early();
(3).
cmdline();
(4).
watchdog_init(1);
(5).
pid = fork();
if (!pid) {
char *kmod[] = { "/sbin/kmodloader", "/etc/modules-boot.d/", NULL };
if (debug < 3)
patch_stdio("/dev/null");
execvp(kmod[0], kmod); // fork() 子进程执行函数 execvp() 函数
/*
* 函数说明:execvp()会从PATH 环境变量所指的目录中查找符合参数file 的文件名, 找到后便执行该文件, 然后将第二个参数argv 传给该欲执行的文件。
* 返回值:如果执行成功则函数不会返回, 执行失败则直接返回-1, 失败原因存于errno 中.
*
* 执行 /sbin/kmodloader 函数
* 执行 /etc/modules-boot.d/ 路径下的模块初始化程序,
* root@ixeRouter:~# ls /etc/modules-boot.d/
* 02-crypto-hash 30-fs-squashfs 50-usb-ohci
* 04-crypto-crc32c 30-gpio-button-hotplug 50-usb-uhci
* 09-crypto-aead 35-usb-ehci 51-usb-ohci-pci
* 09-crypto-manager 40-scsi-core 54-usb3
* 15-libphy 40-usb2 60-leds-gpio
* 15-mii 41-ata-ahci mmc
* 20-usb-core 42-usb2-pci sdhci-mt7620
*/
ERROR("Failed to start kmodloader: %m\n");
exit(EXIT_FAILURE);
}
if (pid <= 0) {
ERROR("Failed to start kmodloader instance: %m\n");
} else {//主线程执行下面程序
const struct timespec req = {0, 10 * 1000 * 1000};
int i;
for (i = 0; i < 1200; i++) {
if (waitpid(pid, NULL, WNOHANG) > 0)
break;
nanosleep(&req, NULL);
watchdog_ping();
}
}
(6).
uloop_init();
(7).
preinit();
char *init[] = { "/bin/sh", "/etc/preinit", NULL };
char *plug[] = { "/sbin/procd", "-h", "/etc/hotplug-preinit.json", NULL };
plugd_proc.pid = fork(); // fork() 创建子进程
if (!plugd_proc.pid) {
execvp(plug[0], plug); // 子进程 执行/sbin/procd 函数, 1# 线程创建成功
ERROR("Failed to start plugd: %m\n");
exit(EXIT_FAILURE);
}
if (plugd_proc.pid <= 0) {
ERROR("Failed to start new plugd instance: %m\n");
return;
}
uloop_process_add(&plugd_proc); // 把 procd 放到 uloop 链表中
setenv("PREINIT", "1", 1);
fd = creat("/tmp/.preinit", 0600);
if (fd < 0)
ERROR("Failed to create sentinel file: %m\n");
else
close(fd);
preinit_proc.cb = spawn_procd;
preinit_proc.pid = fork(); //fork() 创建子进程
if (!preinit_proc.pid) {
execvp(init[0], init); // 子进程 执行/bin/sh , 线程创建成功
ERROR("Failed to start preinit: %m\n");
exit(EXIT_FAILURE);
}
if (preinit_proc.pid <= 0) {
ERROR("Failed to start new preinit instance: %m\n");
return;
}
uloop_process_add(&preinit_proc); 把 spawn_procd 放到 uloop 链表中
(8).
uloop_run();
static void spawn_procd(struct uloop_process *proc, int ret)
{
char *wdt_fd = watchdog_fd();
char *argv[] = { "/sbin/procd", NULL};
if (plugd_proc.pid > 0)
kill(plugd_proc.pid, SIGKILL);
unsetenv("INITRAMFS");
unsetenv("PREINIT");
unlink("/tmp/.preinit");
check_sysupgrade(); //检查系统软件更新
DEBUG(2, "Exec to real procd now\n");
if (wdt_fd)
setenv("WDTFD", wdt_fd, 1);
check_dbglvl();
if (debug > 0) {
snprintf(dbg, 2, "%d", debug);
setenv("DBGLVL", dbg, 1);
}
execvp(argv[0], argv);
}
static void check_sysupgrade(void)
{
char *prefix = NULL, *path = NULL, *command = NULL;
size_t n;
if (chdir("/"))
return;
FILE *sysupgrade = fopen("/tmp/sysupgrade", "r");
if (!sysupgrade)
return;
n = 0;
if (getdelim(&prefix, &n, 0, sysupgrade) < 0)
goto fail;
n = 0;
if (getdelim(&path, &n, 0, sysupgrade) < 0)
goto fail;
n = 0;
if (getdelim(&command, &n, 0, sysupgrade) < 0)
goto fail;
fclose(sysupgrade);
sysupgrade_exec_upgraded(prefix, path, NULL, command, NULL); // 执行系统升级 /sbin/upgraded
while (true)
sleep(1);
fail:
fclose(sysupgrade);
free(prefix);
free(path);
free(command);
}
(1).
uloop_init();
(1.0)
uloop_init_pollfd();
epoll || kqueue 初始化
(1.1).
waker_init();
waker_fd.cb = waker_consume()
uloop_fd_add(&waker_fd, ULOOP_READ)
(1.2).
uloop_setup_signals(true);
uloop_install_handler(SIGINT, uloop_handle_sigint, &old_sigint, add); //初始化信号 SIGINT
uloop_install_handler(SIGTERM, uloop_handle_sigint, &old_sigterm, add); //初始化信号 SIGTERM
uloop_install_handler(SIGCHLD, uloop_sigchld, &old_sigchld, add); //初始化信号 SIGCHLD
uloop_ignore_signal(SIGPIPE, add); //初始化信号 SIGCHLD
(2).
procd_signal()
sigaction(SIGTERM, &sa_shutdown, NULL);
sigaction(SIGINT, &sa_shutdown, NULL);
sigaction(SIGUSR1, &sa_shutdown, NULL);
sigaction(SIGUSR2, &sa_shutdown, NULL);
sigaction(SIGPWR, &sa_shutdown, NULL);
signal_shutdown(); //reboot
sigaction(SIGSEGV, &sa_crash, NULL);
sigaction(SIGBUS, &sa_crash, NULL);
do_reboot(); //reboot
sigaction(SIGHUP, &sa_dummy, NULL);
sigaction(SIGKILL, &sa_dummy, NULL);
sigaction(SIGSTOP, &sa_dummy, NULL);
signal_dummy(); //输出错误提示信息
#ifndef DISABLE_INIT
reboot(RB_DISABLE_CAD); //
#endif
(3).
if (getpid() != 1)
procd_connect_ubus(); // 初始化 ubus 总线
else
procd_state_next(); //循环状态机变量,切换程序运行状态
void procd_state_next(void)
{
DEBUG(4, "Change state %d -> %d\n", state, state + 1);
state++;
state_enter(); //调用状态机处理函数 state_enter()
}
(4).
uloop_run();
uloop_run_timeout(-1);
uloop_handle_processes(); //
uloop_run_events(next_time); // 执行ubus的 RPC 回调函数
(5).
uloop_done();
// 状态机参数定义
static int state = STATE_NONE;
enum {
STATE_NONE = 0,
STATE_EARLY,
STATE_UBUS,
STATE_INIT,
STATE_RUNNING,
STATE_SHUTDOWN,
STATE_HALT,
__STATE_MAX,
};
// 状态机运行次序: STATE_EARLY -> STATE_UBUS -> STATE_INIT -> STATE_RUNNING;
// STATE_SHUTDOWN、STATE_HALT
static void state_enter(void)
{
char ubus_cmd[] = "/sbin/ubusd";
switch (state) {
case STATE_EARLY:
LOG("- early -\n");
watchdog_init(0); // 初始化 看门狗
hotplug("/etc/hotplug.json"); // 热插拔函数入口参数是:/etc/hotplug.json 见文件内容
procd_coldplug(); // 冷插拔
break;
case STATE_UBUS:
// try to reopen incase the wdt was not available before coldplug
watchdog_init(0); // 初始化 看门狗
set_stdio("console"); // 设置console参数
LOG("- ubus -\n");
procd_connect_ubus(); // 连接ubus总线
service_start_early("ubus", ubus_cmd); // 启动 RPC
break;
case STATE_INIT:
LOG("- init -\n");
procd_inittab(); //执行 /etc/inittab 文件中脚本函数,
::sysinit:/etc/init.d/rcS S boot
::shutdown:/etc/init.d/rcS K shutdown
::askconsole:/usr/libexec/login.sh
procd_inittab_run("respawn"); //
procd_inittab_run("askconsole");
procd_inittab_run("askfirst");
procd_inittab_run("sysinit");
// switch to syslog log channel
ulog_open(ULOG_SYSLOG, LOG_DAEMON, "procd");
break;
case STATE_RUNNING:
LOG("- init complete -\n");
procd_inittab_run("respawnlate");
procd_inittab_run("askconsolelate");
break;
case STATE_SHUTDOWN:
/* Redirect output to the console for the users' benefit */
set_console(); // 依据 uboot传递到 /proc/cmdline文件中的参数,重定向 console 口
LOG("- shutdown -\n");
procd_inittab_run("shutdown"); // 初始化 shutdown 回调函数
sync();
break;
case STATE_HALT:
// To prevent killed processes from interrupting the sleep
signal(SIGCHLD, SIG_IGN);
LOG("- SIGTERM processes -\n");
kill(-1, SIGTERM);
sync();
sleep(1);
LOG("- SIGKILL processes -\n");
kill(-1, SIGKILL);
sync();
sleep(1);
#ifndef DISABLE_INIT
perform_halt();
#else
exit(EXIT_SUCCESS);
#endif
break;
default:
ERROR("Unhandled state %d\n", state);
return;
};
}
static struct uloop_timeout wdt_timeout;
static int wdt_fd = -1;
static int wdt_drv_timeout = 30; //看门狗超时时间:30s
static int wdt_frequency = 5; //喂狗频率:5s
static bool wdt_magicclose = false;
void watchdog_init(int preinit)
{
wdt_timeout.cb = watchdog_timeout_cb;
if (watchdog_open(!preinit) < 0)
return;
LOG("- watchdog -\n");
watchdog_set_drv_timeout(); //设置看门狗的时间=wdt_drv_timeout=30s
watchdog_timeout_cb(&wdt_timeout);
DEBUG(4, "Opened watchdog with timeout %ds\n", watchdog_timeout(0));
}
[
[ "case", "ACTION", {
"add": [
[ "if",
[ "and",
[ "has", "MAJOR" ],
[ "has", "MINOR" ]
],
[
[ "if",
[ "eq", "DEVNAME",
[ "null", "full", "ptmx", "zero", "tty", "net", "random", "urandom" ]
],
[
[ "makedev", "/dev/%DEVNAME%", "0666" ],
[ "return" ]
]
],
[ "if",
[ "regex", "DEVNAME", "^snd" ],
[ "makedev", "/dev/%DEVNAME%", "0660", "audio" ]
],
[ "if",
[ "regex", "DEVNAME", "^tty" ],
[ "makedev", "/dev/%DEVNAME%", "0660", "dialout" ]
],
[ "if",
[ "has", "DEVNAME" ],
[ "makedev", "/dev/%DEVNAME%", "0600" ]
]
]
],
[ "if",
[ "has", "FIRMWARE" ],
[
[ "exec", "/sbin/hotplug-call", "%SUBSYSTEM%" ],
[ "load-firmware", "/lib/firmware" ],
[ "return" ]
]
]
],
"remove" : [
[ "if",
[ "and",
[ "has", "DEVNAME" ],
[ "has", "MAJOR" ],
[ "has", "MINOR" ]
],
[ "rm", "/dev/%DEVNAME%" ]
]
]
} ],
[ "if",
[ "and", //键盘事件
[ "has", "BUTTON" ],
[ "eq", "SUBSYSTEM", "button" ]
],
[ "button", "/etc/rc.button/%BUTTON%" ] // 键盘事件执行的脚本位置 /etc/rc.button/
],
[ "if",
[ "and", // usb串口事件
[ "eq", "SUBSYSTEM", "usb-serial" ],
[ "regex", "DEVNAME",
[ "^ttyUSB", "^ttyACM" ]
]
],
[ "exec", "/sbin/hotplug-call", "tty" ],
[ "if",
[ "isdir", "/etc/hotplug.d/%SUBSYSTEM%" ],
[ "exec", "/sbin/hotplug-call", "%SUBSYSTEM%" ]
]
]
]
// 数组 handlers[] 初始化参数
static struct init_handler handlers[] = {
{
.name = "sysinit",
.cb = runrc, // 执行 /etc/rc.d 路径下的文件
}, {
.name = "shutdown",
.cb = runrc,
}, {
.name = "askfirst",
.cb = askfirst,
.multi = 1,
}, {
.name = "askconsole",
.cb = askconsole,
.multi = 1,
}, {
.name = "respawn",
.cb = rcrespawn,
.multi = 1,
}, {
.name = "askconsolelate",
.cb = askconsole,
.multi = 1,
}, {
.name = "respawnlate",
.cb = rcrespawn,
.multi = 1,
}
};
struct init_action {
struct list_head list;
char *id;
char *argv[MAX_ARGS];
char *line;
struct init_handler *handler;
struct uloop_process proc;
int respawn;
struct uloop_timeout tout;
};
static const char *tab = "/etc/inittab";
static char *ask = "/sbin/askfirst";
static LIST_HEAD(actions);
入口函数
void procd_inittab(void){
// 读取 /etc/inittab 文件中内容,把此文件中 ::sysinit:/etc/init.d/rcS S boot
// ::shutdown:/etc/init.d/rcS K shutdown
// ::askconsole:/usr/libexec/login.sh
// 内容添加到 handlers[] 数组中,
while (fgets(line, LINE_LEN, fp)) {
add_action(a, tags[TAG_ACTION]) //逐项添加到
}
}
// 回调函数中,都调用此 fork_worker , fork 其他线程。
static void fork_worker(struct init_action *a)
{
pid_t p;
a->proc.pid = fork();
if (!a->proc.pid) {
p = setsid();
if (patch_stdio(a->id))
ERROR("Failed to setup i/o redirection\n");
ioctl(STDIN_FILENO, TIOCSCTTY, 1);
tcsetpgrp(STDIN_FILENO, p);
execvp(a->argv[0], a->argv); //给*a 参数,fork 子线程
ERROR("Failed to execute %s: %m\n", a->argv[0]);
exit(-1);
}
if (a->proc.pid > 0) {
DEBUG(4, "Launched new %s action, pid=%d\n",
a->handler->name,
(int) a->proc.pid);
uloop_process_add(&a->proc); //把 *a 注册到 ubus 总线上
}
}
// uloop 线程的回调参数
struct uloop_process
{
struct list_head list;
bool pending;
uloop_process_handler cb;
pid_t pid;
};
// ubus 总线rpc接口的全局链表
static struct list_head timeouts = LIST_HEAD_INIT(timeouts); //超时回调 rpc 接口函数
static struct list_head processes = LIST_HEAD_INIT(processes); //显性回调 rpc 接口函数
// 注册函数
int uloop_process_add(struct uloop_process *p)
{
struct uloop_process *tmp;
struct list_head *h = &processes;
if (p->pending)
return -1;
list_for_each_entry(tmp, &processes, list) {
if (tmp->pid > p->pid) {
h = &tmp->list;
break;
}
}
list_add_tail(&p->list, h); //把 *p 添加到 processes链表中,注册到 ubus 总线,
p->pending = true;
return 0;
}
此部分是设备文件系统注册过程
int main(int argc, char *argv[], char *envp[])
{
openlog("udevtrigger", LOG_PID | LOG_CONS, LOG_DAEMON);
scan_subdir("/sys/bus", "/devices", false, 1);
scan_subdir("/sys/class", NULL, false, 1); // 处理 /sys/class 目录下设备
if (stat("/sys/class/block", &statbuf) != 0)
scan_subdir("/sys/block", NULL, true, 1); //处理系统分区信息
exit:
closelog();
return 0;
}
// 递归扫描设备目录
static void scan_subdir(const char *base, const char *subdir,
bool insert, int depth)
{
DIR *dir;
struct dirent *dent;
dir = opendir(base);
if (dir == NULL)
return;
for (dent = readdir(dir); dent != NULL; dent = readdir(dir)) {
char dirname[PATH_SIZE];
if (dent->d_name[0] == '.')
continue;
strlcpy(dirname, base, sizeof(dirname));
strlcat(dirname, "/", sizeof(dirname));
strlcat(dirname, dent->d_name, sizeof(dirname));
if (insert) {
int err;
err = device_list_insert(dirname);
if (err)
continue;
}
if (subdir)
strlcat(dirname, subdir, sizeof(base));
if (depth)
scan_subdir(dirname, NULL, true, depth - 1);
}
closedir(dir);
}
// 把设备添加到 设备链表中
static int device_list_insert(const char *path)
{
char devpath[PATH_SIZE];
struct stat statbuf;
dbg("add '%s'" , path);
/* we only have a device, if we have a dev and an uevent file */
if (!device_has_attribute(path, "/dev", S_IRUSR) ||
!device_has_attribute(path, "/uevent", S_IWUSR))
return -1;
strlcpy(devpath, &path[4], sizeof(devpath));
/* resolve possible link to real target */
if (lstat(path, &statbuf) < 0)
return -1;
if (S_ISLNK(statbuf.st_mode))
if (sysfs_resolve_link(devpath, sizeof(devpath)) != 0)
return -1;
trigger_uevent(devpath);
return 0;
}
// 把设备文件映射到内存,按照指定路径
static int sysfs_resolve_link(char *devpath, size_t size)
{
char link_path[PATH_SIZE];
char link_target[PATH_SIZE];
int len;
int i;
int back;
strlcpy(link_path, "/sys", sizeof(link_path));
strlcat(link_path, devpath, sizeof(link_path));
len = readlink(link_path, link_target, sizeof(link_target) - 1);
/*
* 定义函数:int readlink(const char * path, char * buf, size_t bufsiz);
* 函数说明:readlink()会将参数path 的符号连接内容存到参数buf 所指的内存空间,
* 返回的内容不是以NULL作字符串结尾, 但会将字符串的字符数返回.
* 若参数bufsiz 小于符号连接的内容长度, 过长的内容会被截断.
* 返回值:执行成功则传符号连接所指的文件路径字符串, 失败则返回-1, 错误代码存于errno.
*/
if (len <= 0)
return -1;
link_target[len] = '\0';
dbg("path link '%s' points to '%s'", devpath, link_target);
for (back = 0; strncmp(&link_target[back * 3], "../", 3) == 0; back++)
;
dbg("base '%s', tail '%s', back %i", devpath, &link_target[back * 3], back);
for (i = 0; i <= back; i++) {
char *pos = strrchr(devpath, '/');
if (pos == NULL)
return -1;
pos[0] = '\0';
}
dbg("after moving back '%s'", devpath);
strlcat(devpath, "/", size);
strlcat(devpath, &link_target[back * 3], size);
return 0;
}