openwrt procd启动流程和脚本分析

Linux内核执行start_kernel函数时会调用kernel_init来启动init进程，流程如下图：

graph LR A[start_kernel] -->B(rest_init) B --> C(kernel_init) C --> D[try_to_run_init_process]

kernel_init部分代码如下：

 994     if (execute_command) {
 995         ret = run_init_process(execute_command);
 996         if (!ret)
 997             return 0;
 998         panic("Requested init %s failed (error %d).",
 999               execute_command, ret);
1000     }
1001     if (!try_to_run_init_process("/sbin/init") ||
1002         !try_to_run_init_process("/etc/init") ||
1003         !try_to_run_init_process("/bin/init") ||
1004         !try_to_run_init_process("/bin/sh"))
1005         return 0;
1006
1007     panic("No working init found.  Try passing init= option to kernel. "
1008           "See Linux Documentation/init.txt for guidance.");

接着分析openwrt中package/system/procd/Makefile，这里将procd源码编译生成的可执行文件安装到文件系统的/sbin目录中。

define Package/procd/install
    $(INSTALL_DIR) $(1)/sbin $(1)/etc $(1)/lib/functions

    $(INSTALL_BIN) $(PKG_INSTALL_DIR)/usr/sbin/{init,procd,askfirst,udevtrigger} $(1)/sbin/
    $(INSTALL_DATA) $(PKG_INSTALL_DIR)/usr/lib/libsetlbf.so $(1)/lib
    $(INSTALL_BIN) ./files/reload_config $(1)/sbin/
    $(INSTALL_DATA) ./files/hotplug*.json $(1)/etc/
    $(INSTALL_DATA) ./files/procd.sh $(1)/lib/functions/
endef

查看procd源码目录的CMakeList.txt，以init为例，对应源码编译文件如下

 56 IF(DISABLE_INIT)
 57 ADD_DEFINITIONS(-DDISABLE_INIT)
 58 ELSE()
 59 ADD_EXECUTABLE(init initd/init.c initd/early.c initd/preinit.c initd/mkdev.c watchdog.c
 60     utils/utils.c ${SOURCES_ZRAM})
 61 TARGET_LINK_LIBRARIES(init ${LIBS})
 62 INSTALL(TARGETS init
 63     RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
 64 )
 65
 66 ADD_EXECUTABLE(udevtrigger plug/udevtrigger.c)
 67 INSTALL(TARGETS udevtrigger
 68     RUNTIME DESTINATION ${CMAKE_INSTALL_SBINDIR}
 69 )
 70 ENDIF()

main函数入口位于initd/init.c

int
main(int argc, char **argv)
{
    pid_t pid;

    ulog_open(ULOG_KMSG, LOG_DAEMON, "init");

    sigaction(SIGTERM, &sa_shutdown, NULL);
    sigaction(SIGUSR1, &sa_shutdown, NULL);
    sigaction(SIGUSR2, &sa_shutdown, NULL);

	early();
    
	cmdline();
    
	watchdog_init(1);

	pid = fork();
    if (!pid) {
        char *kmod[] = { "/sbin/kmodloader", "/etc/modules-boot.d/", NULL };

        if (debug < 3)
            patch_stdio("/dev/null");

        execvp(kmod[0], kmod);
        ERROR("Failed to start kmodloader
");
        exit(-1);
    }
    if (pid <= 0) {
        ERROR("Failed to start kmodloader instance
");
    } else {
        int i;

        for (i = 0; i < 1200; i++) {
            if (waitpid(pid, NULL, WNOHANG) > 0)
                break;
            usleep(10 * 1000);
            watchdog_ping();
        }
    }
    
	uloop_init();
	preinit();
    uloop_run();

    return 0;
}

uloop_init实现位于libubox源码uloop.c

int uloop_init(void)
{
	if (uloop_init_pollfd() < 0)
        return -1;

    if (waker_init() < 0) {
		uloop_done();
		return -1;
    }

    return 0;
}

static int uloop_init_pollfd(void)
{
    if (poll_fd >= 0)
        return 0;

    poll_fd = epoll_create(32);	
    if (poll_fd < 0)
        return -1;

    fcntl(poll_fd, F_SETFD, fcntl(poll_fd, F_GETFD) | FD_CLOEXEC);
    return 0;
}

preinit实现位于procd源码文件initd/preinit.c

static struct uloop_process preinit_proc;
static struct uloop_process plugd_proc;

void
preinit(void)
{
    char *init[] = { "/bin/sh", "/etc/preinit", NULL };
    char *plug[] = { "/sbin/procd", "-h", "/etc/hotplug-preinit.json", NULL };
    int fd;

    LOG("- preinit -
");

    plugd_proc.cb = plugd_proc_cb;
    plugd_proc.pid = fork();
    if (!plugd_proc.pid) {
        execvp(plug[0], plug);
        ERROR("Failed to start plugd
");
        exit(-1);
    }
    if (plugd_proc.pid <= 0) {
        ERROR("Failed to start new plugd instance
");
        return;
    }
    uloop_process_add(&plugd_proc);	

    setenv("PREINIT", "1", 1);

    fd = creat("/tmp/.preinit", 0600);

    if (fd < 0)
        ERROR("Failed to create sentinel file
");
    else
        close(fd);

    preinit_proc.cb = spawn_procd;
    preinit_proc.pid = fork();
    if (!preinit_proc.pid) {
        execvp(init[0], init);
        ERROR("Failed to start preinit
");
        exit(-1);
    }
    if (preinit_proc.pid <= 0) {
        ERROR("Failed to start new preinit instance
");
        return;
    }
    uloop_process_add(&preinit_proc);

    DEBUG(4, "Launched preinit instance, pid=%d
", (int) preinit_proc.pid);
}

这里fork出2个子进程，执行procd和/etc/preinit，

• 首先看procd，因为带有参数“-h /etc/hotplug-preinit.json”，所以会执行hotplug_run函数。

int main(int argc, char **argv)
{
    int ch;
    char *dbglvl = getenv("DBGLVL");
    int ulog_channels = ULOG_KMSG;

    if (dbglvl) {
        debug = atoi(dbglvl);
        unsetenv("DBGLVL");
    }

    while ((ch = getopt(argc, argv, "d:s:h:S")) != -1) {
        switch (ch) {
        case 'h':
            return hotplug_run(optarg);
        case 's':
            ubus_socket = optarg;
            break;
        case 'd':
            debug = atoi(optarg);
            break;
        case 'S':
            ulog_channels = ULOG_STDIO;
            break;
        default:
            return usage(argv[0]);
        }
    }

    ulog_open(ulog_channels, LOG_DAEMON, "procd");

    setsid();
    uloop_init();
    procd_signal();
    if (getpid() != 1)
        procd_connect_ubus();
    else
        procd_state_next();
    uloop_run();
    uloop_done();

    return 0;
}

hotplug实现如下，这里是建立netlink通信机制，完成用户层和内核的交互，监听内核的uevent事件。

void hotplug(char *rules)
{
    struct sockaddr_nl nls;
    int nlbufsize = 512 * 1024;

    rule_file = strdup(rules);
    memset(&nls,0,sizeof(struct sockaddr_nl));
    nls.nl_family = AF_NETLINK;
    nls.nl_pid = getpid();
    nls.nl_groups = -1;

    if ((hotplug_fd.fd = socket(PF_NETLINK, SOCK_DGRAM | SOCK_CLOEXEC, NETLINK_KOBJECT_UEVENT)) == -1) {
        ERROR("Failed to open hotplug socket: %s
", strerror(errno));
        exit(1);
    }
    if (bind(hotplug_fd.fd, (void *)&nls, sizeof(struct sockaddr_nl))) {
        ERROR("Failed to bind hotplug socket: %s
", strerror(errno));
        exit(1);
    }

    if (setsockopt(hotplug_fd.fd, SOL_SOCKET, SO_RCVBUFFORCE, &nlbufsize, sizeof(nlbufsize)))
        ERROR("Failed to resize receive buffer: %s
", strerror(errno));

    json_script_init(&jctx);
    queue_proc.cb = queue_proc_cb;
    uloop_fd_add(&hotplug_fd, ULOOP_READ);
}

int hotplug_run(char *rules)
{
    uloop_init();
    hotplug(rules);
    uloop_run();

    return 0;
}

• /etc/preinit脚本大致内容如下，先调用另外的shell脚本，获取函数定义

. /lib/functions.sh
. /lib/functions/preinit.sh
. /lib/functions/system.sh

# 初始化hook链
boot_hook_init preinit_essential
boot_hook_init preinit_main
boot_hook_init failsafe
boot_hook_init initramfs

# 依次执行/lib/preinit目录中的脚本，将函数调用添加到hook链中
for pi_source_file in /lib/preinit/*; do
    . $pi_source_file
done

# 执行preinit_essential注册的hook链的所有函数
boot_run_hook preinit_essential
 
# 执行preinit_main注册的hook链的所有函数
boot_run_hook preinit_main

lib/functions/preinit.sh中定义

boot_hook_init() {
    local hook="${1}_hook"
    export -n "PI_STACK_LIST=${PI_STACK_LIST:+$PI_STACK_LIST }$hook"
    export -n "$hook="
}

/lib/preinit/10_sysinfo中添加hook函数

boot_hook_add preinit_main do_sysinfo_generic

/etc/preinit脚本执行完成后，调用spawn_procd

static void
spawn_procd(struct uloop_process *proc, int ret)
{
    char *wdt_fd = watchdog_fd();
    char *argv[] = { "/sbin/procd", NULL};
    struct stat s;
    char dbg[2];

    if (plugd_proc.pid > 0)
        kill(plugd_proc.pid, SIGKILL);

    if (!stat("/tmp/sysupgrade", &s))
        while (true)
            sleep(1);

    unsetenv("INITRAMFS");
    unsetenv("PREINIT");
    unlink("/tmp/.preinit");
    DEBUG(2, "Exec to real procd now
");
    if (wdt_fd)
        setenv("WDTFD", wdt_fd, 1);
    check_dbglvl();
    if (debug > 0) {
        snprintf(dbg, 2, "%d", debug);
        setenv("DBGLVL", dbg, 1);
    }

	//调用procd
    execvp(argv[0], argv);
}

此时getpid()等于1，所以调用procd_state_next，进入到状态机处理中。

对应的procd log如下，procd state不断迁移，包括STATE_EARLY，STATE_UBUS，STATE_INIT等。

[    3.161338@3] init: Console is alive		
[    3.173921@3] init: Ping		
[    3.184207@3] init: Ping
[    3.192558@1] kmodloader: loading kernel modules from /etc/modules-boot.d/*	
[    3.194447@3] init: Ping
[    3.196209@1] kmodloader: done loading kernel modules from /etc/modules-boot.d/*
[    3.204716@3] init: Ping
[    3.206180@3] init: - preinit -
[    3.208671@3] init: Launched preinit instance, pid=1308
[    3.302967@3] init: Exec to real procd now
[    3.308865@3] procd: - early -
[    3.524654@2] procd: Finished udevtrigger
[    4.024929@2] procd: Coldplug complete
[    4.028061@2] procd: - ubus -
[    4.029198@2] procd: Create service ubus
[    4.030829@2] procd: Create instance ubus::instance1
[    4.032109@2] procd: Started instance ubus::instance1[1554]
[    4.098895@2] procd: Connected to ubus, id=459ede6c
[    4.099092@2] procd: - init -
[    4.102474@2] procd: Launched new askconsole action, pid=1555
[    4.104142@2] procd: Launched new askfirst action, pid=1556

以STATE_INIT为例，执行procd_inittab_run("xxx")会调用对应handlers的callback，对应所有的init_action是在procd_inittab()中添加的。

    case STATE_INIT:
        LOG("- init -
");
        procd_inittab();
        procd_inittab_run("respawn");
        procd_inittab_run("askconsole");
        procd_inittab_run("askfirst");
        procd_inittab_run("sysinit");

static struct init_handler handlers[] = {
    {
        .name = "sysinit",
        .cb = runrc,
    }, {
        .name = "shutdown",
        .cb = runrc,
    }, {
        .name = "askfirst",
        .cb = askfirst,
        .multi = 1,
    }, {
        .name = "askconsole",
        .cb = askconsole,
        .multi = 1,
    }, {
        .name = "respawn",
        .cb = rcrespawn,
        .multi = 1,
    }
};

void procd_inittab_run(const char *handler)
{
    struct init_action *a;

    list_for_each_entry(a, &actions, list) {
        if (!strcmp(a->handler->name, handler)) {
            if (a->handler->multi) {
                a->handler->cb(a);
                continue;
            }
            a->handler->cb(a);
            break;
        }
    }
}

这里来看runrc的实现，代码位于inittab.c

static void runrc(struct init_action *a)
{
    if (!a->argv[1] || !a->argv[2]) {
        ERROR("valid format is rcS <S|K> <param>
");
        return;
    }

    /* proceed even if no init or shutdown scripts run */
    if (rcS(a->argv[1], a->argv[2], rcdone)) {
        printf("---rcdone---
");
        rcdone(NULL);
    } else {
        printf("----rcdone error
");
    }
}

rcS.c

int rcS(char *pattern, char *param, void (*q_empty)(struct runqueue *))
{
    runqueue_init(&q);
    q.empty_cb = q_empty;
    q.max_running_tasks = 1;

    return _rc(&q, "/etc/rc.d", pattern, "*", param);
}

执行/etc/rc.d目录下S/K开头的脚本