从内核的角度看进程以及多线程
前言：就如我之前所说的一样，程序只是一个被编译器（包括汇编器以及连接器）将你的抽象代码转换为计算机能理解的有一定格式的二进制文件，它有一定格式（ELF，PE之类），就如之前的我所说的例子，就好比程序就如一段铁轨，你是铁路设计师，编译器负责把铁轨做好，那么之后很多复杂的事都被内核承包了，也就是铺铁轨以及让火车（CPU）在上面跑起来。接下来我描述的问题会由浅入深，来纠正很多人对于多线程的疑惑，因为我发现很多我认为常识性的问题实际上很多人不懂装懂，包括很多所谓面试官，当他们问我多线程或者与操作系统有关的东西的时候我内心的崩溃的，于是我想着有必要写这个东西，我之前hack的内核源码我能把这东西说得比较透彻，且会结合测试程序来分析其原理。（本文涉及的内核源码为4.9版本）
1.进程（以linux下ELF格式为例）
这张图看似简单其实是重要的一张图，它涉及到一个程序的映射（mm_struct 布局），举个很简单的例子，用的比较多的malloc函数，你第一次使用它返回的地址就是指向strart_brk也就是堆的开始这个地方（如果你没有开ALSR堆栈随机化保护措施那么这个地址是固定的），关于其它段这些是基础知识我之前的博客有写就不过多描述，接下来就是内核对一个进程的定义以及描述（task_struct）：（linux-4.9/include/linux/Sched.h）
struct task_struct {
#ifdef CONFIG_THREAD_INFO_IN_TASK
/*
* For reasons of header soup (see current_thread_info()), this
* must be the first element of task_struct.
*/
struct thread_info thread_info;
#endif
volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */
void *stack;
atomic_t usage;
unsigned int flags; /* per process flags, defined below */
unsigned int ptrace;
#ifdef CONFIG_SMP
struct llist_node wake_entry;
int on_cpu;
#ifdef CONFIG_THREAD_INFO_IN_TASK
unsigned int cpu; /* current CPU */
#endif
unsigned int wakee_flips;
unsigned long wakee_flip_decay_ts;
struct task_struct *last_wakee;
int wake_cpu;
#endif
int on_rq;
int prio, static_prio, normal_prio;
unsigned int rt_priority;
const struct sched_class *sched_class;
struct sched_entity se;
struct sched_rt_entity rt;
#ifdef CONFIG_CGROUP_SCHED
struct task_group *sched_task_group;
#endif
struct sched_dl_entity dl;
#ifdef CONFIG_PREEMPT_NOTIFIERS
/* list of struct preempt_notifier: */
struct hlist_head preempt_notifiers;
#endif
#ifdef CONFIG_BLK_DEV_IO_TRACE
unsigned int btrace_seq;
#endif
unsigned int policy;
int nr_cpus_allowed;
cpumask_t cpus_allowed;
#ifdef CONFIG_PREEMPT_RCU
int rcu_read_lock_nesting;
union rcu_special rcu_read_unlock_special;
struct list_head rcu_node_entry;
struct rcu_node *rcu_blocked_node;
#endif /* #ifdef CONFIG_PREEMPT_RCU */
#ifdef CONFIG_TASKS_RCU
unsigned long rcu_tasks_nvcsw;
bool rcu_tasks_holdout;
struct list_head rcu_tasks_holdout_list;
int rcu_tasks_idle_cpu;
#endif /* #ifdef CONFIG_TASKS_RCU */
#ifdef CONFIG_SCHED_INFO
struct sched_info sched_info;
#endif
struct list_head tasks;
#ifdef CONFIG_SMP
struct plist_node pushable_tasks;
struct rb_node pushable_dl_tasks;
#endif
struct mm_struct *mm, *active_mm;
/* per-thread vma caching */
u32 vmacache_seqnum;
struct vm_area_struct *vmacache[VMACACHE_SIZE];
#if defined(SPLIT_RSS_COUNTING)
struct task_rss_stat rss_stat;
#endif
/* task state */
int exit_state;
int exit_code, exit_signal;
int pdeath_signal; /* The signal sent when the parent dies */
unsigned long jobctl; /* JOBCTL_*, siglock protected */
/* Used for emulating ABI behavior of previous Linux versions */
unsigned int personality;
/* scheduler bits, serialized by scheduler locks */
unsigned sched_reset_on_fork:1;
unsigned sched_contributes_to_load:1;
unsigned sched_migrated:1;
unsigned sched_remote_wakeup:1;
unsigned :0; /* force alignment to the next boundary */
/* unserialized, strictly 'current' */
unsigned in_execve:1; /* bit to tell LSMs we're in execve */
unsigned in_iowait:1;
#if !defined(TIF_RESTORE_SIGMASK)
unsigned restore_sigmask:1;
#endif
#ifdef CONFIG_MEMCG
unsigned memcg_may_oom:1;
#ifndef CONFIG_SLOB
unsigned memcg_kmem_skip_account:1;
#endif
#endif
#ifdef CONFIG_COMPAT_BRK
unsigned brk_randomized:1;
#endif
unsigned long atomic_flags; /* Flags needing atomic access. */
struct restart_block restart_block;
pid_t pid;
pid_t tgid;
#ifdef CONFIG_CC_STACKPROTECTOR
/* Canary value for the -fstack-protector gcc feature */
unsigned long stack_canary;
#endif
/*
* pointers to (original) parent process, youngest child, younger sibling,
* older sibling, respectively. (p->father can be replaced with
* p->real_parent->pid)
*/
struct task_struct __rcu *real_parent; /* real parent process */
struct task_struct __rcu *parent; /* recipient of SIGCHLD, wait4() reports */
/*
* children/sibling forms the list of my natural children
*/
struct list_head children; /* list of my children */
struct list_head sibling; /* linkage in my parent's children list */
struct task_struct *group_leader; /* threadgroup leader */
/*
* ptraced is the list of tasks this task is using ptrace on.
* This includes both natural children and PTRACE_ATTACH targets.
* p->ptrace_entry is p's link on the p->parent->ptraced list.
*/
struct list_head ptraced;
struct list_head ptrace_entry;
/* PID/PID hash table linkage. */
struct pid_link pids[PIDTYPE_MAX];
struct list_head thread_group;
struct list_head thread_node;
struct completion *vfork_done; /* for vfork() */
int __user *set_child_tid; /* CLONE_CHILD_SETTID */
int __user *clear_child_tid; /* CLONE_CHILD_CLEARTID */
cputime_t utime, stime, utimescaled, stimescaled;
cputime_t gtime;
struct prev_cputime prev_cputime;
#ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
seqcount_t vtime_seqcount;
unsigned long long vtime_snap;
enum {
/* Task is sleeping or running in a CPU with VTIME inactive */
VTIME_INACTIVE = 0,
/* Task runs in userspace in a CPU with VTIME active */
VTIME_USER,
/* Task runs in kernelspace in a CPU with VTIME active */
VTIME_SYS,
} vtime_snap_whence;
#endif
#ifdef CONFIG_NO_HZ_FULL
atomic_t tick_dep_mask;
#endif
unsigned long nvcsw, nivcsw; /* context switch counts */
u64 start_time; /* monotonic time in nsec */
u64 real_start_time; /* boot based time in nsec */
/* mm fault and swap info: this can arguably be seen as either mm-specific or thread-specific */
unsigned long min_flt, maj_flt;
struct task_cputime cputime_expires;
struct list_head cpu_timers[3];
/* process credentials */
const struct cred __rcu *real_cred; /* objective and real subjective task
* credentials (COW) */
const struct cred __rcu *cred; /* effective (overridable) subjective task
* credentials (COW) */
char comm[TASK_COMM_LEN]; /* executable name excluding path
- access with [gs]et_task_comm (which lock
it with task_lock())
- initialized normally by setup_new_exec */
/* file system info */
struct nameidata *nameidata;
#ifdef CONFIG_SYSVIPC
/* ipc stuff */
struct sysv_sem sysvsem;
struct sysv_shm sysvshm;
#endif
#ifdef CONFIG_DETECT_HUNG_TASK
/* hung task detection */
unsigned long last_switch_count;
#endif
/* filesystem information */
struct fs_struct *fs;
/* open file information */
struct files_struct *files;
/* namespaces */
struct nsproxy *nsproxy;
/* signal handlers */
struct signal_struct *signal;
struct sighand_struct *sighand;
sigset_t blocked, real_blocked;
sigset_t saved_sigmask; /* restored if set_restore_sigmask() was used */
struct sigpending pending;
unsigned long sas_ss_sp;
size_t sas_ss_size;
unsigned sas_ss_flags;
struct callback_head *task_works;
struct audit_context *audit_context;
#ifdef CONFIG_AUDITSYSCALL
kuid_t loginuid;
unsigned int sessionid;
#endif
struct seccomp seccomp;
/* Thread group tracking */
u32 parent_exec_id;
u32 self_exec_id;
/* Protection of (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed,
* mempolicy */
spinlock_t alloc_lock;
/* Protection of the PI data structures: */
raw_spinlock_t pi_lock;
struct wake_q_node wake_q;
#ifdef CONFIG_RT_MUTEXES
/* PI waiters blocked on a rt_mutex held by this task */
struct rb_root pi_waiters;
struct rb_node *pi_waiters_leftmost;
/* Deadlock detection and priority inheritance handling */
struct rt_mutex_waiter *pi_blocked_on;
#endif
#ifdef CONFIG_DEBUG_MUTEXES
/* mutex deadlock detection */
struct mutex_waiter *blocked_on;
#endif
#ifdef CONFIG_TRACE_IRQFLAGS
unsigned int irq_events;
unsigned long hardirq_enable_ip;
unsigned long hardirq_disable_ip;
unsigned int hardirq_enable_event;
unsigned int hardirq_disable_event;
int hardirqs_enabled;
int hardirq_context;
unsigned long softirq_disable_ip;
unsigned long softirq_enable_ip;
unsigned int softirq_disable_event;
unsigned int softirq_enable_event;
int softirqs_enabled;
int softirq_context;
#endif
#ifdef CONFIG_LOCKDEP
# define MAX_LOCK_DEPTH 48UL
u64 curr_chain_key;
int lockdep_depth;
unsigned int lockdep_recursion;
struct held_lock held_locks[MAX_LOCK_DEPTH];
gfp_t lockdep_reclaim_gfp;
#endif
#ifdef CONFIG_UBSAN
unsigned int in_ubsan;
#endif
/* journalling filesystem info */
void *journal_info;
/* stacked block device info */
struct bio_list *bio_list;
#ifdef CONFIG_BLOCK
/* stack plugging */
struct blk_plug *plug;
#endif
/* VM state */
struct reclaim_state *reclaim_state;
struct backing_dev_info *backing_dev_info;
struct io_context *io_context;
unsigned long ptrace_message;
siginfo_t *last_siginfo; /* For ptrace use. */
struct task_io_accounting ioac;
#if defined(CONFIG_TASK_XACCT)
u64 acct_rss_mem1; /* accumulated rss usage */
u64 acct_vm_mem1; /* accumulated virtual memory usage */
cputime_t acct_timexpd; /* stime + utime since last update */
#endif
#ifdef CONFIG_CPUSETS
nodemask_t mems_allowed; /* Protected by alloc_lock */
seqcount_t mems_allowed_seq; /* Seqence no to catch updates */
int cpuset_mem_spread_rotor;
int cpuset_slab_spread_rotor;
#endif
#ifdef CONFIG_CGROUPS
/* Control Group info protected by css_set_lock */
struct css_set __rcu *cgroups;
/* cg_list protected by css_set_lock and tsk->alloc_lock */
struct list_head cg_list;
#endif
#ifdef CONFIG_FUTEX
struct robust_list_head __user *robust_list;
#ifdef CONFIG_COMPAT
struct compat_robust_list_head __user *compat_robust_list;
#endif
struct list_head pi_state_list;
struct futex_pi_state *pi_state_cache;
#endif
#ifdef CONFIG_PERF_EVENTS
struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts];
struct mutex perf_event_mutex;
struct list_head perf_event_list;
#endif
#ifdef CONFIG_DEBUG_PREEMPT
unsigned long preempt_disable_ip;
#endif
#ifdef CONFIG_NUMA
struct mempolicy *mempolicy; /* Protected by alloc_lock */
short il_next;
short pref_node_fork;
#endif
#ifdef CONFIG_NUMA_BALANCING
int numa_scan_seq;
unsigned int numa_scan_period;
unsigned int numa_scan_period_max;
int numa_preferred_nid;
unsigned long numa_migrate_retry;
u64 node_stamp; /* migration stamp */
u64 last_task_numa_placement;
u64 last_sum_exec_runtime;
struct callback_head numa_work;
struct list_head numa_entry;
struct numa_group *numa_group;
/*
* numa_faults is an array split into four regions:
* faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer
* in this precise order.
*
* faults_memory: Exponential decaying average of faults on a per-node
* basis. Scheduling placement decisions are made based on these
* counts. The values remain static for the duration of a PTE scan.
* faults_cpu: Track the nodes the process was running on when a NUMA
* hinting fault was incurred.
* faults_memory_buffer and faults_cpu_buffer: Record faults per node
* during the current scan window. When the scan completes, the counts
* in faults_memory and faults_cpu decay and these values are copied.
*/
unsigned long *numa_faults;
unsigned long total_numa_faults;
/*
* numa_faults_locality tracks if faults recorded during the last
* scan window were remote/local or failed to migrate. The task scan
* period is adapted based on the locality of the faults with different
* weights depending on whether they were shared or private faults
*/
unsigned long numa_faults_locality[3];
unsigned long numa_pages_migrated;
#endif /* CONFIG_NUMA_BALANCING */
#ifdef CONFIG_ARCH_WANT_BATCHED_UNMAP_TLB_FLUSH
struct tlbflush_unmap_batch tlb_ubc;
#endif
struct rcu_head rcu;
/*
* cache last used pipe for splice
*/
struct pipe_inode_info *splice_pipe;
struct page_frag task_frag;
#ifdef CONFIG_TASK_DELAY_ACCT
struct task_delay_info *delays;
#endif
#ifdef CONFIG_FAULT_INJECTION
int make_it_fail;
#endif
/*
* when (nr_dirtied >= nr_dirtied_pause), it's time to call
* balance_dirty_pages() for some dirty throttling pause
*/
int nr_dirtied;
int nr_dirtied_pause;
unsigned long dirty_paused_when; /* start of a write-and-pause period */
#ifdef CONFIG_LATENCYTOP
int latency_record_count;
struct latency_record latency_record[LT_SAVECOUNT];
#endif
/*
* time slack values; these are used to round up poll() and
* select() etc timeout values. These are in nanoseconds.
*/
u64 timer_slack_ns;
u64 default_timer_slack_ns;
#ifdef CONFIG_KASAN
unsigned int kasan_depth;
#endif
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* Index of current stored address in ret_stack */
int curr_ret_stack;
/* Stack of return addresses for return function tracing */
struct ftrace_ret_stack *ret_stack;
/* time stamp for last schedule */
unsigned long long ftrace_timestamp;
/*
* Number of functions that haven't been traced
* because of depth overrun.
*/
atomic_t trace_overrun;
/* Pause for the tracing */
atomic_t tracing_graph_pause;
#endif
#ifdef CONFIG_TRACING
/* state flags for use by tracers */
unsigned long trace;
/* bitmask and counter of trace recursion */
unsigned long trace_recursion;
#endif /* CONFIG_TRACING */
#ifdef CONFIG_KCOV
/* Coverage collection mode enabled for this task (0 if disabled). */
enum kcov_mode kcov_mode;
/* Size of the kcov_area. */
unsigned kcov_size;
/* Buffer for coverage collection. */
void *kcov_area;
/* kcov desciptor wired with this task or NULL. */
struct kcov *kcov;
#endif
#ifdef CONFIG_MEMCG
struct mem_cgroup *memcg_in_oom;
gfp_t memcg_oom_gfp_mask;
int memcg_oom_order;
/* number of pages to reclaim on returning to userland */
unsigned int memcg_nr_pages_over_high;
#endif
#ifdef CONFIG_UPROBES
struct uprobe_task *utask;
#endif
#if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE)
unsigned int sequential_io;
unsigned int sequential_io_avg;
#endif
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
unsigned long task_state_change;
#endif
int pagefault_disabled;
#ifdef CONFIG_MMU
struct task_struct *oom_reaper_list;
#endif
#ifdef CONFIG_VMAP_STACK
struct vm_struct *stack_vm_area;
#endif
#ifdef CONFIG_THREAD_INFO_IN_TASK
/* A live task holds one reference. */
atomic_t stack_refcount;
#endif
/* CPU-specific state of this task */
struct thread_struct thread;
/*
* WARNING: on x86, 'thread_struct' contains a variable-sized
* structure. It *MUST* be at the end of 'task_struct'.
*
* Do not put anything below here!
*/
1.1与thread_info相关的东西：
我来解释一下整个进程描述符，首先第一个thread_info,这个字段存放一个内核态堆栈中一个名为thread_info的结构体的头部的地址，这个结构体十分重要，是内核用来进行软进程切换进程上下文存放的地方，也就是在进程切换时，其关键信息存放的地方，其实内核只存放了一部分进程上下文在堆栈上，还有一部分是在TSS段中（TSS段中主要是存各类寄存器的硬件上下文），我认为我有必要再深入一下，请看内核CPU中关于这个TSS段的定义的定义：
static inline void load_cr3(pgd_t *pgdir)
{
write_cr3(__pa(pgdir));
}
#ifdef CONFIG_X86_32
/* This is the TSS defined by the hardware. */
struct x86_hw_tss {
unsigned short back_link, __blh;
unsigned long sp0;
unsigned short ss0, __ss0h;
unsigned long sp1;
/*
* We don't use ring 1, so ss1 is a convenient scratch space in
* the same cacheline as sp0. We use ss1 to cache the value in
* MSR_IA32_SYSENTER_CS. When we context switch
* MSR_IA32_SYSENTER_CS, we first check if the new value being
* written matches ss1, and, if it's not, then we wrmsr the new
* value and update ss1.
*
* The only reason we context switch MSR_IA32_SYSENTER_CS is
* that we set it to zero in vm86 tasks to avoid corrupting the
* stack if we were to go through the sysenter path from vm86
* mode.
*/
unsigned short ss1; /* MSR_IA32_SYSENTER_CS */
unsigned short __ss1h;
unsigned long sp2;
unsigned short ss2, __ss2h;
unsigned long __cr3;
unsigned long ip;
unsigned long flags;
unsigned long ax;
unsigned long cx;
unsigned long dx;
unsigned long bx;
unsigned long sp;
unsigned long bp;
unsigned long si;
unsigned long di;
unsigned short es, __esh;
unsigned short cs, __csh;
unsigned short ss, __ssh;
unsigned short ds, __dsh;
unsigned short fs, __fsh;
unsigned short gs, __gsh;
unsigned short ldt, __ldth;
unsigned short trace;
unsigned short io_bitmap_base;
} __attribute__((packed));
#else
struct x86_hw_tss {
u32 reserved1;
u64 sp0;
u64 sp1;
u64 sp2;
u64 reserved2;
u64 ist[7];
u32 reserved3;
u32 reserved4;
u16 reserved5;
u16 io_bitmap_base;
} __attribute__((packed)) ____cacheline_aligned;
#endif
需要注意的是TSS段是每个cpu只有一个，但是一个cpu可能要维持多个进程信息故在32位机器下tss段还是一些通用寄存器的临时寄存处，但是在64位机器也就是多核处理器开始逐渐流行的机器上，内核用thread字段暂存进程硬件上下文，tss段作一些io位图暂存以及进程从用户态到内核态这个过程中内核态堆栈的寻址功能，thread字段如下：
struct thread_struct {
/* Cached TLS descriptors: */
struct desc_struct tls_array[GDT_ENTRY_TLS_ENTRIES];
unsigned long sp0;
unsigned long sp;
#ifdef CONFIG_X86_32
unsigned long sysenter_cs;
#else
unsigned short es;
unsigned short ds;
unsigned short fsindex;
unsigned short gsindex;
#endif
u32 status; /* thread synchronous flags */
#ifdef CONFIG_X86_64
unsigned long fsbase;
unsigned long gsbase;
#else
/*
* XXX: this could presumably be unsigned short. Alternatively,
* 32-bit kernels could be taught to use fsindex instead.
*/
unsigned long fs;
unsigned long gs;
#endif
/* Save middle states of ptrace breakpoints */
struct perf_event *ptrace_bps[HBP_NUM];
/* Debug status used for traps, single steps, etc... */
unsigned long debugreg6;
/* Keep track of the exact dr7 value set by the user */
unsigned long ptrace_dr7;
/* Fault info: */
unsigned long cr2;
unsigned long trap_nr;
unsigned long error_code;
#ifdef CONFIG_VM86
/* Virtual 86 mode info */
struct vm86 *vm86;
#endif
/* IO permissions: */
unsigned long *io_bitmap_ptr;
unsigned long iopl;
/* Max allowed port in the bitmap, in bytes: */
unsigned io_bitmap_max;
mm_segment_t addr_limit;
unsigned int sig_on_uaccess_err:1;
unsigned int uaccess_err:1; /* uaccess failed */
/* Floating point and extended processor state */
struct fpu fpu;
/*
* WARNING: 'fpu' is dynamically-sized. It *MUST* be at
* the end.
*/
};然而eax那类通用寄存器的硬件上下文是保存在内核堆栈上的，可以通过thread_info来找到，thread_info放在内核堆栈底部。而其中cr3控制寄存器指向的是当前cpu控制进程的页目录指针表（PDPT 64位机器必有的一个东西，要不然寻址能力突破不了4GB也就是所谓内存空间）或者页全局目录表也就是PDGT（这个是32位机器的分页寻址第一级分页），cr3在startup阶段被设置代表着内核开启了分页，每当进程被__switch_to调用切换时，cr3的值就会改变指向当前cpu活跃进程的页目录基地址，tss段是全局段描述符GDT中的一项，这个结构体每一项都特别重要：
在linux-4.9/arch/x86/boot/Pm.c中有对其初始化的代码，凡是关于内核初始化的代码都是linus写的:p,当时他才20几岁，其编程水平可想而知：
struct gdt_ptr {
u16 len;
u32 ptr;
} __attribute__((packed));
static void setup_gdt(void)
{
/* There are machines which are known to not boot with the GDT
being 8-byte unaligned. Intel recommends 16 byte alignment. */
static const u64 boot_gdt[] __attribute__((aligned(16))) = {
/* CS: code, read/execute, 4 GB, base 0 */
[GDT_ENTRY_BOOT_CS] = GDT_ENTRY(0xc09b, 0, 0xfffff),
/* DS: data, read/write, 4 GB, base 0 */
[GDT_ENTRY_BOOT_DS] = GDT_ENTRY(0xc093, 0, 0xfffff),
/* TSS: 32-bit tss, 104 bytes, base 4096 */
/* We only have a TSS here to keep Intel VT happy;
we don't actually use it for anything. */
[GDT_ENTRY_BOOT_TSS] = GDT_ENTRY(0x0089, 4096, 103),
};
/* Xen HVM incorrectly stores a pointer to the gdt_ptr, instead
of the gdt_ptr contents. Thus, make it static so it will
stay in memory, at least long enough that we switch to the
proper kernel GDT. */
static struct gdt_ptr gdt;
gdt.len = sizeof(boot_gdt)-1;
gdt.ptr = (u32)&boot_gdt + (ds() << 4);
asm volatile("lgdtl %0" : : "m" (gdt));
}
这段代码很有意思，intel VT自行搜索一下就知道，就如我所说，位于GDT中的tss是每个cpu都有一个，用于硬件上下文（寄存器）存储，而thread_info也是如此，内核态堆栈占用2个页框，在4KB页框机制下也就是8KB大小，thread_info也有进程上下文：
struct thread_info {
unsigned long flags; /* low level flags */
mm_segment_t addr_limit; /* address limit */
struct task_struct *task; /* main task structure */
int preempt_count; /* 0 => preemptable, <0 => bug */
int cpu; /* cpu */
};
struct thread_info {
unsigned long flags; /* low level flags */
int preempt_count; /* 0 => preemptable, <0 => bug */
mm_segment_t addr_limit; /* address limit */
struct task_struct *task; /* main task structure */
__u32 cpu; /* cpu */
__u32 cpu_domain; /* cpu domain */
struct cpu_context_save cpu_context; /* cpu context */
__u32 syscall; /* syscall number */
__u8 used_cp[16]; /* thread used copro */
unsigned long tp_value[2]; /* TLS registers */
#ifdef CONFIG_CRUNCH
struct crunch_state crunchstate;
#endif
union fp_state fpstate __attribute__((aligned(8)));
union vfp_state vfpstate;
#ifdef CONFIG_ARM_THUMBEE
unsigned long thumbee_state; /* ThumbEE Handler Base register */
#endif
这个是两个架构的thread_info定义，可见其包含的进程上下文有此时内核态堆栈指向的进程描述符，以及CPU组，cpu标号，以及抢占等级等等，接下来我们可以看看进程切换的关键函数__swich_to（linux-4.9/arch/x86/kernel/Process_32.c）:
__switch_to(struct task_struct *prev_p, struct task_struct *next_p)
{
struct thread_struct *prev = &prev_p->thread,
*next = &next_p->thread;
struct fpu *prev_fpu = &prev->fpu;
struct fpu *next_fpu = &next->fpu;
int cpu = smp_processor_id();
struct tss_struct *tss = &per_cpu(cpu_tss, cpu);
fpu_switch_t fpu_switch;
/* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
fpu_switch = switch_fpu_prepare(prev_fpu, next_fpu, cpu);
/*
* Save away %gs. No need to save %fs, as it was saved on the
* stack on entry. No need to save %es and %ds, as those are
* always kernel segments while inside the kernel. Doing this
* before setting the new TLS descriptors avoids the situation
* where we temporarily have non-reloadable segments in %fs
* and %gs. This could be an issue if the NMI handler ever
* used %fs or %gs (it does not today), or if the kernel is
* running inside of a hypervisor layer.
*/
lazy_save_gs(prev->gs);
/*
* Load the per-thread Thread-Local Storage descriptor.
*/
load_TLS(next, cpu);
/*
* Restore IOPL if needed. In normal use, the flags restore
* in the switch assembly will handle this. But if the kernel
* is running virtualized at a non-zero CPL, the popf will
* not restore flags, so it must be done in a separate step.
*/
if (get_kernel_rpl() && unlikely(prev->iopl != next->iopl))
set_iopl_mask(next->iopl);
/*
* Now maybe handle debug registers and/or IO bitmaps
*/
if (unlikely(task_thread_info(prev_p)->flags & _TIF_WORK_CTXSW_PREV ||
task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT))
__switch_to_xtra(prev_p, next_p, tss);
/*
* Leave lazy mode, flushing any hypercalls made here.
* This must be done before restoring TLS segments so
* the GDT and LDT are properly updated, and must be
* done before fpu__restore(), so the TS bit is up
* to date.
*/
arch_end_context_switch(next_p);
/*
* Reload esp0 and cpu_current_top_of_stack. This changes
* current_thread_info().
*/
load_sp0(tss, next);
this_cpu_write(cpu_current_top_of_stack,
(unsigned long)task_stack_page(next_p) +
THREAD_SIZE);
/*
* Restore %gs if needed (which is common)
*/
if (prev->gs | next->gs)
lazy_load_gs(next->gs);
switch_fpu_finish(next_fpu, fpu_switch);
this_cpu_write(current_task, next_p);
return prev_p;
}
1.2mm_strcut
这个是进程内存地址映射空间，它描述了整个当前进程的虚拟地址空间，就是这个东西描述了图一的整个地址空间：
struct mm_struct {
struct vm_area_struct *mmap; /* list of VMAs */
struct rb_root mm_rb;
u32 vmacache_seqnum; /* per-thread vmacache */
#ifdef CONFIG_MMU
unsigned long (*get_unmapped_area) (struct file *filp,
unsigned long addr, unsigned long len,
unsigned long pgoff, unsigned long flags);
#endif
unsigned long mmap_base; /* base of mmap area */
unsigned long mmap_legacy_base; /* base of mmap area in bottom-up allocations */
unsigned long task_size; /* size of task vm space */
unsigned long highest_vm_end; /* highest vma end address */
pgd_t * pgd; //页目录起始地址
atomic_t mm_users; /* How many users with user space? */
atomic_t mm_count; /* How many references to "struct mm_struct" (users count as 1) */
atomic_long_t nr_ptes; /* PTE page table pages */
#if CONFIG_PGTABLE_LEVELS > 2
atomic_long_t nr_pmds; /* PMD page table pages */
#endif
int map_count; /* number of VMAs */
spinlock_t page_table_lock; /* Protects page tables and some counters */
struct rw_semaphore mmap_sem;
struct list_head mmlist; /* List of maybe swapped mm's. These are globally strung
* together off init_mm.mmlist, and are protected
* by mmlist_lock
*/
unsigned long hiwater_rss; /* High-watermark of RSS usage */
unsigned long hiwater_vm; /* High-water virtual memory usage */
unsigned long total_vm; /* Total pages mapped */
unsigned long locked_vm; /* Pages that have PG_mlocked set */
unsigned long pinned_vm; /* Refcount permanently increased */
unsigned long data_vm; /* VM_WRITE & ~VM_SHARED & ~VM_STACK */
unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE & ~VM_STACK */
unsigned long stack_vm; /* VM_STACK */
unsigned long def_flags;
unsigned long start_code, end_code, start_data, end_data;/*这些就是之前图一的各种段的初始化，包括BSS,DATA,BRK,STACK*/
unsigned long start_brk, brk, start_stack;
unsigned long arg_start, arg_end, env_start, env_end;
unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */
/*
* Special counters, in some configurations protected by the
* page_table_lock, in other configurations by being atomic.
*/
struct mm_rss_stat rss_stat;
struct linux_binfmt *binfmt;
cpumask_var_t cpu_vm_mask_var;
/* Architecture-specific MM context */
mm_context_t context;
unsigned long flags; /* Must use atomic bitops to access the bits */
struct core_state *core_state; /* coredumping support */
#ifdef CONFIG_AIO
spinlock_t ioctx_lock;
struct kioctx_table __rcu *ioctx_table;
#endif
#ifdef CONFIG_MEMCG
/*
* "owner" points to a task that is regarded as the canonical
* user/owner of this mm. All of the following must be true in
* order for it to be changed:
*
* current == mm->owner
* current->mm != mm
* new_owner->mm == mm
* new_owner->alloc_lock is held
*/
struct task_struct __rcu *owner;
#endif
/* store ref to file /proc//exe symlink points to */
struct file __rcu *exe_file;
#ifdef CONFIG_MMU_NOTIFIER
struct mmu_notifier_mm *mmu_notifier_mm;
#endif
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
pgtable_t pmd_huge_pte; /* protected by page_table_lock */
#endif
#ifdef CONFIG_CPUMASK_OFFSTACK
struct cpumask cpumask_allocation;
#endif
#ifdef CONFIG_NUMA_BALANCING
/*
* numa_next_scan is the next time that the PTEs will be marked
* pte_numa. NUMA hinting faults will gather statistics and migrate
* pages to new nodes if necessary.
*/
unsigned long numa_next_scan;
/* Restart point for scanning and setting pte_numa */
unsigned long numa_scan_offset;
/* numa_scan_seq prevents two threads setting pte_numa */
int numa_scan_seq;
#endif
#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION)
/*
* An operation with batched TLB flushing is going on. Anything that
* can move process memory needs to flush the TLB when moving a
* PROT_NONE or PROT_NUMA mapped page.
*/
bool tlb_flush_pending;
#endif
struct uprobes_state uprobes_state;
#ifdef CONFIG_X86_INTEL_MPX
/* address of the bounds directory */
void __user *bd_addr;
#endif
#ifdef CONFIG_HUGETLB_PAGE
atomic_long_t hugetlb_usage;
#endif
struct work_struct async_put_work;
}; 因为你的程序被内核映射到多个页上，这些页被内核设置成不连续的vm_area_struct，内核用红黑树管理这些不连续的vm_area_strcut,限于篇幅这部分暂不深入， 这样做的结果导致了一些事情，那就是你的进程的data段和text段等等是不连续的，内核通过双链表以及红黑树管理这些不连续虚拟地址片段，但是整个进程的地址空间被视为一个整个也就是一个mm描述符，故mm_struct有当前进程内存空间的页目录基址，而且还会对引用这个进程内存整体的用户进行计数。
1.3run_list以及state
这两个字段主要是描述进程状态的，以及特权等级，run_list其实是由多个描述不同权限的双链表构成它将不同优先权的运行进程链接在一起，state则描述了进程的状态，这两个字段深入下去可以写很多字，但与本文内容关系不太故不作介绍，感兴趣可以自己hack。
1.4real_parent与parent
这两个字段代表真正父进程以及当前父进程，一般情况下是一样的，但是用ptrace之类的调试手法对进程进行跟踪时，为了跟踪进程能遍历整个被跟踪进程的进程关系树（也就是父子以及兄弟之类的），这时的parent就指向跟踪进程。
1.5fs_struct
指向内核的文件系统中该进程映射的程序的节点，关于文件系统我之前写过博客。
1.6files_struct
这个是维护当前进程打开文件的文件描述符的，默认0是标准输入，1输出，2错误输出，而后的就是打开文件的文件描述符，一般打开的一个文件是3。
2.多线程
其实多线程在linux下只是一个进程的裁剪体，当前进程的线程的mm基地址是存放在GDT中的TLS字段的，在task_struct的thread字段中有个
struct desc_struct tls_array[GDT_ENTRY_TLS_ENTRIES];#include
#include
#include
#include
static pthread_mutex_t mtx = PTHREAD_MUTEX_INITIALIZER;
static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
static void pthread_func_1 (void);
static void pthread_func_2 (void);
int main (int argc, char** argv)
{
pthread_t pt_1 = 0;
pthread_t pt_2 = 0;
pthread_attr_t attr = {0};
int ret = 0;
pthread_attr_init (&attr);
pthread_attr_setscope (&attr, PTHREAD_SCOPE_SYSTEM);
pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_DETACHED);
ret = pthread_create (&pt_1, NULL, (void *)pthread_func_1, NULL);
if (ret != 0)
{
perror ("pthread_1_create");
}
ret = pthread_create (&pt_2, NULL,(void *) pthread_func_2, NULL);
if (ret != 0)
{
perror ("pthread_2_create");
}
pthread_join (pt_1, NULL);
pthread_join (pt_2, NULL);
return 0;
}
static void pthread_func_1 (void)
{
while (1)
{
int i = 0;
int pid= getpid();
for (; i < 6; i++)
{
printf ("This is pthread_1,the tpid is %d\n ",pid);
if (i==2)
{
pthread_cond_signal(&cond);
return 0;
}
}
}
}
static void pthread_func_2 (void)
{
while(1)
{
int rc;
int pid= getpid();
pthread_mutex_lock(&mtx);
rc = pthread_cond_wait(&cond, &mtx);
if(rc ==0)
{
int i = 0;
for (; i < 3 ; i ++)
{
printf ("This is pthread_2.the tpid is %d\n",pid);
}
pthread_mutex_unlock(&mtx);
}
}
}
为啥要添加getuid()，因为我只想更正一个错误观念，getuid()的返回值是tpid也就是当前线程所在线程组的pid而不是所谓pid，线程是有独立pid的，但是一个进程里面的线程的tpid是相同的，来我们跟踪一下：
execve("./test3", ["./test3"], [/* 42 vars */]) = 0
brk(NULL) = 0x1615000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f98d8606000
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=130450, ...}) = 0
mmap(NULL, 130450, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f98d85e6000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libpthread.so.0", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20o\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=137440, ...}) = 0
mmap(NULL, 2213008, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f98d81cb000
mprotect(0x7f98d81e3000, 2093056, PROT_NONE) = 0
mmap(0x7f98d83e2000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x17000) = 0x7f98d83e2000
mmap(0x7f98d83e4000, 13456, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f98d83e4000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
open("/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0P\34\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1729984, ...}) = 0
mmap(NULL, 3836448, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f98d7e22000
mprotect(0x7f98d7fc1000, 2097152, PROT_NONE) = 0
mmap(0x7f98d81c1000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x19f000) = 0x7f98d81c1000
mmap(0x7f98d81c7000, 14880, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f98d81c7000
close(3) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f98d85e5000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f98d85e4000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f98d85e3000
arch_prctl(ARCH_SET_FS, 0x7f98d85e4700) = 0
mprotect(0x7f98d81c1000, 16384, PROT_READ) = 0
mprotect(0x7f98d83e2000, 4096, PROT_READ) = 0
mprotect(0x7f98d8608000, 4096, PROT_READ) = 0
munmap(0x7f98d85e6000, 130450) = 0
set_tid_address(0x7f98d85e49d0) = 4298
set_robust_list(0x7f98d85e49e0, 24) = 0
rt_sigaction(SIGRTMIN, {sa_handler=0x7f98d81d19f0, sa_mask=[], sa_flags=SA_RESTORER|SA_SIGINFO, sa_restorer=0x7f98d81da8d0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {sa_handler=0x7f98d81d1a80, sa_mask=[], sa_flags=SA_RESTORER|SA_RESTART|SA_SIGINFO, sa_restorer=0x7f98d81da8d0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM64_INFINITY}) = 0
mmap(NULL, 8392704, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f98d7621000
brk(NULL) = 0x1615000
brk(0x1636000) = 0x1636000
mprotect(0x7f98d7621000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f98d7e20ff0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f98d7e219d0, tls=0x7f98d7e21700, child_tidptr=0x7f98d7e219d0) = 4299
mmap(NULL, 8392704, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_STACK, -1, 0) = 0x7f98d6e20000
mprotect(0x7f98d6e20000, 4096, PROT_NONE) = 0
clone(child_stack=0x7f98d761fff0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f98d76209d0, tls=0x7f98d7620700, child_tidptr=0x7f98d76209d0) = 4300
futex(0x7f98d7e219d0, FUTEX_WAIT, 4299, NULLstrace: Process 4300 attached
[pid 4300] set_robust_list(0x7f98d76209e0, 24) = 0
[pid 4300] futex(0x601304, FUTEX_WAIT_PRIVATE, 1, NULLstrace: Process 4299 attached
[pid 4299] set_robust_list(0x7f98d7e219e0, 24) = 0
[pid 4299] fstat(1, {st_mode=S_IFCHR|0620, st_rdev=makedev(136, 0), ...}) = 0
[pid 4299] mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f98d8605000
[pid 4299] write(1, "This is pthread_1,the tpid is 42"..., 35This is pthread_1,the tpid is 4298
) = 35
[pid 4299] write(1, " This is pthread_1,the tpid is 4"..., 36 This is pthread_1,the tpid is 4298
) = 36
[pid 4299] write(1, " This is pthread_1,the tpid is 4"..., 36 This is pthread_1,the tpid is 4298
) = 36
[pid 4299] futex(0x601304, FUTEX_WAKE_OP_PRIVATE, 1, 1, 0x601300, FUTEX_OP_SET<<28|0<<12|FUTEX_OP_CMP_GT<<24|0x1) = 1
[pid 4299] madvise(0x7f98d7621000, 8368128, MADV_DONTNEED) = 0
[pid 4299] exit(0) = ?
[pid 4299] +++ exited with 0 +++
[pid 4298] <... futex resumed> ) = 0
[pid 4298] futex(0x7f98d76209d0, FUTEX_WAIT, 4300, NULL
[pid 4300] <... futex resumed> ) = 0
[pid 4300] write(1, " This is pthread_2.the tpid is 4"..., 36 This is pthread_2.the tpid is 4298
) = 36
[pid 4300] write(1, "This is pthread_2.the tpid is 42"..., 35This is pthread_2.the tpid is 4298
) = 35
[pid 4300] write(1, "This is pthread_2.the tpid is 42"..., 35This is pthread_2.the tpid is 4298
) = 35
[pid 4300] futex(0x6012c0, FUTEX_WAKE_PRIVATE, 1) = 0
[pid 4300] futex(0x601304, FUTEX_WAIT_PRIVATE, 3, NULL
首先是shell调用execve执行命令指行程序，用brk(NULL)得到当前主线程的堆的开始（此时也是尾部）地址，接下来的一系列操作都是在把当前进程需要的动态链接库通过mmap映射到主程序的mm_struct，让主程序能够寻址到，这中间的手法就是不断的open，read，然后通过state这个结构体获得动态库的长度，然后再mmap再close，这时的文件操作符都是3，而且为了亲属线程能够同时维护那部分地址，都采用anonymous模式进行mmap，用mmprotect设置内存保护模式，这时候主线程程该干的事干完了，就调用
set_tid_address()设置得到子线程的主线程的pid，然后调用
brk(NULL) = 0x1615000
brk(0x1636000) = 0x1636000
开辟新的堆空间，最后的clone调用是最经典的
clone(child_stack=0x7f98d7e20ff0, flags=CLONE_VM|CLONE_FS|CLONE_FILES|CLONE_SIGHAND|CLONE_THREAD|CLONE_SYSVSEM|CLONE_SETTLS|CLONE_PARENT_SETTID|CLONE_CHILD_CLEARTID, parent_tidptr=0x7f98d7e219d0, tls=0x7f98d7e21700, child_tidptr=0x7f98d7e219d0) = 4299这些选项说明了很多问题，这代表了子线程共享了主线程的VM虚拟地址意味着页目录那些一样，FS也就是文件系统，FILES打开的文件描述符也共享这个是linux能用管道这种牛逼操作的原因所在，就是因为子线程的这种共享，让linux的shell焕发活力，clone函数还有个tls参数，这个参数来自于我之前所说的在GDT中的TLS线程描述符字段，专门用来存储该进程的子线程内存映射地址。
3.了解这些有啥用
首先，你对程序怎样起来，怎样被内核运行有了大致了解（限于篇幅我没有叙述进程调度部分），然后你会理解一些东西：
1.单核CPU的多线程与多核的区别：
每一个进程都会被分配一个内核堆栈以及挨着的thread_info结构体但是对于每一个CPU（每个CPU单独有一套寄存器也就是硬件上下文）只能作一个内核堆栈寻址（通过esp找到它），只有在进程切换时，这个地址被内核控制路径找到，很明显在单核CPU编程根本不需要锁的存在，因为无论何时只有一个执行流在CPU上运行，不会产生共享资源冲突，在单核CPU下的多线程是伪多线程，只是CPU依次处理各个进程被内核分配的时间片（800ms，400ms不等看优先级之后有缘会写），只需要注意一些编译器优化问题，于是编程人员想到了用violate来限制编译器优化一些冲突变量（因为编译器有一些很邪恶的优化它会根据你这条指令的lantency也就是效率进行优化读这种操作在多核处理器中读是不会产生cache缓存同步故往往编译器会尽可能让其提前让写滞后从而产生data race ），保持程序sc（sequential consistency），而这种violate也是之后被一些编程人员误解的缘由，他们会感叹这个东西怎么不起作用了，这个必然的，多核CPU的并行编程是远远比这复杂的，有部分是因为编译器的遗留问题，因为编译器至今所做的优化是不会考虑多核情况，（听说最近C++11有了方面的新特性），为了这个目的，不仅在编程上会设有各种内存避障以及锁，这些东西在内核源码随处可见：
#define mb() alternative("lock; addl $0,0(%%esp)", "mfence", X86_FEATURE_XMM2)
#define rmb() alternative("lock; addl $0,0(%%esp)", "lfence", X86_FEATURE_XMM2)
#define wmb() alternative("lock; addl $0,0(%%esp)", "sfence", X86_FEATURE_XMM)2.对逆向更成熟于胸，而且加深了对shell的了解，且对bug定位有很大帮助：
比如说 ls | more这种操作，其实就是利用了两个线程共享文件描述符，由之前我的分析你应该能清楚ls打开的文件描述符应该是3，more是4，0是ls的输入也是整个主线程的输入，1 是输出，那么这个管道的作用就是把4这个文件流给转移到1，把3的文件流转移到0，这样管道就形成了，而这样形成的条件就是这两个进程是兄弟进程和shell共享文件描述符。而对逆向，当你对整个程序执行流了解过后，那么思路会比较清晰。