rtmpdump源代码分析------HandShake

 HandShake的流程图：

１：握手以客户端发送 C0 和 C1 块开始。
２：客户端必须等待接收到 S1 才能发送 C2。
３：客户端必须等待接收到 S2 才能发送任何其他数据。
４：服务器端必须等待接收到 C0 才能发送 S0 和 S1，也可以等待接收到 C1 再发送 S0 和 S1。服务器端必须等待接收到 C1 才能发送 S2。服务器端必须等待接收到 C2 才能发送任何其他数据。
  C0 和 S0 的格式
C0 和 S0 包都是一个单一的八位字节，以一个单独的八位整型域进行处理：

        以下是 C0/S0 包中的字段：
        版本号 (八位)：在 C0 中，这一字段指示出客户端要求的 RTMP 版本号。在 S0 中，这一字段指示出服务器端选择的 RTMP 版本号。本文档中规范的版本号为 3。0、1、2 三个值是由早期其他产品使用的，是废弃值；4 - 31 被保留为 RTMP 协议的未来实现版本使用；32 - 255 不允许使用 (以区分开 RTMP 和其他常以一个可打印字符开始的文本协议)。无法识别客户端所请求版本号的服务器应该以版本 3 响应，(收到响应的) 客户端可以选择降低到版本 3，或者放弃握手。
 C1 和 S1 的格式
        C1 和 S1 数据包的长度都是 1536 字节，包含以下字段：

        Time (四个字节)：这个字段包含一个 timestamp，用于本终端发送的所有后续块的时间起点。这个值可以是 0，或者一些任意值。要同步多个块流，终端可以发送其他块流当前的 timestamp 的值。
        Zero (四个字节)：这个字段必须都是 0。
        Random data (1528 个字节)：这个字段可以包含任意值。终端需要区分出响应来自它发起的握手还是对端发起的握手，这个数据应该发送一些足够随机的数。这个不需要对随机数进行加密保护，也不需要动态值。
 C2 和 S2 的格式
        C2 和 S2 数据包长度都是 1536 字节，基本就是 S1 和 C1 的副本 (分别)，包含有以下字段：

        Time (四个字节)：这个字段必须包含终端在 S1 (给 C2) 或者 C1 (给 S2) 发的 timestamp。
        Time2 (四个字节)：这个字段必须包含终端先前发出数据包 (s1 或者 c1) timestamp。
        Random echo (1528 个字节)：这个字段必须包含终端发的 S1 (给 C2) 或者 S2 (给 C1) 的随机数。两端都可以一起使用 time 和 time2 字段再加当前 timestamp 以快速估算带宽和/或者连接延迟，但这不太可能是有多大用处。
 握手示意图

如果觉得上面的流程相对来说比较难理解的话：

代码的实现:

static int HandShake(RTMP * r, int FP9HandShake)
{
	int i, offalg = 0;
	int dhposClient = 0;
	int digestPosClient = 0;
	int encrypted = r->Link.protocol & RTMP_FEATURE_ENC;

	RC4_handle keyIn = 0;
	RC4_handle keyOut = 0;

	int32_t *ip;
	uint32_t uptime;

	uint8_t clientbuf[RTMP_SIG_SIZE + 4], *clientsig = clientbuf + 4;
	uint8_t serversig[RTMP_SIG_SIZE], client2[RTMP_SIG_SIZE], *reply;
	uint8_t type;
	getoff *getdh = NULL, *getdig = NULL;

	if (encrypted || r->Link.SWFSize)
		FP9HandShake = TRUE;//加密的
	else
		FP9HandShake = FALSE;//普通的

	r->Link.rc4keyIn = r->Link.rc4keyOut = 0;

	if (encrypted)
	{
		clientsig[-1] = 0x06; /* 0x08 is RTMPE as well */
		offalg = 1;
	}
	else
		clientsig[-1] = 0x03;//普通rtmp的Version

	uptime = htonl(RTMP_GetTime());//获取的时间戳
	memcpy(clientsig, &uptime, 4);

	if (FP9HandShake)
	{
		/* set version to at least 9.0.115.0 */
		if (encrypted)
		{
			clientsig[4] = 128;
			clientsig[6] = 3;
		}
		else
		{
			clientsig[4] = 10;
			clientsig[6] = 45;
		}
		clientsig[5] = 0;
		clientsig[7] = 2;

		RTMP_Log(RTMP_LOGDEBUG, "%s: Client type: %02X", __FUNCTION__, clientsig[-1]);
		getdig = digoff[offalg];
		getdh = dhoff[offalg];
	}
	else
	{
		memset(&clientsig[4], 0, 4);//zero,占据４ byte
	}

	/* generate random data */
#ifdef _DEBUG
	memset(clientsig+8, 0, RTMP_SIG_SIZE-8);
#else
	ip = (int32_t *)(clientsig + 8);//从第八位开始，随机数据，进行填充clientsig
	for (i = 2; i < RTMP_SIG_SIZE / 4; i++)
		*ip++ = rand();
#endif

	/* set handshake digest */
	if (FP9HandShake)
	{
		if (encrypted)
		{
			/* generate Diffie-Hellmann parameters */
			r->Link.dh = DHInit(1024);
			if (!r->Link.dh)
			{
				RTMP_Log(RTMP_LOGERROR, "%s: Couldn't initialize Diffie-Hellmann!", __FUNCTION__);
				return FALSE;
			}

			dhposClient = getdh(clientsig, RTMP_SIG_SIZE);
			RTMP_Log(RTMP_LOGDEBUG, "%s: DH pubkey position: %d", __FUNCTION__, dhposClient);

			if (!DHGenerateKey(r->Link.dh))
			{
				RTMP_Log(RTMP_LOGERROR, "%s: Couldn't generate Diffie-Hellmann public key!", __FUNCTION__);
				return FALSE;
			}

			if (!DHGetPublicKey(r->Link.dh, &clientsig[dhposClient], 128))
			{
				RTMP_Log(RTMP_LOGERROR, "%s: Couldn't write public key!", __FUNCTION__);
				return FALSE;
			}
		}

		digestPosClient = getdig(clientsig, RTMP_SIG_SIZE); /* reuse this value in verification */
		RTMP_Log(RTMP_LOGDEBUG, "%s: Client digest offset: %d", __FUNCTION__, digestPosClient);

		CalculateDigest(digestPosClient, clientsig, GenuineFPKey, 30, &clientsig[digestPosClient]);

		RTMP_Log(RTMP_LOGDEBUG, "%s: Initial client digest: ", __FUNCTION__);
		RTMP_LogHex(RTMP_LOGDEBUG, clientsig + digestPosClient,
		SHA256_DIGEST_LENGTH);
	}

#ifdef _DEBUG
	RTMP_Log(RTMP_LOGDEBUG, "Clientsig: ");
	RTMP_LogHex(RTMP_LOGDEBUG, clientsig, RTMP_SIG_SIZE);
#endif

	//sed to rtmp server,主要发送的是c0+c1,其中c0为version,c1为1536 byte,包裹时间戳以及zero及填充的信息
	if (!WriteN(r, (char *)clientsig - 1, RTMP_SIG_SIZE + 1))
		return FALSE;

	//recv version,一般是0x03 or 0x06
	if (ReadN(r, (char *)&type, 1) != 1) /* 0x03 or 0x06 */
		return FALSE;

	RTMP_Log(RTMP_LOGDEBUG, "%s: Type Answer   : %02X", __FUNCTION__, type);
	//比较版本是否一致，也就是发送的c0 给服务器，服务器寻找所支持的版本后，返回给client.
	if (type != clientsig[-1])
		RTMP_Log(RTMP_LOGWARNING, "%s: Type mismatch: client sent %d, server answered %d", __FUNCTION__, clientsig[-1], type);
	//获取Ｓ1的数据，这部分的数据，用于Ｃ２发送的数据.
	if (ReadN(r, (char *)serversig, RTMP_SIG_SIZE) != RTMP_SIG_SIZE)
		return FALSE;

	/* decode server response */
	memcpy(&uptime, serversig, 4);
	uptime = ntohl(uptime);//大小端转换

	RTMP_Log(RTMP_LOGDEBUG, "%s: Server Uptime : %d", __FUNCTION__, uptime);
	RTMP_Log(RTMP_LOGDEBUG, "%s: FMS Version   : %d.%d.%d.%d", __FUNCTION__, serversig[4], serversig[5], serversig[6], serversig[7]);

	if (FP9HandShake && type == 3 && !serversig[4])
		FP9HandShake = FALSE;

#ifdef _DEBUG
	RTMP_Log(RTMP_LOGDEBUG, "Server signature:");
	RTMP_LogHex(RTMP_LOGDEBUG, serversig, RTMP_SIG_SIZE);
#endif

	if (FP9HandShake)
	{
		uint8_t digestResp[SHA256_DIGEST_LENGTH];
		uint8_t *signatureResp = NULL;

		/* we have to use this signature now to find the correct algorithms for getting the digest and DH positions */
		int digestPosServer = getdig(serversig, RTMP_SIG_SIZE);

		if (!VerifyDigest(digestPosServer, serversig, GenuineFMSKey, 36))
		{
			RTMP_Log(RTMP_LOGWARNING, "Trying different position for server digest!");
			offalg ^= 1;
			getdig = digoff[offalg];
			getdh = dhoff[offalg];
			digestPosServer = getdig(serversig, RTMP_SIG_SIZE);

			if (!VerifyDigest(digestPosServer, serversig, GenuineFMSKey, 36))
			{
				RTMP_Log(RTMP_LOGERROR, "Couldn't verify the server digest"); /* continuing anyway will probably fail */
				return FALSE;
			}
		}

		/* generate SWFVerification token (SHA256 HMAC hash of decompressed SWF, key are the last 32 bytes of the server handshake) */
		if (r->Link.SWFSize)
		{
			const char swfVerify[] = {0x01, 0x01};
			char *vend = r->Link.SWFVerificationResponse + sizeof(r->Link.SWFVerificationResponse);

			memcpy(r->Link.SWFVerificationResponse, swfVerify, 2);
			AMF_EncodeInt32(&r->Link.SWFVerificationResponse[2], vend, r->Link.SWFSize);
			AMF_EncodeInt32(&r->Link.SWFVerificationResponse[6], vend, r->Link.SWFSize);
			HMACsha256(r->Link.SWFHash, SHA256_DIGEST_LENGTH, &serversig[RTMP_SIG_SIZE - SHA256_DIGEST_LENGTH],
			SHA256_DIGEST_LENGTH, (uint8_t *)&r->Link.SWFVerificationResponse[10]);
		}

		/* do Diffie-Hellmann Key exchange for encrypted RTMP */
		if (encrypted)
		{
			/* compute secret key */
			uint8_t secretKey[128] = {0};
			int len, dhposServer;

			dhposServer = getdh(serversig, RTMP_SIG_SIZE);
			RTMP_Log(RTMP_LOGDEBUG, "%s: Server DH public key offset: %d", __FUNCTION__, dhposServer);
			len = DHComputeSharedSecretKey(r->Link.dh, &serversig[dhposServer], 128, secretKey);
			if (len < 0)
			{
				RTMP_Log(RTMP_LOGDEBUG, "%s: Wrong secret key position!", __FUNCTION__);
				return FALSE;
			}

			RTMP_Log(RTMP_LOGDEBUG, "%s: Secret key: ", __FUNCTION__);
			RTMP_LogHex(RTMP_LOGDEBUG, secretKey, 128);

			InitRC4Encryption(secretKey, (uint8_t *)&serversig[dhposServer], (uint8_t *)&clientsig[dhposClient], &keyIn, &keyOut);
		}

		reply = client2;
#ifdef _DEBUG
		memset(reply, 0xff, RTMP_SIG_SIZE);
#else
		ip = (int32_t *)reply;
		for (i = 0; i < RTMP_SIG_SIZE / 4; i++)
			*ip++ = rand();
#endif
		/* calculate response now */
		signatureResp = reply + RTMP_SIG_SIZE - SHA256_DIGEST_LENGTH;

		HMACsha256(&serversig[digestPosServer], SHA256_DIGEST_LENGTH, GenuineFPKey, sizeof(GenuineFPKey), digestResp);
		HMACsha256(reply, RTMP_SIG_SIZE - SHA256_DIGEST_LENGTH, digestResp,
		SHA256_DIGEST_LENGTH, signatureResp);

		/* some info output */
		RTMP_Log(RTMP_LOGDEBUG, "%s: Calculated digest key from secure key and server digest: ", __FUNCTION__);
		RTMP_LogHex(RTMP_LOGDEBUG, digestResp, SHA256_DIGEST_LENGTH);

#ifdef FP10
		if (type == 8)
		{
			uint8_t *dptr = digestResp;
			uint8_t *sig = signatureResp;
			/* encrypt signatureResp */
			for (i = 0; i < SHA256_DIGEST_LENGTH; i += 8)
				rtmpe8_sig(sig + i, sig + i, dptr[i] % 15);
		}
#if 0
		else if (type == 9))
		{
			uint8_t *dptr = digestResp;
			uint8_t *sig = signatureResp;
			/* encrypt signatureResp */
			for (i=0; i<SHA256_DIGEST_LENGTH; i+=8)
			rtmpe9_sig(sig+i, sig+i, dptr[i] % 15);
		}
#endif
#endif
		RTMP_Log(RTMP_LOGDEBUG, "%s: Client signature calculated:", __FUNCTION__);
		RTMP_LogHex(RTMP_LOGDEBUG, signatureResp, SHA256_DIGEST_LENGTH);
	}
	else
	{
		reply = serversig;
#if 0
		uptime = htonl(RTMP_GetTime());
		memcpy(reply+4, &uptime, 4);
#endif
	}

#ifdef _DEBUG
	RTMP_Log(RTMP_LOGDEBUG, "%s: Sending handshake response: ",
			__FUNCTION__);
	RTMP_LogHex(RTMP_LOGDEBUG, reply, RTMP_SIG_SIZE);
#endif
	//send C2到服务器
	if (!WriteN(r, (char *)reply, RTMP_SIG_SIZE))
		return FALSE;

	/* 2nd part of handshake */
	//read到s2的内容
	if (ReadN(r, (char *)serversig, RTMP_SIG_SIZE) != RTMP_SIG_SIZE)
		return FALSE;

#ifdef _DEBUG
	RTMP_Log(RTMP_LOGDEBUG, "%s: 2nd handshake: ", __FUNCTION__);
	RTMP_LogHex(RTMP_LOGDEBUG, serversig, RTMP_SIG_SIZE);
#endif

	if (FP9HandShake)
	{
		uint8_t signature[SHA256_DIGEST_LENGTH];
		uint8_t digest[SHA256_DIGEST_LENGTH];

		if (serversig[4] == 0 && serversig[5] == 0 && serversig[6] == 0 && serversig[7] == 0)
		{
			RTMP_Log(RTMP_LOGDEBUG, "%s: Wait, did the server just refuse signed authentication?", __FUNCTION__);
		}
		RTMP_Log(RTMP_LOGDEBUG, "%s: Server sent signature:", __FUNCTION__);
		RTMP_LogHex(RTMP_LOGDEBUG, &serversig[RTMP_SIG_SIZE - SHA256_DIGEST_LENGTH],
		SHA256_DIGEST_LENGTH);

		/* verify server response */
		HMACsha256(&clientsig[digestPosClient], SHA256_DIGEST_LENGTH, GenuineFMSKey, sizeof(GenuineFMSKey), digest);
		HMACsha256(serversig, RTMP_SIG_SIZE - SHA256_DIGEST_LENGTH, digest,
		SHA256_DIGEST_LENGTH, signature);

		/* show some information */
		RTMP_Log(RTMP_LOGDEBUG, "%s: Digest key: ", __FUNCTION__);
		RTMP_LogHex(RTMP_LOGDEBUG, digest, SHA256_DIGEST_LENGTH);

#ifdef FP10
		if (type == 8)
		{
			uint8_t *dptr = digest;
			uint8_t *sig = signature;
			/* encrypt signature */
			for (i = 0; i < SHA256_DIGEST_LENGTH; i += 8)
				rtmpe8_sig(sig + i, sig + i, dptr[i] % 15);
		}
#if 0
		else if (type == 9)
		{
			uint8_t *dptr = digest;
			uint8_t *sig = signature;
			/* encrypt signatureResp */
			for (i=0; i<SHA256_DIGEST_LENGTH; i+=8)
			rtmpe9_sig(sig+i, sig+i, dptr[i] % 15);
		}
#endif
#endif
		RTMP_Log(RTMP_LOGDEBUG, "%s: Signature calculated:", __FUNCTION__);
		RTMP_LogHex(RTMP_LOGDEBUG, signature, SHA256_DIGEST_LENGTH);
		if (memcmp(signature, &serversig[RTMP_SIG_SIZE - SHA256_DIGEST_LENGTH],
		SHA256_DIGEST_LENGTH) != 0)
		{
			RTMP_Log(RTMP_LOGWARNING, "%s: Server not genuine Adobe!", __FUNCTION__);
			return FALSE;
		}
		else
		{
			RTMP_Log(RTMP_LOGDEBUG, "%s: Genuine Adobe Flash Media Server", __FUNCTION__);
		}

		if (encrypted)
		{
			char buff[RTMP_SIG_SIZE];
			/* set keys for encryption from now on */
			r->Link.rc4keyIn = keyIn;
			r->Link.rc4keyOut = keyOut;

			/* update the keystreams */
			if (r->Link.rc4keyIn)
			{
				RC4_encrypt(r->Link.rc4keyIn, RTMP_SIG_SIZE, (uint8_t * ) buff);
			}

			if (r->Link.rc4keyOut)
			{
				RC4_encrypt(r->Link.rc4keyOut, RTMP_SIG_SIZE, (uint8_t * ) buff);
			}
		}
	}
	else
	{
		//比较c1的内容跟s2的内容是否一致。
		//其实验证支持的协议版本号、服务器的时间戳等等。确保连接的对端真的是RTMP支持。
		if (memcmp(serversig, clientsig, RTMP_SIG_SIZE) != 0)
		{
			RTMP_Log(RTMP_LOGWARNING, "%s: client signature does not match!", __FUNCTION__);
		}
	}

	RTMP_Log(RTMP_LOGDEBUG, "%s: Handshaking finished....", __FUNCTION__);
	return TRUE;
}

参考：http://blog.csdn.net/defonds/article/details/17534903