简单实现TCP下的大文件高效传输

在TCP下进行大文件传输不象小文件那样直接打包个BUFFER发送出去,因为文件比较大所以不可能把文件读到一个BUFFER发送出去.主要有些文件的大小可能是1G,2G或更大,分配这么大的BUFFER对内存来说显然是不现实的事情;针对服务端的设计来说就更需要严紧些,BUFFER大小的限制也是变得很重要.下面介绍使用Beetle简单地实现大文件在TCP的传输应用.

协议制定

既然需要把文件分块来处理,那在TCP传输的过程需要制定一些协议来规范数据有效性,数据协议主要有三个:告诉服务器需要上传文件,文件块上传和返回每个环节处理的结果.

1)上传文件指令

public class Upload:ObjectMessage
    {
        public string FileMD5
        {
            get;
            set;
        }

        public string FileName
        {
            get;
            set;
        }

        public long FileSize
        {
            get;
            set;
        }

        public override void FromProtocolData(HttpData httpbase)
        {
            FileName = httpbase[CONSTVALUE.HEADER_NAME];
            FileMD5 = httpbase[CONSTVALUE.HEADER_MD5];
            FileSize = long.Parse(httpbase[CONSTVALUE.HEADER_FILESIZE]);
        }

        protected override void OnDisposed()
        {
           
        }

        protected override void OnToProtocolData(HttpData httpbase)
        {
            httpbase.Command = CONSTVALUE.COMMAND_UPLOAD;
            httpbase[CONSTVALUE.HEADER_MD5] = FileMD5;
            httpbase[CONSTVALUE.HEADER_NAME] = FileName;
            httpbase[CONSTVALUE.HEADER_FILESIZE] = FileSize.ToString();
        }
    }

View Code

2)上传文件块指令

public class UploadData:ObjectMessage
    {

        public string FileMD5
        {
            get;
            set;
        }

        public Beetle.ByteArraySegment Data
        {
            get;
            set;
        }
        
        public override void FromProtocolData(HttpData httpbase)
        {
            FileMD5 = httpbase[CONSTVALUE.HEADER_MD5];
            Data = httpbase.Content;
        }

        protected override void OnDisposed()
        {
            if (Data != null)
            {
                FileTransferPackage.BufferPool.Push(Data);
                Data = null;
            }
        }

        protected override void OnToProtocolData(HttpData httpbase)
        {
            httpbase.Command = CONSTVALUE.COMMAND_UPLOAD_DATA;
            httpbase[CONSTVALUE.HEADER_MD5] = FileMD5;
            httpbase.Content = Data;
        }
    }

View Code

3)返回值指令

public class Result :ObjectMessage
    {
        public string FileMD5
        {
            get;
            set;
        }

        public bool Error
        {
            get;
            set;
        }

        public string ErrorDetail
        {
            get;
            set;
        }

        public override void FromProtocolData(HttpData httpbase)
        {
            ErrorDetail = httpbase[CONSTVALUE.HEADER_STATUS_DETAIL];
            Error = httpbase[CONSTVALUE.HEADER_STATUS] == CONSTVALUE.VALUE_SUCCESS;
            FileMD5 = httpbase[CONSTVALUE.HEADER_MD5];
        }

        protected override void OnDisposed()
        {

        }

        protected override void OnToProtocolData(HttpData httpbase)
        {
            httpbase.Command = CONSTVALUE.COMMAND_RESULT;
            if (Error)
            {
                httpbase[CONSTVALUE.HEADER_STATUS] = CONSTVALUE.VALUE_SUCCESS;
            }
            else
            {
                httpbase[CONSTVALUE.HEADER_STATUS] = CONSTVALUE.VALUE_ERROR;
            }
            httpbase[CONSTVALUE.HEADER_STATUS_DETAIL] = ErrorDetail;
            httpbase[CONSTVALUE.HEADER_MD5] = FileMD5;
        }
    }

View Code

ObjectMessage是Beetle一个简化HTTP协议的扩展对象,它提供自定义Header和Body等功能.

文件读写器

既然需要处理文件块,那提供一些简单的文件块读取和写入方法是比较重要的.它不仅从设计解决功能的偶合度,还可以方便今后的利用.

1)UploadReader 

public class UploadReader : IDisposable
    {
        public UploadReader(string file)
        {

            mStream = System.IO.File.OpenRead(file);
            StringBuilder sb = new StringBuilder();
            MD5 md5Hasher = MD5.Create();
            foreach (Byte b in md5Hasher.ComputeHash(mStream))
                sb.Append(b.ToString("x2").ToLower());
            FileMD5 = sb.ToString();
            mStream.Position = 0;
            FileSize = mStream.Length;
            FileName = System.IO.Path.GetFileName(file);
                
        }

        private System.IO.FileStream mStream = null;

        public string FileName
        {
            get;
            set;
        }

        public long LastReadLength
        {
            get;
            set;
        }

        public long ReadLength
        {
            get;
            set;
        }

        public long FileSize
        {
            get;
            set;
        }

        public string FileMD5
        {
            get;
            set;
        }

        public bool Completed
        {
            get
            {
                return mStream != null && ReadLength == mStream.Length;
            }
        }

        public void Close()
        {
            if (mStream != null)
            {
                mStream.Close();
                mStream.Dispose();
            }
        }

        public void Reset()
        {
            mStream.Position = 0;
            LastReadLength = 0;
            ReadLength = 0;
        }

        public void Read(ByteArraySegment segment)
        {
            int loads = mStream.Read(segment.Array, 0, FileTransferPackage.BUFFER_SIZE);
            segment.SetInfo(0, loads);
            ReadLength += loads;
        }

        public void Dispose()
        {
            mStream.Dispose();
        }

        public override string ToString()
        {
            string value= string.Format("{0}(MD5:{4})\r\n\r\n[{1}/{2}({3}/秒)]",FileName,ReadLength,FileSize,ReadLength-LastReadLength,FileMD5);
            if (!Completed)
            {
                LastReadLength = ReadLength;
            }
            return value;
        }
    }

View Code

UploadReader的功能主要是把文件流读取到指定大小的Buffer中,并提供方法获取当前的读取情况

2)UploadWriter

public class UploadWriter
    {
        public UploadWriter(string rootPath, string filename,string fileMD5,long size)
        {
            mFullName = rootPath + filename;
            FileName = filename;
            FileMD5 = fileMD5;
            Size = size;
        }

        private string mFullName;

        private System.IO.FileStream mStream;

        public System.IO.FileStream Stream
        {
            get
            {
                if (mStream == null)
                {
                    mStream = System.IO.File.Create(mFullName+".up");
                }
                return mStream;
            }
        }

        public long WriteLength
        {
            get;
            set;
        }

        public long LastWriteLength
        {
            get;
            set;
        }

        public long Size
        {
            get;
            set;
        }

        public string FileName
        {
            get;
            set;
        }

        public string FileMD5
        {
            get;
            set;
        }

        public bool Write(ByteArraySegment segment)
        {
            Stream.Write(segment.Array, 0, segment.Count);
            WriteLength += segment.Count;
            Stream.Flush();
            if (WriteLength == Size)
            {
                Stream.Close();
                if (System.IO.File.Exists(mFullName))
                    System.IO.File.Delete(mFullName);
                System.IO.File.Move(mFullName + ".up", mFullName);
                return true;
            }
            return false;
        }
    }

View Code

UploadWriter的功能主要是把文件写入到临时文件中,写入完成后再更改相应的名称,为了方便查询同样也提供了一些写入情况信息.

服务端代码

 如果有了解过Beetle的服务端制定的话,那服务端的实现是非常简单的,只需要写一个对象承继ServerBase并实现数据接收方法处理即可以,接收的数据会会自动转换成之前定义的消息对象,而服务端内部处理的细节是完全不用关心.

protected override void OnMessageReceive(Beetle.PacketRecieveMessagerArgs e)
        {
            base.OnMessageReceive(e);
            if (e.Message is Protocol.Upload)
            {
                OnUpload(e.Channel, e.Message as Protocol.Upload);
            }
            else if (e.Message is Protocol.UploadData)
            {
                OnUploadData(e.Channel, e.Message as Protocol.UploadData);
            }
        }

        private Protocol.Result GetErrorResult(string detail)
        {
            Protocol.Result result = new Protocol.Result();
            result.Error = true;
            result.ErrorDetail = detail;
            return result;
        }

        private void OnUpload(Beetle.TcpChannel channel, Protocol.Upload e)
        {
            Protocol.Result result;
            if (mTask[e.FileMD5] != null)
            { 
                result = GetErrorResult( "该文件正在上传任务中!");
                channel.Send(result);
                return;
            }
            UploadWriter writer = new UploadWriter(mRootPath, e.FileName, e.FileMD5, e.FileSize);
            lock (mTask)
            {
                mTask[e.FileMD5] = writer;
            }
            result = new Protocol.Result();
            channel.Send(result);
        }

        private void OnUploadData(Beetle.TcpChannel channel, Protocol.UploadData e)
        {
            using (e)
            {
                Protocol.Result result;
                UploadWriter writer = (UploadWriter)mTask[e.FileMD5];
                if (writer == null)
                {
                    result = GetErrorResult("上传任务不存在!");
                    channel.Send(result);
                    return;
                }
                if (writer.Write(e.Data))
                {
                    lock (mTask)
                    {
                        mTask.Remove(e.FileMD5);
                    }
                }
                result = new Protocol.Result();
                result.FileMD5 = writer.FileMD5;
                channel.Send(result);
            }
        }

当接收到客户求上传请求后会建立对应MD5的文件写入器,后面文件块的上传写入相关对象即可.

客户端代码

 Beetle对于Client的支持也是非常简单方便,只需要定义一个TcpChannel直接发送定义的对象消息并获取服务器端返回的消息即可.

 1 private void OnUpload(object state)
 2         {
 3             Lib.UploadReader reader = (Lib.UploadReader)state;
 4             try
 5             {
 6                 IsUpload = true;
 7                 Lib.Protocol.Upload upload = new Lib.Protocol.Upload();
 8                 upload.FileMD5 = reader.FileMD5;
 9                 upload.FileName = reader.FileName;
10                 upload.FileSize = reader.FileSize;
11                 Lib.Protocol.Result result = mClient.Send<Lib.Protocol.Result>(upload);
12                 if (result.Error)
13                 {
14                     mLastError = result.ErrorDetail;
15                     return;
16                 }
17                 while (!reader.Completed)
18                 {
19                     mLastError = "文件上传中...";
20                     Lib.Protocol.UploadData data = new Lib.Protocol.UploadData();
21                     data.Data = Lib.FileTransferPackage.BufferPool.Pop();
22                     data.FileMD5 = reader.FileMD5;
23                     reader.Read(data.Data);
24                     result = mClient.Send<Lib.Protocol.Result>(data);
25                     if (result.Error)
26                     {
27                         mLastError = result.ErrorDetail;
28                         return;
29                     }
30                 }
31                 mLastError = "文件上传完成!";
32                 
33             }
34             catch (Exception e_)
35             {
36                 mLastError = e_.Message;
37             }
38             mReader.Reset();
39             IsUpload = false;
40                
41         }

View Code

整个过程只需要一个方法却可完成,首先把需要上传的文件信息发送到服务器,当服务器确认后不停地把文件块信息输送到服务端即可.

使用测试

下载代码

FileTransfer.Lib.rar (644.64 kb)