golang文件上传服务器
适合在工作中临时需要对外传输文件却被防火墙限制的情况
单文件，直接build
package main
import (
"encoding/json"
"flag"
"net/http"
"os"
"crypto/rand"
"github.com/sirupsen/logrus"
"crypto/sha1"
"errors"
"fmt"
"io"
"io/ioutil"
"path"
"regexp"
"strings"
)
var logger *logrus.Logger
var (
rePathUpload = regexp.MustCompile(`^/upload$`)
rePathFiles = regexp.MustCompile(`^/files/([^/]+)$`)
errTokenMismatch = errors.New("token mismatched")
errMissingToken = errors.New("missing token")
protectedMethods = []string{http.MethodGet, http.MethodHead, http.MethodPost, http.MethodPut}
)
type response struct {
OK bool `json:"ok"`
}
type uploadedResponse struct {
response
Path string `json:"path"`
}
func newUploadedResponse(path string) uploadedResponse {
return uploadedResponse{response: response{OK: true}, Path: path}
}
type errorResponse struct {
response
Message string `json:"error"`
}
func newErrorResponse(err error) errorResponse {
return errorResponse{response: response{OK: false}, Message: err.Error()}
}
func writeError(w http.ResponseWriter, err error) (int, error) {
body := newErrorResponse(err)
b, e := json.Marshal(body)
// if an error is occured on marshaling, write empty value as response.
if e != nil {
return w.Write([]byte{})
}
return w.Write(b)
}
func writeSuccess(w http.ResponseWriter, path string) (int, error) {
body := newUploadedResponse(path)
b, e := json.Marshal(body)
// if an error is occured on marshaling, write empty value as response.
if e != nil {
return w.Write([]byte{})
}
return w.Write(b)
}
func getSize(content io.Seeker) (int64, error) {
size, err := content.Seek(0, os.SEEK_END)
if err != nil {
return 0, err
}
_, err = content.Seek(0, io.SeekStart)
if err != nil {
return 0, err
}
return size, nil
}
// Server represents a simple-upload server.
type Server struct {
DocumentRoot string
// MaxUploadSize limits the size of the uploaded content, specified with "byte".
MaxUploadSize int64
SecureToken string
EnableCORS bool
}
// NewServer creates a new simple-upload server.
func NewServer(documentRoot string, maxUploadSize int64, token string, enableCORS bool) Server {
return Server{
DocumentRoot: documentRoot,
MaxUploadSize: maxUploadSize,
SecureToken: token,
EnableCORS: enableCORS,
}
}
func (s Server) handleGet(w http.ResponseWriter, r *http.Request) {
if !rePathFiles.MatchString(r.URL.Path) {
w.WriteHeader(http.StatusNotFound)
writeError(w, fmt.Errorf("\"%s\" is not found", r.URL.Path))
return
}
if s.EnableCORS {
w.Header().Set("Access-Control-Allow-Origin", "*")
}
http.StripPrefix("/files/", http.FileServer(http.Dir(s.DocumentRoot))).ServeHTTP(w, r)
}
func (s Server) handlePost(w http.ResponseWriter, r *http.Request) {
srcFile, info, err := r.FormFile("file")
if err != nil {
logger.WithError(err).Error("failed to acquire the uploaded content")
w.WriteHeader(http.StatusInternalServerError)
writeError(w, err)
return
}
defer srcFile.Close()
logger.Debug(info)
size, err := getSize(srcFile)
if err != nil {
logger.WithError(err).Error("failed to get the size of the uploaded content")
w.WriteHeader(http.StatusInternalServerError)
writeError(w, err)
return
}
if size > s.MaxUploadSize {
logger.WithField("size", size).Info("file size exceeded")
w.WriteHeader(http.StatusRequestEntityTooLarge)
writeError(w, errors.New("uploaded file size exceeds the limit"))
return
}
body, err := ioutil.ReadAll(srcFile)
if err != nil {
logger.WithError(err).Error("failed to read the uploaded content")
w.WriteHeader(http.StatusInternalServerError)
writeError(w, err)
return
}
filename := info.Filename
if filename == "" {
filename = fmt.Sprintf("%x", sha1.Sum(body))
}
dstPath := path.Join(s.DocumentRoot, filename)
dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
if err != nil {
logger.WithError(err).WithField("path", dstPath).Error("failed to open the file")
w.WriteHeader(http.StatusInternalServerError)
writeError(w, err)
return
}
defer dstFile.Close()
if written, err := dstFile.Write(body); err != nil {
logger.WithError(err).WithField("path", dstPath).Error("failed to write the content")
w.WriteHeader(http.StatusInternalServerError)
writeError(w, err)
return
} else if int64(written) != size {
logger.WithFields(logrus.Fields{
"size": size,
"written": written,
}).Error("uploaded file size and written size differ")
w.WriteHeader(http.StatusInternalServerError)
writeError(w, fmt.Errorf("the size of uploaded content is %d, but %d bytes written", size, written))
}
uploadedURL := strings.TrimPrefix(dstPath, s.DocumentRoot)
if !strings.HasPrefix(uploadedURL, "/") {
uploadedURL = "/" + uploadedURL
}
uploadedURL = "/files" + uploadedURL
logger.WithFields(logrus.Fields{
"path": dstPath,
"url": uploadedURL,
"size": size,
}).Info("file uploaded by POST")
if s.EnableCORS {
w.Header().Set("Access-Control-Allow-Origin", "*")
}
w.WriteHeader(http.StatusOK)
writeSuccess(w, uploadedURL)
}
func (s Server) handlePut(w http.ResponseWriter, r *http.Request) {
matches := rePathFiles.FindStringSubmatch(r.URL.Path)
if matches == nil {
logger.WithField("path", r.URL.Path).Info("invalid path")
w.WriteHeader(http.StatusNotFound)
writeError(w, fmt.Errorf("\"%s\" is not found", r.URL.Path))
return
}
targetPath := path.Join(s.DocumentRoot, matches[1])
// We have to create a new temporary file in the same device to avoid "invalid cross-device link" on renaming.
// Here is the easiest solution: create it in the same directory.
tempFile, err := ioutil.TempFile(s.DocumentRoot, "upload_")
if err != nil {
logger.WithError(err).Error("failed to create a temporary file")
w.WriteHeader(http.StatusInternalServerError)
writeError(w, err)
return
}
defer r.Body.Close()
srcFile, info, err := r.FormFile("file")
if err != nil {
logger.WithError(err).WithField("path", targetPath).Error("failed to acquire the uploaded content")
w.WriteHeader(http.StatusInternalServerError)
writeError(w, err)
return
}
defer srcFile.Close()
// dump headers for the file
logger.Debug(info.Header)
size, err := getSize(srcFile)
if err != nil {
logger.WithError(err).WithField("path", targetPath).Error("failed to get the size of the uploaded content")
w.WriteHeader(http.StatusInternalServerError)
writeError(w, err)
return
}
if size > s.MaxUploadSize {
logger.WithFields(logrus.Fields{
"path": targetPath,
"size": size,
}).Info("file size exceeded")
w.WriteHeader(http.StatusRequestEntityTooLarge)
writeError(w, errors.New("uploaded file size exceeds the limit"))
return
}
n, err := io.Copy(tempFile, srcFile)
if err != nil {
logger.WithError(err).WithField("path", tempFile.Name()).Error("failed to write body to the file")
w.WriteHeader(http.StatusInternalServerError)
writeError(w, err)
return
}
// excplicitly close file to flush, then rename from temp name to actual name in atomic file
// operation if on linux or other unix-like OS (windows hosts should look into https://github.com/natefinch/atomic
// package for atomic file write operations)
tempFile.Close()
if err := os.Rename(tempFile.Name(), targetPath); err != nil {
os.Remove(tempFile.Name())
logger.WithError(err).WithField("path", targetPath).Error("failed to rename temp file to final filename for upload")
w.WriteHeader(http.StatusInternalServerError)
writeError(w, err)
return
}
logger.WithFields(logrus.Fields{
"path": r.URL.Path,
"size": n,
}).Info("file uploaded by PUT")
if s.EnableCORS {
w.Header().Set("Access-Control-Allow-Origin", "*")
}
w.WriteHeader(http.StatusOK)
writeSuccess(w, r.URL.Path)
}
func (s Server) handleOptions(w http.ResponseWriter, r *http.Request) {
var allowedMethods []string
if rePathFiles.MatchString(r.URL.Path) {
allowedMethods = []string{http.MethodPut, http.MethodGet, http.MethodHead}
} else if rePathUpload.MatchString(r.URL.Path) {
allowedMethods = []string{http.MethodPost}
} else {
w.WriteHeader(http.StatusNotFound)
writeError(w, errors.New("not found"))
return
}
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", strings.Join(allowedMethods, ","))
w.WriteHeader(http.StatusNoContent)
}
func (s Server) checkToken(r *http.Request) error {
// first, try to get the token from the query strings
token := r.URL.Query().Get("token")
// if token is not found, check the form parameter.
if token == "" {
token = r.FormValue("token")
}
if token == "" {
return errMissingToken
}
if token != s.SecureToken {
return errTokenMismatch
}
return nil
}
func isAuthenticationRequired(r *http.Request) bool {
for _, m := range protectedMethods {
if m == r.Method {
return true
}
}
return false
}
func (s Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if err := s.checkToken(r); isAuthenticationRequired(r) && err != nil {
w.WriteHeader(http.StatusUnauthorized)
writeError(w, err)
return
}
switch r.Method {
case http.MethodGet, http.MethodHead:
s.handleGet(w, r)
case http.MethodPost:
s.handlePost(w, r)
case http.MethodPut:
s.handlePut(w, r)
case http.MethodOptions:
s.handleOptions(w, r)
default:
w.Header().Add("Allow", "GET,HEAD,POST,PUT")
w.WriteHeader(http.StatusMethodNotAllowed)
writeError(w, fmt.Errorf("method \"%s\" is not allowed", r.Method))
}
}
func run(args []string) int {
bindAddress := flag.String("ip", "0.0.0.0", "IP address to bind")
listenPort := flag.Int("port", 8001, "port number to listen on")
tlsListenPort := flag.Int("tlsport", 25443, "port number to listen on with TLS")
// 5,242,880 bytes == 5 MiB
maxUploadSize := flag.Int64("upload_limit", 5242880, "max size of uploaded file (byte)")
tokenFlag := flag.String("token", "", "specify the security token (it is automatically generated if empty)")
logLevelFlag := flag.String("loglevel", "info", "logging level")
certFile := flag.String("cert", "", "path to certificate file")
keyFile := flag.String("key", "", "path to key file")
corsEnabled := flag.Bool("cors", false, "if true, add ACAO header to support CORS")
flag.Parse()
serverRoot := flag.Arg(0)
if len(serverRoot) == 0 {
flag.Usage()
return 2
}
if logLevel, err := logrus.ParseLevel(*logLevelFlag); err != nil {
logrus.WithError(err).Error("failed to parse logging level, so set to default")
} else {
logger.Level = logLevel
}
token := *tokenFlag
if token == "" {
count := 10
b := make([]byte, count)
if _, err := rand.Read(b); err != nil {
logger.WithError(err).Fatal("could not generate token")
return 1
}
token = fmt.Sprintf("%x", b)
logger.WithField("token", token).Warn("token generated")
}
tlsEnabled := *certFile != "" && *keyFile != ""
server := NewServer(serverRoot, *maxUploadSize, token, *corsEnabled)
http.Handle("/upload", server)
http.Handle("/files/", server)
errors := make(chan error)
go func() {
logger.WithFields(logrus.Fields{
"ip": *bindAddress,
"port": *listenPort,
"token": token,
"upload_limit": *maxUploadSize,
"root": serverRoot,
"cors": *corsEnabled,
}).Info("start listening")
if err := http.ListenAndServe(fmt.Sprintf("%s:%d", *bindAddress, *listenPort), nil); err != nil {
errors <- err
}
}()
if tlsEnabled {
go func() {
logger.WithFields(logrus.Fields{
"cert": *certFile,
"key": *keyFile,
"port": *tlsListenPort,
}).Info("start listening TLS")
if err := http.ListenAndServeTLS(fmt.Sprintf("%s:%d", *bindAddress, *tlsListenPort), *certFile, *keyFile, nil); err != nil {
errors <- err
}
}()
}
err := <-errors
logger.WithError(err).Info("closing server")
return 0
}
func main() {
logger = logrus.New()
logger.Info("starting up simple-upload-server")
result := run(os.Args)
os.Exit(result)
}
监听端口8001
上传方法：
./simple_upload_server -token f9403fc5f537b4ab332d /tmp
echo 'Hello, world!' > sample.txt
curl -Ffile=@sample.txt 'http://localhost:8001/upload?token=f9403fc5f537b4ab332d'
{"ok":true,"path":"/files/sample.txt"}
使用put上传
curl -X PUT -Ffile=@sample.txt "http://localhost:8001/files/another_sample.txt?token=f9403fc5f537b4ab332d"
{"ok":true,"path":"/files/another_sample.txt"}
下载
curl 'http://localhost:8001/files/sample.txt?token=f9403fc5f537b4ab332d'
hello, world!
TLS
./simple_upload_server -cert ./cert.pem -key ./key.pem root/
查看文件
$ curl -I 'http://localhost:8001/files/foobar.txt?token=f9403fc5f537b4ab332d'
HTTP/1.1 200 OK
Accept-Ranges: bytes
Content-Length: 9
Content-Type: text/plain; charset=utf-8
Last-Modified: Sun, 09 Oct 2016 14:35:39 GMT
Date: Sun, 09 Oct 2016 14:35:43 GMT
$ curl 'http://localhost:8001/files/foobar.txt?token=f9403fc5f537b4ab332d'
hello!!!
$ curl -I 'http://localhost:8001/files/unknown?token=f9403fc5f537b4ab332d'
HTTP/1.1 404 Not Found
Content-Type: text/plain; charset=utf-8
X-Content-Type-Options: nosniff
Date: Sun, 09 Oct 2016 14:37:48 GMT
Content-Length: 19
参考：
https://github.com/mayth/go-simple-upload-server
思路扩展：
使用rsync和netcat得方法也是可以的，最终目标就是在不改动软硬件防火墙的情况下将文件上传到指定服务器，方法很多，只要利用机器现有工具达成目标即可。