golang 不同方式插入postgres比较

golang 不同方式插入postgres比较

 

Description:

• 常规插入: 28.855439529s
• 事物循环插入: 15.848329755s
• pg copy 插入: 37.207166ms

Example:

package main

import (
	"database/sql"
	"fmt"
	_ "github.com/lib/pq"
	"time"
	"github.com/lib/pq"
	"strconv"
)

const (
	host     = "localhost"
	port     = 5432
	user     = "root"
	password = "public"
	dbname   = "test"
)


func Connect()(*sql.DB){
	psqlInfo := fmt.Sprintf("host=%s port=%d user=%s "+
		"password=%s dbname=%s sslmode=disable",
		host, port, user, password, dbname)
	db, err := sql.Open("postgres", psqlInfo)
	if err != nil {
		panic(err)
	}
	err = db.Ping()
	if err != nil {
		panic(err)
	}
	fmt.Println("Successfully connected!")
	return db
}


func normalInsert(db *sql.DB){
	sqlStatement := `
		INSERT INTO test("content")
		values ($1)
	`
	db.SetMaxIdleConns(10)
	db.SetMaxOpenConns(10)
	t1 := time.Now()
	for i:=0; i<10000; i++{
		db.Exec(sqlStatement, strconv.Itoa(i))
	}
	elapsed := time.Since(t1)
	fmt.Println("normal insert time: ", elapsed)
}


func transactionInsert(db *sql.DB){
	sqlStatement := `
		INSERT INTO test("content")
		values ($1)
	`
	db.SetMaxIdleConns(10)
	db.SetMaxOpenConns(10)
	txn, err := db.Begin()
	if err != nil {
		panic(err)
	}
	t1 := time.Now()
	for i:=0; i<10000; i++{
		txn.Exec(sqlStatement, strconv.Itoa(i))
	}
	txn.Commit()
	elapsed := time.Since(t1)
	fmt.Println("transaction insert time: ", elapsed)
}


func pgCopyInsert(db *sql.DB) {
	db.SetMaxIdleConns(10)
	db.SetMaxOpenConns(10)
	db.SetConnMaxLifetime(0)

	txn, err := db.Begin()
	if err != nil {
		panic(err)
	}
	t1 := time.Now()
	stmt, _ := txn.Prepare(pq.CopyIn("test", "content"))
	for i:=0; i< 10000; i++{
		_, err := stmt.Exec("copy insert")
		if err != nil {
			panic(err)
		}
	}
	_, err = stmt.Exec()
	if err != nil {
		panic(err)
	}
	err = stmt.Close()
	if err != nil {
		panic(err)
	}
	err = txn.Commit()
	if err != nil {
		panic(err)
	}
	elapsed := time.Since(t1)
	fmt.Println("copy insert time: ", elapsed)
}


func main() {
	db := Connect()
	normalInsert(db)
	transactionInsert(db)
	pgCopyInsert(db)
}

Remakes:

使用pg copy 插入速度差不多是百倍，但是copy也有不足之处

reference：

Insert Optimisations in Golang