三、七牛老许
qlang: github qiniu/qlang
microservice architecture: http://martinfowler.com/articles/microservices.html
Service Governance
- Authorization
- Logging
- Change management
- Central configuration
- Scale in and scale out
- Overload Protection
- Service degradation
- Monitor performance and health
- Manage how and by whom services are used
- Topology discovery and failure recovery
Overload Protection
- N = Alert threshold
- Important things
- keep limit to N*2,not N
- kill slow requests(SLA)
四、Dave Cheney 澳洲golang社区负责人，运行悉尼go用户组
http://dave.cheney.net/
Further reading: runtime/pprof
Further reading: Google Perf Tools
- pprof
-memprofilerate=N adjusts the profile rate to 1/N
#visualise mem profile go build -gcflags='-memprofile=/tmp/m.p' go tool pprof --alloc_objects -svg $(go tool -n compile) /tmp/m.p > alloc_objects.svg go tool pprof --inuse_objects -svg $(go tool -n compile) /tmp/m.p > alloc_objects.svg
- benchmarking
#fib.go:
func Fib(n int) int {
if n<2 {
return n
}
return Fib(n-1)+Fib(n-2)
}
#fib_test.go:
import "testing"
func BenchmarkFib(b *testing.B) {
for n:=0;n
-
- do benchmark multiple times or use "-count=" flag
- tools like rsc.io/benchstat are useful for comparing results
go test -bench=. -count=5 ./fib> old.txt
go test -bench=. -count=5 ./fib |tee new.txt
benchstat old.txt new.txt
benchstat {old,new}.txt
-
- how fast will this bench? I think it's super fast!
package popcnt
import "testing"
const m1 = 0x5555555555
const m2 = 0x33333
const m4 = 0x0f0f0f
const h01 = 0x0101010101
func popcnt(x uint64) uint64 {
x -= (x >> 1) & m1
x = (x & m2) + ((x >> 2) & m2)
x = (x + (x >> 4)) & m4
return (x * h01) >> 56
}
func benchmarkPopcnt(b *testing.B) {
for i := 0; i < b.N; i++ {
popcnt(uint64(i)) //optimied away
}
}
#go test -bench=. ./popcnt
-
- 但是被优化没有效果，要改为如下才有bench的效果
#change it to this:
var result uint64
func benchmarkPopcnt(b *testing.B) {
var r uint64
for i:=0;i
-
- 自己测试还是没效果,不知道为什么
- CPU profile
go test -run=XXX -bench=IndexByte -cpuprofile=/tmp/c.p bytes go tool pprof bytes.test /tmp/c.p #note: -run=XXX to disable test,you only need benchmark
- godoc
cd godoc golang.org/x/tools/cmd/godoc vim main.go add one line in main--defer profile.Start().Stop() then exec in vim:GoImports go install -v . godoc -http=9000 open browser and open localhost:9000 go tool pprof $(which godoc) /var/...cpu.pprof
- enable output of GCloggin
env GODEBUG=gctrace=1 godoc -http=:8080
- import net/http/pprof package will register a handler at /debug/pprof and default http.serveMux
it will visible if you use http.listenSndServe(adress,nil)
godoc -http:=8080,show /debug/pprof
- some slice and string translate...
五、晚场辩论
rust-lang.org
//GO
err := f(x)
if err != nil {
retrun err;
}
//Rust
try!(f(x));
f(x)?
一个关于冗长与magic的讨论
六、原射手CEO
gomobile和动态库的使用
七、米嘉
github.com/mijia/gobuildweb
github.com/mijia/web-starter-kit
@fswatch $(GO_FILES) $(TEMPLATES) | xargs -n1 -l{} make restart ||make kill
- chubby--at the heart of google
- go Proverbs: simple,Poetic,Pithy,no magic,composing,interface,adaptor
- request --> onion --> response
- basiclly we put a lot of things inside handler: like parse request,query db,user auth, CRUD model objects, biz logic, then render the result to response xml,json etc. or panic/error.
- go web packages:
dozens of frameworks
julienschmidt/httprouter
codegangstar/negroni
www.gorillatoolkit.org
- go generate according to database scheme
github.com/mijia/modelg
- go benchmark
load_test.go//测试1000000LRUcache
package demo1
import (
"fmt"
"testing"
"time"
)
//TestPutPerf load tests
func TestPutPerf(t *testing.T) {
fmt.Printf("load testing on 1M size cache\n")
size := 1000000
cache := newLRUCache(size)
start := time.Now()
for i,td := range testData {
if i%10000 == 0 {
end := time.Now()
fmt.Printf("Num: %d, Used time: %v\n", i, end.Sub(start))
start = end
}
cache.Put(td.url,td.body)
}
}
go test -c ./demo1.test -test.run=Perf
//lru_cache_test.go
package demo1
import (
"fmt"
"testing"
)
type testDataType struct {
url string
body string
}
var testData []testDataType
const testDataSize = 10000100
func init () {
testData = make([]testDataType, testDataSize)
for i:= 0; i< testDataSize; i++ {
url := fmt.Sprintf("www.fake.com/%d",i)
testData[i] = testDataType {
url: url,
body: fmt.Sprintf("This is page %s!",url),
}
}
}
func TestBasics(t *testing.T) {
cache := newLRUCache(2)
cache.Put("1","1")
cache.Put("2","2")
res, ok := cache.Get("1")
if !ok := || res != "1" {
t.Errorf("unexpected!")
}
cache.Put("4","4")
if _, ok := cache.Get("2");ok {
t.Errorf("unexpected")
}
}
//go tool pprof
func BenchmarkCacheFull1KSize(b *test.B) {
benchmarkCacheFull(b,1000)
}
func BenchmarkCacheFull1MSize(b *testing.B) {
benchmarkCacheFull(b,1000000)
}
func benchmarkCacheFull(b *testing.B, size int) {
cache := newLRUCache(size)
for i :=0;i#web
f, _ := os.Create("prof.out")
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
//先填满，之后测试每push 一个item需要多长时间
b.ResetTimer()
for i :=0; i< b.N; i++ {
c:=(i+size) % testDataSize
cache.Put(testData[c].url, testData[c].body)
}
}
//demo2
//go test -v . -run XXX -bench Mill
//./demo2.test -test.run XXX -test.bench Milli -test.cpuprofile=prof.out
//go tool pprof demo2.test prof.out --> web
./demo1.test -test.run=xxx -test.bench Full
#./demo1.test -test.run=xxx -test.bench Full -test.cpuprofile=prof.out 这是不准确的，因为有很多noise,用上面方法
八、Marcel van Lohuizen -- Google Go team
golang.org/x/text 实现
- I18n and L10n 国际化与本地化
搜索排序，大小写，双向文本，注入翻译，数字货币，日期，单位转换
- Go uses UTF-8
const beijing="北京市"
for index,runeValue :=range beijing {
fmt.Printf("%#U 从第%d字节开始\n",runeValue,index)
}
//北从0，京从3，市从6
- 文本的序列本质
const flags="????" //国家代码mc+nl fmt.Println(flags[4:])
- Transformer 接口
type Transformer interface {
Transform(dst,src []byte, atEOF bool) (nDst, nSrc int, err error)
Reset()
}
- Normalization 以NFC格式向w写入文本流
norm.NFC.Writer(w)
- 不同语言不同大小写算法
toTitle:=cases.Title(language.Dutch)
fmt.Println(toTitle.String("'n ijsberg"))
//'n IJsberg
- 多语言搜索与排序
e<ℨ ª==aa search 语言匹配 在代码中插入翻译结果 godoc.org/golang.org/x/text blog.golang.org/matchlang blog.golang.org/normalization blog.golang.org/strings golang.org/issue 九、毛剑--bilibili bfs分布式小文件存储: github.com/Terry-Mao/bfs 基于facebook Haystack Paper 十、李炳毅--百度 十一、TiDB PingCAP创始人 把整形赋值给interface会触法一次内存分配，会有一个数量级的开销 换成struct提升10倍以上 十二、陈辉－人工智能初创公司 主协程，用于收发用户请求。分词器协程，负责分词。索引器协程，负责查找索引。排序器。 基于悟空搜索的计算技术 AHA: Ad Hoc Analytics引擎 github.com/huichen/aha Double-Array Trie实现是一般Trie两倍效率 github.com/adamzy/cedar-go 监督式学习：最大熵分类模型，决策树模型 非监督式学习：聚类问题 在线学习：在线梯度递降模型 神经网络：Restricted Boltzmann machine 代码over，郑重声明：部分代码已测试, 但以上代码均不保证可运行，不保证无错误，并仅用于总结演讲ppt。 m:search.New(language.Danish,search.IgnoreCase,search.IgnoreDiacritics)
start,end:=m.IndexString(text,s)
match:=s[start:end]
import (
"fmt"
"golang.org/x/text/collate"
"golang.org/x/text/language"
)
func main () {
a:=[]string("北京市","上海市","广州市")
for _,tag := range []string{"en","zh","zh-u-co-stroke"} {
collate.New(language.Make(tag)).SortStrings(a)
fmt.Println(a)
}
}
// 上 北 广；北广上；上广北
import (
"http"
"golang.org/x/text/language"
)
var matcher = language.NewMatcher([]language.Tag{
language.SimplifiedChinese,
language.AmericanEnglish,
})
func handle(w http.ResponseWriter, r* http.Request){
prefs,_,_:=language.ParseAcceptLanguage(r.Header.Get("Accept-Languge"))
tag,_,_:=matcher.Match(prefs...)
//use tag: it includes carried over user preference
}
import "golang.org/x/text/message"
message.SetString(language.Dutch,"%s went to %s","%s is in %s geweest.")
message.SetString(language.SimplifiedChinese,"%s went to %s","%s 去了 %s ")
if r<= 0x7F {
switch
case '0' <= r && r <= '9':
return false
case 'a' <= r && r <= 'z':
return false
case 'A' <= r && r <= 'Z':
return false
case ==
return false
return true
}
if unicode.IsLetter(r) || unicode.IsDigit(r) {
return false
}
var val interface{}
val = int64(10)
var d Datum
d.SetInt64(10)
package main
import (
"github.com/huichen/wukong/engine"
"github.com/huichen/wukong/types"
"log"
)
var(
//searcher是协程安全的
searcher = engine.Engine{}
)
func main() {
searcher.Init(types.EngineInitOptions{SegmentedDictionaries:"github.com/huichen/wukong/data/dictionary.txt"})
defer searcher.Close()
searcher.IndexDocument(0,types.DocumentIndexData{Content: " XXXXX"})
searcher.IndexDocument(1,types.DocumentIndexData{Content: " XXXXX"})
searcher.IndexDocument(2,types.DocumentIndexData{Content: " XXXXX"})
searcher.FlushIndex()
log.Print(searcher.Search(types.SearchRequest{Text:"XX"}))
}
package main
import (
"fmt"
"github.com/huichen/sego"
)
func main() {
//dictionary
var segmenter sego.Segmenter
segmenter.LoadDictionary("github.com/huichen/sego/data/dictionary.txt")
//分词
text := []byte("北京市东城区XXXX")
segments := segmenter.Segment(text)
//处理分词结果
fmt.Println(sego.SegmentsToString(segments,false))
}
var (
libsvm_file = flag.String("input","","libsvm格式数据文件，训练数据")
model = flag.String("model","model.mlf","写入的模型文件")
//机器学习参数
learning_rate = flag.Float64("learning_rate",0.01,"学习率")
batch_size = flag.Int("batch_size",100,"梯度递降法的batch尺寸：1为stochasitic，其它值为mini batch")
delta = flag.Float64("delta",1e-4,"权重变化量和权重比值，dw/w 的绝对值，小于此值为收敛")
maxIter = flag.Int("max_iter",0,"优化器最多迭代多少次")
hidden = flag.Int("hidden",10,"多少个隐藏单元")
numCD = flag.Int("cd",1,"CD次数")
useBinary = flag.Bool("binary_hidden",true,"是否使用抽样隐藏单元")
)