统计S3 Bucket中key的数量

可以通过ListObjectV2的接口，列出Bucket中所有的key之后，再统计数量。
以京东云OSS为例（京东云OSS兼容aws 的S3）

Python代码

import boto3

"""
接口文档: https://docs.jdcloud.com/cn/object-storage-service/api/get-bucket-2?content=API
"""

ACCESS_KEY = ''
SECRET_KEY = ''
endpoint = "http://s3-internal.cn-north-1.jdcloud-oss.com"
bucket = ""
maxKeys = 1000
outfile = "./oss_list.txt"

s3 = boto3.client(
    's3',
    aws_access_key_id=ACCESS_KEY,
    aws_secret_access_key=SECRET_KEY,
    # 下面给出一个endpoint_url的例子
    endpoint_url=endpoint
)


f = open(outfile, "w")

resp = s3.list_objects_v2(Bucket=bucket, MaxKeys=maxKeys)

for content in resp["Contents"]:
    f.write(content["Key"] + " " + str(content["Size"]) + "\n")
f.flush()

flags = resp["IsTruncated"]

while flags:
    token = resp["NextContinuationToken"]
    resp = s3.list_objects_v2(Bucket=bucket, MaxKeys=maxKeys, ContinuationToken=token)
    for content in resp["Contents"]:
        f.write(content["Key"] + " " + str(content["Size"]) + "\n")
    f.flush()
    flags = resp["IsTruncated"]

f.close()

Golang代码

package main

import (
    "bufio"
    "encoding/json"
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/credentials"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/s3"
    "io/ioutil"
    "os"
    "strconv"
)

type ListObjectsConfig struct {
    AccessKey string `json:"accessKey"`
    SecretKey string `json:"secretKey"`
    Endpoint string `json:"endpoint"`
    Bucket string `json:"bucket"`
    Region string `json:"region"`
}

func listObjects() {
    configFile := os.Args[1]
    configStr, err := ioutil.ReadFile(configFile)
    if err != nil {
        panic(err)
    }
    config := &ListObjectsConfig{}
    err = json.Unmarshal(configStr, config)
    if err != nil {
        panic(err)
    }
    
    outfile := os.Args[2]
    
    creds := credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, "")
    _, err = creds.Get()
    if err != nil {
        panic(err)
    }
    
    awsConfig := &aws.Config{
        Region:      aws.String(config.Region),
        Endpoint:    aws.String(config.Endpoint),
        DisableSSL:  aws.Bool(false),
        Credentials: creds,
    }
    
    client := s3.New(session.New(awsConfig))
    
    f, err := os.OpenFile(outfile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
    if err != nil {
        panic(err)
    }
    defer f.Close()
    w := bufio.NewWriter(f)
    
    isTruncated := true
    var token *string
    for isTruncated {
        resp, err := client.ListObjectsV2(&s3.ListObjectsV2Input{
            Bucket: aws.String(config.Bucket),
            MaxKeys: aws.Int64(1000),
            ContinuationToken: token,
        })
        if err != nil {
            panic(err)
        }
        isTruncated = *resp.IsTruncated
        token = resp.NextContinuationToken
        for _, content := range resp.Contents {
            _, err = w.WriteString(*content.Key + " " + strconv.FormatInt(*content.Size, 10) + "\n")
            if err != nil {
                panic(err)
            }
        }
        w.Flush()
    }
}

func main() {
    if len(os.Args) != 3 {
        fmt.Println("参数数量异常! ")
        fmt.Println("./s3list CONFIG OUTFILE")
    }
    listObjects()
}

golang读取的配置文件

{
  "accessKey": "",
  "secretKey": "",
  "endpoint": "http://s3.cn-north-1.jdcloud-oss.com",
  "bucket": "",
  "region": "cn-north-1"
}