在k8s集群中以svc的方式访问oss
在k8s集群中以svc的方式访问oss
-
- 背景需求
- 前置条件
- 原理及说明
-
- 原理：
- 说明：
- 部署文件结构
- 部署操作
-
- step1：编写oss_auth.lua文件，用于nginx中调用该文件
- step2：编写nginx的主配置文件
- step3：编写nginx主配置文件的secret文件，需要将nginx.conf以secert的方式挂载到容器中
- step4：编写oss_auth.lua的comfigmap文件，需要将oss_auth.lua以comfigmap的方式挂载到容器中
- step5：编写部署服务的文件
- step6：编写部署服务的svc文件
- step7：执行部署命令
- step8：验证
背景需求
有一个想法，就是既然k8s集群可以实现svc的方式访问外部服务（不管是ip：port或者域名二点方式），那可不可以实现一些特殊场景的需要呢，比如访问阿里云的oss服务。
参考链接:https://www.bbsmax.com/A/KE5QK60jzL/
前置条件
- 1、开通阿里云oss服务，获取到oss的ak、sk。
- 2、已部署好的k8s集群。
原理及说明
原理：
- 利用Nginx lua 实现将阿里云OSS存储空间做到同本地磁盘一样使用。核心是利用Nginx lua
对OSS请求进行签名并利用内部跳转将所有访问本地Nginx的请求加上OSS 签名转发给OSS，实现本地Nginx无缝衔接阿里云OSS。
说明：
- 1、将nginx.conf进行base64编码，以secret的方式进行部署，然后在deployment中进行将其挂载到nginx服务中的/etc/nginx/conf.d目录下，oss_auth.lua位于nginx服务的/opt目录下。
- 2、将oss_auth.lua以configmap的方式进行部署，然后在deployment中进行将其挂载，将其挂载到nginx服务中的/opt目录下。
- 3、nginx.conf文件配置了oss的ak，sk，bucketname，oss的url地址以及内部跳转的配置。
部署文件结构
-> /Users/wangkaixuan/k8s_deploy_oss ~#: tree
.
├── 0.oss_auth.lua
├── 1.nginx.conf
├── 2.nginx-conf-secrets.yaml
├── 3.nginx-lua-file-configmap.yaml
├── 4.oss-deployment.yaml
├── 5.oss-service.yaml
└── README.md
文件说明：
- 0.oss_auth.lua：为一个lua语言的文件，主要实现对oss的访问鉴权认证；
- 1.nginx.conf：为nginx的主配置文件，可以实现跳转到oss服务中；
- 2.nginx-conf-secrets.yaml：为nginx.conf在k8s集群以secret存放的方式；
- 3.nginx-lua-file-configmap.yaml：为oss_auth.lua文件在k8s集群以comfigmap存放的方式；
- 4.oss-deployment.yaml：为部署nginx-oss服务的deployment文件；
- 5.oss-service.yaml：为nginx-oss服务的service文件。
部署操作
step1：编写oss_auth.lua文件，用于nginx中调用该文件
cat 0.oss_auth.lua
-- has been sorted in alphabetical order
local signed_subresources = {
'acl',
'append',
'bucketInfo',
'cname',
'commitTransition',
'comp',
'cors',
'delete',
'lifecycle',
'location',
'logging',
'mime',
'notification',
'objectInfo',
'objectMeta',
'partData',
'partInfo',
'partNumber',
'policy',
'position',
'referer',
'replication',
'replicationLocation',
'replicationProgress',
'requestPayment',
'response-cache-control',
'response-content-disposition',
'response-content-encoding',
'response-content-language',
'response-content-type',
'response-expires',
'restore',
'security-token',
'tagging',
'torrent',
'uploadId',
'uploads',
'versionId',
'versioning',
'versions',
'website'
}
function string.startswith(s, start)
return string.sub(s, 1, string.len(start)) == start
end
local function get_canon_sub_resource()
local args = ngx.req.get_uri_args()
-- lower keys
local keys = {}
for k, v in pairs(args) do
keys[k:lower()] = v
end
-- make resource string
local s = ''
local sep = '?'
for i, k in ipairs(signed_subresources) do
v = keys[k]
if v then
-- sub table
v = type(v) == 'table' and v[1] or v
s = s .. string.format("%s%s=%s", sep, k, v)
sep = '&'
end
end
return s
end
local function get_canon_resource()
resource = ''
object = ngx.unescape_uri(ngx.var.uri)
sub = get_canon_sub_resource()
return string.format("/%s%s%s", ngx.var.oss_bucket, object, sub)
end
local function get_canon_headers()
-- default: <lowerkey, value>
local headers = ngx.req.get_headers()
local keys = {}
for k, v in pairs(headers) do
if string.startswith(k, 'x-oss-') then
-- client must assemble the same header keys
if type(v) ~= 'string' then return nil end
table.insert(keys, k)
end
end
-- sorted in alphabetical order
table.sort(keys)
for i, key in ipairs(keys) do
keys[i] = key .. ':' .. headers[key] .. '\n'
end
return table.concat(keys)
end
local function calc_sign(key, method, md5, type_, date, oss_headers, resource)
-- string_to_sign:
-- method + '\n' + content_md5 + '\n' + content_type + '\n'
-- + date + '\n' + canonicalized_oss_headers + canonicalized_resource
local sign_str = string.format('%s\n%s\n%s\n%s\n%s%s',
method, md5, type_,
date, oss_headers, resource)
ngx.log(ngx.ERR, "SignStr:", sign_str, "\n")
local sign_result = ngx.encode_base64(ngx.hmac_sha1(key, sign_str))
return sign_result, sign_str
end
local function oss_auth()
-- ngx.log(ngx.INFO, 'auth')
--local method = ngx.var.request_method
local method = ngx.req.get_method()
local content_md5 = ngx.var.http_content_md5 or ''
local content_type = ngx.var.http_content_type or ''
-- get date
local date = ngx.var.http_x_oss_date or ngx.var.http_date or ''
if date == '' then
date = ngx.http_time(ngx.time())
-- ngx.log(ngx.INFO, 'Date:', date)
ngx.req.set_header('Date', date)
end
local resource = get_canon_resource()
local canon_headers = get_canon_headers()
local sign_result, sign_str = calc_sign(ngx.var.oss_auth_key, method, content_md5,
content_type, date, canon_headers, resource)
-- ngx.log(ngx.INFO, 'sign string:', sign_str)
-- ngx.log(ngx.INFO, 'sign string len:', string.len(sign_str))
local auth = string.format("OSS %s:%s", ngx.var.oss_auth_id, sign_result)
ngx.req.set_header('Authorization', auth)
ngx.exec("@oss")
end
-- main
res = oss_auth()
if res then
ngx.exit(res)
end
step2：编写nginx的主配置文件
cat 1.nginx.conf
server {
listen 80;
location / {
set $oss_bucket "bucketname"; #oss的桶名
set $oss_auth_id "xxxxxxxxxxx"; #oss的access_key
set $oss_auth_key "xxxxxxxxxxxxxxxxxx"; #oss的secret_key
rewrite_by_lua_file /opt/oss_auth.lua; #oss_auth.lua的文件路径
}
location @oss {
proxy_pass http://my.oss-ap-hangzhou.aliyuncs.com; #真实的oss访问地址
}
}
step3：编写nginx主配置文件的secret文件，需要将nginx.conf以secert的方式挂载到容器中
cat 2.nginx-conf-secrets.yaml
apiVersion: v1
data:
#nginx.conf通过base64编码生成的字符串
oss-proxy-lua.conf: c2VydmVyIHsKICAgIGxpc3RdsvsdiA4MDsdvsdKICAgIGxvY2F0aW9uIC8gewogICAgICAgIHNldCAkb3NzX2J1Y2tldCAiZGlwYml0LWRlcGxveS1kZXYiOwogICAgICAgIHNldCAkb3NzX2F1dGhfaWQgIkxUQUlmN1NON0VPaVRzcHciOwogICAgICAgIHNldCAkb3NzX2F1dGhfa2V5ICI1ZUFDZ0wyQlVwb1p3SzBkRUQ3NkRNZkxyMHJFaUgiOwogICAgICAgIHJld3JpdGVfYnlfbHVhX2ZpbGUgL29wdC9vc3NfYXV0aC5sdWE7CiAgICB9CiAgICBsb2NhdGlvbiBAb3NzIHsKICAgICAgICBwcm94eV9wYXNzIGh0dHA6Ly9kaXBiaXQtZGVwbG95LWRldi5vc3MtYXAtc291dGhlYXN0LTEuYWxpeXVuY3MuY29tOwogICAgfQp9Cg==
kind: Secret
metadata:
name: nginx-conf-secret
namespace: default
type: Opaque #Opaque是base64 编码格式的 Secret，用来存储密码、密钥等
step4：编写oss_auth.lua的comfigmap文件，需要将oss_auth.lua以comfigmap的方式挂载到容器中
cat 3.nginx-lua-file-configmap.yaml
apiVersion: v1
data:
oss_auth.lua: |-
-- has been sorted in alphabetical order
local signed_subresources = {
'acl',
'append',
'bucketInfo',
'cname',
'commitTransition',
'comp',
'cors',
'delete',
'lifecycle',
'location',
'logging',
'mime',
'notification',
'objectInfo',
'objectMeta',
'partData',
'partInfo',
'partNumber',
'policy',
'position',
'referer',
'replication',
'replicationLocation',
'replicationProgress',
'requestPayment',
'response-cache-control',
'response-content-disposition',
'response-content-encoding',
'response-content-language',
'response-content-type',
'response-expires',
'restore',
'security-token',
'tagging',
'torrent',
'uploadId',
'uploads',
'versionId',
'versioning',
'versions',
'website'
}
function string.startswith(s, start)
return string.sub(s, 1, string.len(start)) == start
end
local function get_canon_sub_resource()
local args = ngx.req.get_uri_args()
-- lower keys
local keys = {}
for k, v in pairs(args) do
keys[k:lower()] = v
end
-- make resource string
local s = ''
local sep = '?'
for i, k in ipairs(signed_subresources) do
v = keys[k]
if v then
-- sub table
v = type(v) == 'table' and v[1] or v
s = s .. string.format("%s%s=%s", sep, k, v)
sep = '&'
end
end
return s
end
local function get_canon_resource()
resource = ''
object = ngx.unescape_uri(ngx.var.uri)
sub = get_canon_sub_resource()
return string.format("/%s%s%s", ngx.var.oss_bucket, object, sub)
end
local function get_canon_headers()
-- default: <lowerkey, value>
local headers = ngx.req.get_headers()
local keys = {}
for k, v in pairs(headers) do
if string.startswith(k, 'x-oss-') then
-- client must assemble the same header keys
if type(v) ~= 'string' then return nil end
table.insert(keys, k)
end
end
-- sorted in alphabetical order
table.sort(keys)
for i, key in ipairs(keys) do
keys[i] = key .. ':' .. headers[key] .. '\n'
end
return table.concat(keys)
end
local function calc_sign(key, method, md5, type_, date, oss_headers, resource)
-- string_to_sign:
-- method + '\n' + content_md5 + '\n' + content_type + '\n'
-- + date + '\n' + canonicalized_oss_headers + canonicalized_resource
local sign_str = string.format('%s\n%s\n%s\n%s\n%s%s',
method, md5, type_,
date, oss_headers, resource)
ngx.log(ngx.ERR, "SignStr:", sign_str, "\n")
local sign_result = ngx.encode_base64(ngx.hmac_sha1(key, sign_str))
return sign_result, sign_str
end
local function oss_auth()
-- ngx.log(ngx.INFO, 'auth')
--local method = ngx.var.request_method
local method = ngx.req.get_method()
local content_md5 = ngx.var.http_content_md5 or ''
local content_type = ngx.var.http_content_type or ''
-- get date
local date = ngx.var.http_x_oss_date or ngx.var.http_date or ''
if date == '' then
date = ngx.http_time(ngx.time())
-- ngx.log(ngx.INFO, 'Date:', date)
ngx.req.set_header('Date', date)
end
local resource = get_canon_resource()
local canon_headers = get_canon_headers()
local sign_result, sign_str = calc_sign(ngx.var.oss_auth_key, method, content_md5,
content_type, date, canon_headers, resource)
-- ngx.log(ngx.INFO, 'sign string:', sign_str)
-- ngx.log(ngx.INFO, 'sign string len:', string.len(sign_str))
local auth = string.format("OSS %s:%s", ngx.var.oss_auth_id, sign_result)
ngx.req.set_header('Authorization', auth)
ngx.exec("@oss")
end
-- main
res = oss_auth()
if res then
ngx.exit(res)
end
kind: ConfigMap
metadata:
name: nginx-lua-configmap
namespace: default
step5：编写部署服务的文件
cat 4.oss-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "7"
labels:
app: aliyun-oss
name: aliyun-oss
namespace: default
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: aliyun-oss
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
labels:
app: aliyun-oss
spec:
containers:
- env:
- name: LUA_PATH
value: /opt;;
image: registry.ap-southeast-1.aliyuncs.com/mynamespace:openresty-1.13.6.2-0-centos-rpm
imagePullPolicy: IfNotPresent
name: aliyun-oss
ports:
- containerPort: 80
name: aliyun-oss
protocol: TCP
resources:
requests:
cpu: 250m
memory: 256Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /opt
name: nginx-lua-file #oss_auth.lua文件的挂载路径
- mountPath: /etc/nginx/conf.d
name: nginx-conf-file #nginx.conf的挂载路径
- mountPath: /etc/localtime
name: container-localtime #容器的市区
dnsPolicy: ClusterFirst
imagePullSecrets:
- name: 41c11f57a260c99d30449e6d31b77e77 #拉取的密钥
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
volumes:
- configMap: #以confimap的方式挂载到容器中
defaultMode: 420
name: nginx-lua-configmap #oss_auth.lua存放在nginx-lua-configmap中
name: nginx-lua-file
- secret: #以secret的方式挂载到容器中
defaultMode: 420
secretName: nginx-conf-secret #nginx.conf存放在nginx-conf-secret中
name: nginx-conf-file
- hostPath: #以hostpath的方式挂载到容器中
path: /etc/localtime
type: ""
name: container-localtime
step6：编写部署服务的svc文件
cat 5.oss-service.yaml
apiVersion: v1
kind: Service
metadata:
name: oss-svc
namespace: default
spec:
clusterIP: None
ports:
- name: oss-proxy
port: 80
protocol: TCP
targetPort: 80
selector:
app: aliyun-oss
step7：执行部署命令
kubectl apply -f 2.nginx-conf-secrets.yaml
kubectl apply -f 3.nginx-lua-file-configmap.yam
kubectl apply -f 4.oss-deployment.yaml
kubectl apply -f 5.oss-service.yaml
step8：验证
都到这儿了，更多文章，详见个人微信公众号ALL In Linux，来扫一扫吧！
