-
Nova 配置
disk_cachemodes = "network=writeback” (启用)
改为 disk_cachemodes = “network=none"（关闭） -
Ceph 配置
-
开启 ceph rbd cache
client]
rbd_cache = true
rbd_cache_writethrough_until_flush = true
admin_socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
log_file = /var/log/qemu/qemu-guest-$pid.log
rbd_concurrent_management_ops = 20 - 关闭ceph rbd cache
[client]
#rbd_cache = true
#rbd_cache_writethrough_until_flush = true
admin_socket = /var/run/ceph/guests/$cluster-$type.$id.$pid.$cctid.asok
#log_file = /var/log/qemu/qemu-guest-$pid.log
#rbd_concurrent_management_ops = 20
-
-
验证cache是否生效
- 验证 生效命令
[root@l2-4-30 guests]# ceph --admin-daemo ceph-client.cinder.27346.94699498717184.asok perf dump
{
"finisher-librbd::TaskFinisher::m_finisher": {
"queue_len": 0,
"complete_latency": {
"avgcount": 0,
"sum": 0.000000000
}
},
"librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": {
"rd": 32853,
"rd_bytes": 2060131840,
"rd_latency": {
"avgcount": 32853,
"sum": 42.348302430
},
"wr": 0,
"wr_bytes": 0,
"wr_latency": {
"avgcount": 0,
"sum": 0.000000000
},
"discard": 0,
"discard_bytes": 0,
"discard_latency": {
"avgcount": 0,
"sum": 0.000000000
},
"flush": 0,
"aio_flush": 0,
"aio_flush_latency": {
"avgcount": 0,
"sum": 0.000000000
},
"snap_create": 0,
"snap_remove": 0,
"snap_rollback": 0,
"snap_rename": 0,
"notify": 0,
"resize": 0,
"readahead": 103,
"readahead_bytes": 27451904,
"invalidate_cache": 0
},
"librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
"rd": 40116,
"rd_bytes": 650777600,
"rd_latency": {
"avgcount": 40116,
"sum": 38.547805260
},
"wr": 7348,
"wr_bytes": 253340672,
"wr_latency": {
"avgcount": 7348,
"sum": 55.895274365
},
"discard": 0,
"discard_bytes": 0,
"discard_latency": {
"avgcount": 0,
"sum": 0.000000000
},
"flush": 0,
"aio_flush": 434,
"aio_flush_latency": {
"avgcount": 434,
"sum": 0.540130077
},
"snap_create": 0,
"snap_remove": 0,
"snap_rollback": 0,
"snap_rename": 0,
"notify": 0,
"resize": 0,
"readahead": 130,
"readahead_bytes": 37353472,
"invalidate_cache": 0
},
"objectcacher-librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": {
"cache_ops_hit": 2761,
"cache_ops_miss": 30195,
"cache_bytes_hit": 1500713472,
"cache_bytes_miss": 2063611392,
"data_read": 2087583744,
"data_written": 0,
"data_flushed": 0,
"data_overwritten_while_flushing": 0,
"write_ops_blocked": 0,
"write_bytes_blocked": 0,
"write_time_blocked": 0.000000000
},
"objectcacher-librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
"cache_ops_hit": 829,
"cache_ops_miss": 7979,
"cache_bytes_hit": 191516672,
"cache_bytes_miss": 153311232,
"data_read": 168047616,
"data_written": 253340672,
"data_flushed": 253340672,
"data_overwritten_while_flushing": 0,
"write_ops_blocked": 0,
"write_bytes_blocked": 0,
"write_time_blocked": 0.000000000
},
"objecter": {
"op_active": 0,
"op_laggy": 0,
"op_send": 45229,
"op_send_bytes": 0,
"op_resend": 0,
"op_ack": 45105,
"op_commit": 7893,
"op": 45229,
"op_r": 37337,
"op_w": 7892,
"op_rmw": 0,
"op_pg": 0,
"osdop_stat": 7045,
"osdop_create": 0,
"osdop_read": 37319,
"osdop_write": 7519,
"osdop_writefull": 0,
"osdop_append": 0,
"osdop_zero": 0,
"osdop_truncate": 0,
"osdop_delete": 0,
"osdop_mapext": 0,
"osdop_sparse_read": 0,
"osdop_clonerange": 0,
"osdop_getxattr": 0,
"osdop_setxattr": 0,
"osdop_cmpxattr": 0,
"osdop_rmxattr": 0,
"osdop_resetxattrs": 0,
"osdop_tmap_up": 0,
"osdop_tmap_put": 0,
"osdop_tmap_get": 0,
"osdop_call": 1117,
"osdop_watch": 122,
"osdop_notify": 1,
"osdop_src_cmpxattr": 0,
"osdop_pgls": 0,
"osdop_pgls_filter": 0,
"osdop_other": 1,
"linger_active": 1,
"linger_send": 2,
"linger_resend": 0,
"linger_ping": 121,
"poolop_active": 0,
"poolop_send": 0,
"poolop_resend": 0,
"poolstat_active": 0,
"poolstat_send": 0,
"poolstat_resend": 0,
"statfs_active": 0,
"statfs_send": 0,
"statfs_resend": 0,
"command_active": 0,
"command_send": 0,
"command_resend": 0,
"map_epoch": 0,
"map_full": 0,
"map_inc": 0,
"osd_sessions": 27966,
"osd_session_open": 236,
"osd_session_close": 0,
"osd_laggy": 1,
"omap_wr": 0,
"omap_rd": 0,
"omap_del": 0
},
"throttle-msgr_dispatch_throttler-radosclient": {
"val": 0,
"max": 104857600,
"get": 52511,
"get_sum": 2152111179,
"get_or_fail_fail": 0,
"get_or_fail_success": 0,
"take": 0,
"take_sum": 0,
"put": 52511,
"put_sum": 2152111179,
"wait": {
"avgcount": 0,
"sum": 0.000000000
}
},
"throttle-objecter_bytes": {
"val": 0,
"max": 104857600,
"get": 45108,
"get_sum": 2419831296,
"get_or_fail_fail": 0,
"get_or_fail_success": 45108,
"take": 0,
"take_sum": 0,
"put": 44725,
"put_sum": 2419831296,
"wait": {
"avgcount": 0,
"sum": 0.000000000
}
},
"throttle-objecter_ops": {
"val": 0,
"max": 1024,
"get": 45108,
"get_sum": 45108,
"get_or_fail_fail": 0,
"get_or_fail_success": 45108,
"take": 0,
"take_sum": 0,
"put": 45108,
"put_sum": 45108,
"wait": {
"avgcount": 0,
"sum": 0.000000000
}
}
} 完整配置
过滤 cache 字符串
[root@l2-4-30 guests]# ceph --admin-daemo ceph-client.cinder.27346.94699498717184.asok perf dump |grep cache -A 5 -B 5 "snap_rename": 0, "notify": 0, "resize": 0, "readahead": 103, "readahead_bytes": 27451904, "invalidate_cache": 0 }, "librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": { "rd": 57149, "rd_bytes": 1148467200, "rd_latency": {--
"snap_rename": 0,
"notify": 0,
"resize": 0,
"readahead": 130,
"readahead_bytes": 37353472,
"invalidate_cache": 0
},
"objectcacher-librbd-1373b25a3be0cd-glance_images-86e30f80-5a93-47ee-88c9-de5c1db6df10": {
"cache_ops_hit": 2791,
"cache_ops_miss": 45597,
"cache_bytes_hit": 1526398976,
"cache_bytes_miss": 3029461504,
"data_read": 3057748992,
"data_written": 0,
"data_flushed": 0,
"data_overwritten_while_flushing": 0,
"write_ops_blocked": 0,
"write_bytes_blocked": 0,
"write_time_blocked": 0.000000000
},
"objectcacher-librbd-bf1708496d088e-nova_instances-555153af-f088-4aec-9bae-e4d6389b4d98_disk": {
"cache_ops_hit": 886,
"cache_ops_miss": 9957,
"cache_bytes_hit": 248139264,
"cache_bytes_miss": 225620480,
"data_read": 240729088,
"data_written": 462767104,
"data_flushed": 462767104,
"data_overwritten_while_flushing": 0,
"write_ops_blocked": 0,-
验证 无cache命令
[root@l23-4-40 guests]# ceph --admin-daemo ceph-client.cinder.138869.94547912278016.asok perf dump |grep cache -A 5 -B 5
"snap_rename": 0,
"notify": 0,
"resize": 0,
"readahead": 0,
"readahead_bytes": 0,
"invalidate_cache": 0
},
"librbd-bee65365887c0b-nova_instances-5cf85c18-6725-4674-88e4-60408c97f487_disk": {
"rd": 62937,
"rd_bytes": 1261381632,
"rd_latency": {"snap_rename": 0, "notify": 0, "resize": 0, "readahead": 0, "readahead_bytes": 0, "invalidate_cache": 0 }, "objecter": { "op_active": 0, "op_laggy": 0, "op_send": 95327 }
- 验证 生效命令
