原因:2017年4月14日 星期五 学习记录。 说明:整理ceph资料。
《Ceph From Scratch》
一书。使用Docker
docker images
docker pull ubuntu
docker run -i -t ubuntu /bin/bash
docker exec -i -t ubuntu /bin/bash
docker ps
Ceph容器
ifconfig
或者ip addr
,注意IP与使用网卡的对应关系。启动Ceph
docker@dev:~$ docker run -d --net=host -e MON_IP=10.0.2.15 -e CEPH_NETWORK=10.0.2.0/24 ceph/demo
badaf5c8fed1e0edf6f2281539669d8f6522ba54b625076190fe4d6de79745ff
docker@dev:~$ docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
badaf5c8fed1 ceph/demo "/entrypoint.sh" 9 seconds ago Up 9 seconds loving_pasteur
docker@dev:~$ docker exec -i -t badaf5c8fed1 /bin/bash
root@dev:/#
检查状态
ceph -w
,也就是watch整个ceph集群的状态。root@dev:/# ceph -w
cluster fee30c76-aec4-44d4-8138-763969aaa562
health HEALTH_OK
monmap e1: 1 mons at {dev=10.0.2.15:6789/0}
election epoch 2, quorum 0 dev
mdsmap e5: 1/1/1 up {0=0=up:active}
osdmap e18: 1 osds: 1 up, 1 in
pgmap v22: 120 pgs, 8 pools, 2810 bytes data, 63 objects
4519 MB used, 13115 MB / 18603 MB avail
120 active+clean
2015-07-12 06:53:58.454077 mon.0 [INF] pgmap v22: 120 pgs: 120 active+clean; 2810 bytes data, 4519 MB used, 13115 MB / 18603 MB avail
root@dev:/# ceph status
cluster fee30c76-aec4-44d4-8138-763969aaa562
health HEALTH_OK
monmap e1: 1 mons at {dev=10.0.2.15:6789/0}
election epoch 2, quorum 0 dev
mdsmap e5: 1/1/1 up {0=0=up:active}
osdmap e21: 1 osds: 1 up, 1 in
pgmap v30: 120 pgs, 8 pools, 2810 bytes data, 63 objects
4521 MB used, 13114 MB / 18603 MB avail
120 active+clean
Pool简介
Pool相关命令
root@dev:/# rados lspools
rbd
cephfs_data
cephfs_metadata
.rgw.root
.rgw.control
.rgw
.rgw.gc
.users.uid
root@dev:/# rados -p .rgw ls
root@dev:/# rados -p .rgw.root ls
default.region
region_info.default
zone_info.default
容量相关
root@dev:/# rados df
pool name KB objects clones degraded unfound rd rd KB wr wr KB
.rgw 0 0 0 0 0 0 0 0 0
.rgw.control 0 8 0 0 0 0 0 0 0
.rgw.gc 0 32 0 0 0 288 256 192 0
.rgw.root 1 3 0 0 0 0 0 3 3
.users.uid 0 0 0 0 0 0 0 0 0
cephfs_data 0 0 0 0 0 0 0 0 0
cephfs_metadata 2 20 0 0 0 0 0 31 8
rbd 0 0 0 0 0 0 0 0 0
total used 4630192 63
total avail 13428976
total space 19049892
创建Bucket
root@dev:/# ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-1 1.00000 root default
-2 1.00000 host dev
0 1.00000 osd.0 up 1.00000 1.00000
root@dev:/# ceph osd crush add-bucket rack01 rack
added bucket rack01 type rack to crush map
root@dev:/# ceph osd crush add-bucket rack02 rack
added bucket rack02 type rack to crush map
root@dev:/# ceph osd crush add-bucket rack03 rack
added bucket rack03 type rack to crush map
root@dev:/# ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-5 0 rack rack03
-4 0 rack rack02
-3 0 rack rack01
-1 1.00000 root default
-2 1.00000 host dev
0 1.00000 osd.0 up 1.00000 1.00000
移动Rack
root@dev:/# ceph osd crush move rack01 root=default
moved item id -3 name 'rack01' to location {root=default} in crush map
root@dev:/# ceph osd crush move rack02 root=default
moved item id -4 name 'rack02' to location {root=default} in crush map
root@dev:/# ceph osd crush move rack03 root=default
moved item id -5 name 'rack03' to location {root=default} in crush map
root@dev:/# ceph osd tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-1 1.00000 root default
-2 1.00000 host dev
0 1.00000 osd.0 up 1.00000 1.00000
-3 0 rack rack01
-4 0 rack rack02
-5 0 rack rack03
创建Pool
root@dev:/# ceph osd pool create web-services 128 128
pool 'web-services' created
root@dev:/# rados lspools
rbd
cephfs_data
cephfs_metadata
.rgw.root
.rgw.control
.rgw
.rgw.gc
.users.uid
web-services
添加Object
root@dev:/# echo "Hello Ceph, You are Awesome like MJ" > /tmp/helloceph
root@dev:/# rados -p web-services put object1 /tmp/helloceph
root@dev:/# rados -p web-services ls
object1
root@dev:/# ceph osd map web-services object1
osdmap e29 pool 'web-services' (8) object 'object1' -> pg 8.bac5debc (8.3c) -> up ([0], p0) acting ([0], p0)
查看Object
root@dev:/# cd /var/lib/ceph/osd/
root@dev:/var/lib/ceph/osd# ls ceph-0/current/8.3c_head/
__head_0000003C__8 object1__head_BAC5DEBC__8
root@dev:/var/lib/ceph/osd# cat ceph-0/current/8.3c_head/object1__head_BAC5DEBC__8
Hello Ceph, You are Awesome like MJ
简介
长论文
CRUSH论文
RADOS
CephFS论文
CRUSH简介
Controlled Replication Under Scalable Hashing
,是一种数据分发算法,类似于哈希和一致性哈希。哈希的问题在于数据增长时不能动态加Bucket,一致性哈希的问题在于增加Bucket时数据迁移量比较大,其他数据分发算法依赖中心的Metadata服务器来存储元数据效率较低,CRUSH则是通过计算、接受多维参数的来解决动态数据分发的场景。算法基础
算法解读
- 图虽然很复杂,但如果理解了几个基本操作的含义就很好读下来了,这里是三个操作的伪代码,take和emit很好理解,select主要是遍历当前bucket,如果出现重复、失败或者超载就跳过,其中稍微复杂的“first n”部分是一旦遇到失败,第一种情况是直接使用多备份,第二种情况是使用erasing code基本可以忽略。看着下面的图就更好理解具体的算法了
算法总结
算法补充
检查Pool
rados lspools
ceph osd pool create rbd_pool 1024
创建image
rbd_directory
的object。rbd create test_image --size 1024
rbd ls
rbd --image test_image info
rados -p rbd ls
修改image大小
rbd --image test_image resize --size 2000
rbd --image test_image resize --size 1000 --allow-shrink
使用Image
rbd map test_image
rbd showmapped
mkfs.ext4 /dev/rbd0
mount /dev/rbd0 /mnt/
移除Image
umount /dev/rbd0
rbd unmap /dev/rbd0
rbd showmapped
删除Image
rbd --image test_image rm
创建快照
rbd snap create --image test_image --snap test_snap
rbd snap ls --image test_image
快照回滚
rbd snap rollback --image test_image --snap test_snap
rbd snap rollback rbd/test_image@test_snap
删除快照
rbd snap rm --image test_image --snap test_snap
rbd snap purge --image test_image
保护快照
rbd snap protect --image test_image --snap test_snap
rbd snap unprotect --image test_image --snap test_snap
创建clone
rbd create test_image2 --size 1024 --image-format 2
rbd snap create --image test_image2 --snap test_snap2
rbd snap protect --image test_image2 --snap test_snap2
rbd clone --image test_image2 --snap test_snap2 test_clone
列举clone
rbd children --image test_image2 --snap test_snap2
填充克隆
rbd flatten --image test_clone
使用Qemu
apt-get install -y qemu-utils
创建镜像
qemu-img create -f raw rbd:rbd/test_image3 1G
修改镜像大小
qemu-img resize rbd:rbd/test_image3 2G
查看镜像信息
qemu-img info rbd:rbd/test_image3
安装Virsh
apt-get install -y virt-manager
OpenStack介绍
apt-get install -y python-ceph
Nova与RBD
libvirt_images_type=rbd
libvirt_images_rbd_pool=volumes
libvirt_images_rbd_ceph_conf=/etc/ceph/ceph.conf
rbd_user=cinder
rbd_secret_uuid=457eb676-33da-42ec-9a8c-9293d545c337
libvirt_inject_password=false
libvirt_inject_key=false
libvirt_inject_partition=-2
Glance与RBD
default_store=rbd
rbd_store_user=glance
rbd_store_pool=images
show_image_direct_url=True
Cinder与RBD
volume_driver=cinder.volume.drivers.rbd.RBDDriver
rbd_pool=volumes
rbd_ceph_conf=/etc/ceph/ceph.conf
rbd_flatten_volume_from_snapshot=false
rbd_max_clone_depth=5
glance_api_version=2
backup_driver=cinder.backup.drivers.ceph
backup_ceph_conf=/etc/ceph/ceph.conf
backup_ceph_user=cinder-backup
backup_ceph_chunk_size=134217728
backup_ceph_pool=backups
backup_ceph_stripe_unit=0
backup_ceph_stripe_count=0
restore_discard_excess_bytes=true
安装librbd
apt-get install -y python-ceph
创建Image
import rados
import rbd
cluster = rados.Rados(conffile='/etc/ceph/ceph.conf')
cluster.connect()
ioctx = cluster.open_ioctx('rbd')
rbd_inst = rbd.RBD()
size = 1024**3
image_name = "test_image"
rbd_inst.create(ioctx, image_name, size)
image = rbd.Image(ioctx, image_name)
data = 'foo' * 200
image.write(data, 0)
image.close()
ioctx.close()
cluster.shutdown()
cluster = rados.Rados(conffile='/etc/ceph/ceph.conf')
try:
ioctx = cluster.open_ioctx('rbd')
try:
rbd_inst = rbd.RBD()
size = 1024**3
image_name = "test_image"
rbd_inst.create(ioctx, image_name, size)
image = rbd.Image(ioctx, image_name)
try:
data = 'foo' * 200
image.write(data, 0)
finally:
image.close()
finally:
ioctx.close()
finally:
cluster.shutdown()
with rados.Rados(conffile='/etc/ceph/ceph.conf') as cluster:
with cluster.open_ioctx('rbd') as ioctx:
rbd_inst = rbd.RBD()
size = 1024**3
image_name = "test_image"
rbd_inst.create(ioctx, image_name, size)
with rbd.Image(ioctx, image_name) as image:
data = 'foo' * 200
image.write(data, 0)
使用RGW
docker run -d --net=host -e MON_IP=10.251.0.105 -e CEPH_NETWORK=10.251.0.0/24 ceph/demo
用户操作
radosgw-admin user info --uid=mona
Bucket操作
root@dev:~# radosgw-admin bucket stats
[]
创建用户
root@dev:/# radosgw-admin user create --uid=mona --display-name="Monika Singh" --email=mona@example.com
{
"user_id": "mona",
"display_name": "Monika Singh",
"email": "mona@example.com",
"suspended": 0,
"max_buckets": 1000,
"auid": 0,
"subusers": [],
"keys": [
{
"user": "mona",
"access_key": "2196PJ0MA6FLHCVKVFDW",
"secret_key": "eO1\/dmEOEU5LlooexlWwcqJYZrt3Gzp\/nBXsQCwz"
}
],
"swift_keys": [],
"caps": [],
"op_mask": "read, write, delete",
"default_placement": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"temp_url_keys": []
}
添加Capabilities
radosgw-admin caps add --uid=mona --caps="users=*"
radosgw-admin caps add --uid=mona --caps="buckets=*"
radosgw-admin caps add --uid=mona --caps="metadata=*"
radosgw-admin caps add --uid=mona --caps="zone=*"
安装s3cmd
apt-get install python-setuptools
git clone https://github.com/s3tools/s3cmd.git
cd s3cmd/
python setup.py install
使用s3cmd
s3cmd ls
创建用户
root@dev:~# radosgw-admin subuser create --uid=mona --subuser=mona:swift --access=full --secret=secretkey --key-type=swift
{
"user_id": "mona",
"display_name": "Monika Singh",
"email": "mona@example.com",
"suspended": 0,
"max_buckets": 1000,
"auid": 0,
"subusers": [
{
"id": "mona:swift",
"permissions": "<none>"
}
],
"keys": [
{
"user": "mona",
"access_key": "2196PJ0MA6FLHCVKVFDW",
"secret_key": "eO1\/dmEOEU5LlooexlWwcqJYZrt3Gzp\/nBXsQCwz"
},
{
"user": "mona:swift",
"access_key": "2FTDLNGGOWALF1ZS5XHY",
"secret_key": ""
}
],
"swift_keys": [
{
"user": "mona:swift",
"secret_key": "secretkey"
}
],
"caps": [
{
"type": "buckets",
"perm": "*"
},
{
"type": "metadata",
"perm": "*"
},
{
"type": "users",
"perm": "*"
},
{
"type": "zone",
"perm": "*"
}
],
"op_mask": "read, write, delete",
"default_placement": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"max_size_kb": -1,
"max_objects": -1
},
"temp_url_keys": []
}
安装Swift客户端
apt-get install python-pip
pip install python-swiftclient
Swift命令
swift -V 1.0 -A http://localhost/auth -U mona:swift -K secretkey post example-bucket
swift -V 1.0 -A http://localhost/auth -U mona:swift -K secretkey list
简介
启动API
docker run -d --net=host -e MON_IP=10.0.2.15 -e CEPH_NETWORK=10.0.2.0/24 ceph/demo
ceph-rest-api -n client.admin
测试API
root@dev:/# curl 127.0.0.1:5000/api/v0.1/health
HEALTH_OK
root@dev:/# curl 127.0.0.1:5000/api/v0.1/osd/tree
ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY
-1 1.00000 root default
-2 1.00000 host dev
0 1.00000 osd.0 up 1.00000 1.00000
-3 0 rack rack01
-4 0 rack rack02
-5 0 rack rack03
监控工具
Ceph-web
docker run -d --net=host tobegit3hub/ceph-web