注意点

虽然部署的mongodb的版本是4.1.3，但是它使用的配置和命令都是4.2版本的（→_→）
最明显的改变时ssl的配置和命令都改成了tls

环境

ip	端口	角色
192.168.43.111	[27001,27002.27003]	shardsvr
192.168.43.112	[27001,27002.27003]	shardsvr
192.168.43.113	[27001,27002.27003]	shardsvr
192.168.43.111	[27010,27020]	configsvr,mongos
192.168.43.112	[27010,27020]	configsvr,mongos
192.168.43.113	[27010,27020]	configsvr,mongos

建立相关目录和配置文件

执行命令的服务器192.168.43.111，192.168.43.112，192.168.43.113

开启防火墙端口

firewall-cmd --zone=public --add-port=27001/tcp --permanent
firewall-cmd --zone=public --add-port=27002/tcp --permanent
firewall-cmd --zone=public --add-port=27003/tcp --permanent
firewall-cmd --zone=public --add-port=27010/tcp --permanent
firewall-cmd --zone=public --add-port=27020/tcp --permanent
systemctl restart firewalld.service

建立临时环境变量

# 副本集mgo01
export mgo01_path=/home/mongo/shardsvr/mgo01
export mgo01_replSetName=mgo01

export conf_dir=confdir

# 副本集mgo02
export mgo02_path=/home/mongo/shardsvr/mgo02
export mgo02_replSetName=mgo02

# 副本集mgo03
export mgo03_path=/home/mongo/shardsvr/mgo03
export mgo03_replSetName=mgo03

# 配置服务器
export configsvr_path=/home/mongo/configsvr
export conf_replSetName=cfgReplSet

# mongos
export mongos_path=/home/mongo/mongos

# x.509证书文件存放位置
export keyfile=/home/mongo/keyfile

# docker-compose.yml文件位置
export docker_path=/home/mongo

建立目录

# 副本集mgo01
mkdir -p ${mgo01_path}/${conf_dir}
mkdir -p ${mgo01_path}/${conf_dir}/log 
mkdir -p ${mgo01_path}/${conf_dir}/pid
# 设置目录权限
chmod 777 ${mgo01_path}/${conf_dir}/log ${mgo01_path}/${conf_dir}/pid

# 副本集mgo02
mkdir -p ${mgo02_path}/${conf_dir}
mkdir -p ${mgo02_path}/${conf_dir}/log 
mkdir -p ${mgo02_path}/${conf_dir}/pid
chmod 777 ${mgo02_path}/${conf_dir}/log ${mgo02_path}/${conf_dir}/pid

# 副本集mgo03
mkdir -p ${mgo03_path}/${conf_dir}
mkdir -p ${mgo03_path}/${conf_dir}/log 
mkdir -p ${mgo03_path}/${conf_dir}/pid
chmod 777 ${mgo03_path}/${conf_dir}/log ${mgo03_path}/${conf_dir}/pid

# mongos
mkdir -p ${mongos_path}/${conf_dir}
mkdir -p ${mongos_path}/${conf_dir}/log
mkdir -p ${mongos_path}/${conf_dir}/pid
chmod 777 ${mongos_path}/${conf_dir}/log ${mongos_path}/${conf_dir}/pid

# 证书文件
mkdir -p ${keyfile}

mkdir -p ${docker_path}

# 配置服务
mkdir -p ${configsvr_path}/${conf_dir}
mkdir -p ${configsvr_path}/${conf_dir}/log
mkdir -p ${configsvr_path}/${conf_dir}/pid
chmod 777 ${configsvr_path}/${conf_dir}/log ${configsvr_path}/${conf_dir}/pid

建立配置文件

shardsvr：mgo01

cat <<EOF> ${mgo01_path}/${conf_dir}/mgo.conf
systemLog:
   path: /data/${conf_dir}/log/mgo.log
   destination: file

storage:
  dbPath: /data/db
  journal:
    enabled: true
  directoryPerDB: true

security:
   keyFile: /keyfile/mgo.pem
   clusterAuthMode: x509
   authorization: enabled

processManagement:
  pidFilePath: /data/${conf_dir}/pid/mgo.pid

net:
  bindIpAll: true
  tls:
    CAFile: /keyfile/ca.pem
    mode: requireTLS
    certificateKeyFile: /keyfile/mgo.pem

replication:
  replSetName: ${mgo01_replSetName}

sharding:
  clusterRole: shardsvr
EOF

shardsvr：mgo02

cat <<EOF> ${mgo02_path}/${conf_dir}/mgo.conf
systemLog:
   path: /data/${conf_dir}/log/mgo.log
   destination: file

storage:
  dbPath: /data/db
  journal:
    enabled: true
  directoryPerDB: true

security:
   keyFile: /keyfile/mgo.pem
   clusterAuthMode: x509
   authorization: enabled

processManagement:
  pidFilePath: /data/${conf_dir}/pid/mgo.pid

net:
  bindIpAll: true
  tls:
    CAFile: /keyfile/ca.pem
    mode: requireTLS
    certificateKeyFile: /keyfile/mgo.pem

replication:
  replSetName: ${mgo02_replSetName}

sharding:
  clusterRole: shardsvr
EOF

shardsvr：mgo03

cat <<EOF> ${mgo03_path}/${conf_dir}/mgo.conf
systemLog:
   path: /data/${conf_dir}/log/mgo.log
   destination: file

storage:
  dbPath: /data/db
  journal:
    enabled: true
  directoryPerDB: true

security:
   keyFile: /keyfile/mgo.pem
   clusterAuthMode: x509
   authorization: enabled

processManagement:
  pidFilePath: /data/${conf_dir}/pid/mgo.pid

net:
  bindIpAll: true
  tls:
    CAFile: /keyfile/ca.pem
    mode: requireTLS
    certificateKeyFile: /keyfile/mgo.pem 

replication:
  replSetName: ${mgo03_replSetName}

sharding:
  clusterRole: shardsvr
EOF

configsvr

cat <<EOF> ${configsvr_path}/${conf_dir}/mgo.conf
systemLog:
  path: /data/${conf_dir}/log/mgo.log
  destination: file

storage:
  dbPath: /data/db
  journal:
    enabled: true
  directoryPerDB: true

security:
   keyFile: /keyfile/mgo.pem
   clusterAuthMode: x509
   authorization: enabled

replication:
  replSetName: ${conf_replSetName}

processManagement:
  pidFilePath: /data/${conf_dir}/pid/mgo.pid

net:
  bindIpAll: true
  tls:
    CAFile: /keyfile/ca.pem
    mode: requireTLS
    certificateKeyFile: /keyfile/mgo.pem

sharding:
  clusterRole: configsvr
EOF

mongos

cat <<EOF> ${mongos_path}/${conf_dir}/mgo.conf
systemLog:
  path: /data/${conf_dir}/log/mgo.log
  destination: file

processManagement:
  pidFilePath: /data/${conf_dir}/pid/mgo.pid

security:
  keyFile: /keyfile/mgo.pem
  clusterAuthMode: x509
net:
  bindIpAll: true
  tls:
    CAFile: /keyfile/ca.pem
    mode: requireTLS
    certificateKeyFile: /keyfile/mgo.pem

sharding:
  configDB: ${conf_replSetName}/192.168.43.111:27010,192.168.43.112:27010,192.168.43.113:27010
EOF

docker-compose.yml

cat <<EOF> ${docker_path}/docker-compose.yml
version: '3'
services:
  mgo01:
    image: mongo:4.1
    volumes:
      - ${mgo01_path}:/data
      - ${keyfile}:/keyfile
      - /etc/localtime:/etc/localtime:ro
    ports:
      - "27001:27018"
    command: -f /data/${conf_dir}/mgo.conf

  mgo02:
    image: mongo:4.1
    volumes:
      - ${mgo02_path}:/data
      - ${keyfile}:/keyfile
      - /etc/localtime:/etc/localtime:ro
    ports:
      - "27002:27018"
    command: -f /data/${conf_dir}/mgo.conf
  
  mgo03:
    image: mongo:4.1
    volumes:
      - ${mgo03_path}:/data
      - ${keyfile}:/keyfile
    ports:
      - "27003:27018"
    command: -f /data/${conf_dir}/mgo.conf

  mgoConfig:
    image: mongo:4.1
    volumes:
      - ${configsvr_path}:/data
      - ${keyfile}:/keyfile
      - /etc/localtime:/etc/localtime:ro
    ports:
      - "27010:27019"
    command: -f /data/${conf_dir}/mgo.conf

  mongos:
    image: mongo:4.1
    volumes:
      - ${mongos_path}:/data
      - ${keyfile}:/keyfile
      - /etc/localtime:/etc/localtime:ro
    ports:
      - "27020:27017"
    entrypoint:
      "mongos"
    command:
      -f /data/${conf_dir}/mgo.conf 
EOF

创建x.509证书文件(只要一台服务器执行）

本例使用的是192.168.43.113服务器

mkdir -p /home/mgo_keyfile
cd /home/mgo_keyfile

openssl req -passout pass:password -new -x509 -keyout ca_p.pem -out ca.pem -subj "/CN=CHC/OU=MGO/O=MongoDB/L=NY/ST=NY/C=CN"

# 192.168.43.111的证书
openssl req -newkey rsa:2048 -nodes -out mgo-111.csr -keyout mgo-111.key -subj '/CN=192.168.43.111/OU=MGO/O=MongoDB/L=NY/ST=NY/C=CN'

openssl x509 -passin pass:password -days 3650 -sha256 -req -in mgo-111.csr -signkey mgo-111.key -CA ca.pem -CAkey ca_p.pem -CAcreateserial -out mgo-111.crt

cat mgo-111.crt mgo-111.key > mgo-111.pem

# 192.168.43.112的证书
openssl req -newkey rsa:2048 -nodes -out mgo-112.csr -keyout mgo-112.key -subj '/CN=192.168.43.112/OU=MGO/O=MongoDB/L=NY/ST=NY/C=CN'

openssl x509 -passin pass:password -days 3650 -sha256 -req -in mgo-112.csr -signkey mgo-112.key -CA ca.pem -CAkey ca_p.pem -CAcreateserial -out mgo-112.crt

cat mgo-112.crt mgo-112.key > mgo-112.pem

# 192.168.43.113的证书
openssl req -newkey rsa:2048 -nodes -out mgo-113.csr -keyout mgo-113.key -subj '/CN=192.168.43.113/OU=MGO/O=MongoDB/L=NY/ST=NY/C=CN'

openssl x509 -passin pass:password -days 3650 -sha256 -req -in mgo-113.csr -signkey mgo-113.key -CA ca.pem -CAkey ca_p.pem -CAcreateserial -out mgo-113.crt

cat mgo-113.crt mgo-113.key > mgo-113.pem

# 创建具有超级管理员权限的客户端证书文件
openssl req -newkey rsa:2048 -nodes -out root.csr -keyout root.key -subj '/CN=ROOT/OU=MGO/O=MongoDB/L=NY/ST=NY/C=CN'

openssl x509 -passin pass:password -sha256 -days 3650 -req -in root.csr -signkey root.key -CA ca.pem -CAkey ca_p.pem -CAcreateserial -nameopt RFC2253 -out root.crt

cat root.crt root.key > root.pem


# 拷贝证书到192.168.43.113的证书目录
cp ca.pem ${keyfile}
cp mgo-113.pem ${keyfile}/mgo.pem
cp root.pem ${keyfile}/root.pem

# 拷贝证书到192.168.43.112的证书目录
scp ca.pem root@192.168.43.112:${keyfile}
scp mgo-112.pem root@192.168.43.112:${keyfile}/mgo.pem

# 拷贝证书到192.168.43.111的证书目录
scp ca.pem root@192.168.43.111:${keyfile}
scp mgo-111.pem root@192.168.43.111:${keyfile}/mgo.pem

执行docker-compose

1 2	cd ${docker_path} docker-compose up -d

创建复制集

192.168.43.113

创建mgo01复制集

1
2
3

# 进入docker的交互界面
cd ${docker_path}
docker-compose exec mgo01 bash

# 连接mongodb
cd /keyfile

mongo --host 192.168.43.111 --port 27001 --tls --tlsCertificateKeyFile root.pem --tlsCAFile ca.pem --authenticationMechanism MONGODB-X509 --authenticationDatabase='$external'

# 创建mgo01复制集
config = { _id:"mgo01", members:[
{_id:0,host:"192.168.43.111:27001"},
{_id:1,host:"192.168.43.112:27001"},
{_id:2,host:"192.168.43.113:27001"}]
};
rs.initiate(config);
rs.conf();
# 退出mongo shell
exit

创建mgo02复制集

mongo --host 192.168.43.111 --port 27002 --tls --tlsCertificateKeyFile root.pem --tlsCAFile ca.pem --authenticationMechanism MONGODB-X509 --authenticationDatabase='$external'

config = { _id:"mgo02", members:[
{_id:0,host:"192.168.43.111:27002"},
{_id:1,host:"192.168.43.112:27002"},
{_id:2,host:"192.168.43.113:27002"}]
};
rs.initiate(config);
rs.conf();

# 退出mongo shell
exit

创建mgo03复制集

mongo --host 192.168.43.111 --port 27003 --tls --tlsCertificateKeyFile root.pem --tlsCAFile ca.pem --authenticationMechanism MONGODB-X509 --authenticationDatabase='$external'

config = { _id:"mgo03", members:[
{_id:0,host:"192.168.43.111:27003"},
{_id:1,host:"192.168.43.112:27003"},
{_id:2,host:"192.168.43.113:27003"}]
};
rs.initiate(config);
rs.conf();

创建配置服务configsvr

mongo --host 192.168.43.111 --port 27010 --tls --tlsCertificateKeyFile root.pem --tlsCAFile ca.pem --authenticationMechanism MONGODB-X509 --authenticationDatabase='$external'


config = { _id:"cfgReplSet",configsvr:true,members:[
{_id:0,host:"192.168.43.111:27010"},
{_id:1,host:"192.168.43.112:27010"},
{_id:2,host:"192.168.43.113:27010"}]
};
rs.initiate(config);
rs.conf();

配置仲裁节点或者权重

本例使用的是权重方式

权重

# 注意配置权重要在PRIMARY节点才可以

# 192.168.43.111
conf = rs.config()			    

conf.members[0].priority = 3	

conf.members[1].priority = 2

conf.members[2].priority = 1

rs.reconfig(conf)

# 192.168.43.112
conf = rs.config()			    

conf.members[0].priority = 2	

conf.members[1].priority = 3

conf.members[2].priority = 1

rs.reconfig(conf)

# 192.168.43.113
conf = rs.config()

conf.members[0].priority = 1

conf.members[1].priority = 2

conf.members[2].priority = 3

rs.reconfig(conf)

仲裁节点

# mgo01
rs.addArb('192.168.43.114:27001');
# mgo02
rs.addArb('192.168.43.114:27002');
# mgo03
rs.addArb('192.168.43.114:27003');

添加隐藏节点

本例不使用隐藏节点

# mgo01
rs.add('192.168.43.114:27031');
cfg = rs.conf()
cfg.members[5].priority = 0
cfg.members[5].hidden = true
rs.reconfig(cfg)

# mgo02
rs.add('192.168.43.114:27032');
cfg = rs.conf()
cfg.members[5].priority = 0
cfg.members[5].hidden = true
rs.reconfig(cfg)

# mgo01
rs.add('192.168.43.114:27032');
cfg = rs.conf()
cfg.members[5].priority = 0
cfg.members[5].hidden = true
rs.reconfig(cfg)

添加分片

添加分片，不能添加arbiter节点


docker-compose exec mgo01 bash
cd keyfile/

mongo --host 192.168.43.113 --port 27020 --tls --tlsCertificateKeyFile root.pem --tlsCAFile ca.pem --authenticationMechanism MONGODB-X509 --authenticationDatabase='$external'

sh.addShard("mgo01/192.168.43.111:27001,192.168.43.112:27001,192.168.43.113:27001");
sh.addShard("mgo02/192.168.43.111:27002,192.168.43.112:27002,192.168.43.113:27002");
sh.addShard("mgo03/192.168.43.111:27003,192.168.43.112:27003,192.168.43.113:27003");

# 查看配置状态
sh.status()*

声明库和表要分片

mongo --host 192.168.43.113 --port 27020 --tls --tlsCertificateKeyFile root.pem --tlsCAFile ca.pem --authenticationMechanism MONGODB-X509 --authenticationDatabase='$external'

use admin
db.runCommand({enablesharding:"testdb"})
sh.shardCollection( "testdb.books", {id : "hashed" } )

sh.shardCollection( "testdb.books", {id : "hashed" } )

测试

use testdb
for (var i = 1; i <= 20000; i++){db.books.save({id:i,name:"ttbook",sex:"male",age:27,value:"test"})}

# 查看分片情况
db.books.stats()
sh.status()

创建指定权限的用户

先用root证书的用户在创建一个库

mongos> use dd
switched to db dd
mongos> db.users.insert({"username":"chc","age":20})
WriteResult({ "nInserted" : 1 })
mongos> show dbs
admin   0.000GB
config  0.002GB
dd      0.000GB
testdb  0.001GB
mongos>

创建一个只有testdb读写权限的用户

cd /home/mgo_keyfile
# 创建新用户证书
openssl req -newkey rsa:2048 -nodes -out test.csr -keyout test.key -subj '/C=CN/ST=NY/L=NY/O=MongoDB/OU=testdb/CN=CHC'

openssl x509 -passin pass:password -sha256 -days 3650 -req  -in test.csr -signkey test.key -CA ca.pem -CAkey ca_p.pem -CAcreateserial -out test.crt -nameopt RFC2253

cat test.crt test.key > test.pem

cp test.pem ${keyfile}
cd ${docker_path}
docker-compose exec mgo01 bash
cd keyfile/

mongo --host 192.168.43.111 --port 27020 --tls --tlsCertificateKeyFile root.pem --tlsCAFile ca.pem --authenticationMechanism MONGODB-X509 --authenticationDatabase='$external'

# 创建用户
db.getSiblingDB("$external").runCommand(
  {
    createUser: "CN=CHC,OU=testdb,O=MongoDB,L=NY,ST=NY,C=CN",
    roles: [
         { role: "readWrite", db: "testdb" },
    ]
  }
)

# 退出
exit


mongo --host 192.168.43.111 --port 27020 --tls --tlsCertificateKeyFile test.pem --tlsCAFile ca.pem --authenticationMechanism MONGODB-X509 --authenticationDatabase='$external'

# 就只能看到testdb
mongos> show dbs
testdb  0.001GB