redis集群部署之codis 维护脚本

搞了几天redis cluster codis 的部署安装，测试，架构优化，配合研发应用整合，这里记一些心得！

背景需求：

之前多个业务都在应用到redis库，各业务独立占用主从两台服务器，硬件资源利用不合理，主从架构冗余度不高，主redis故障的话，从redis恢复需要时间，降低业务的可用性，所以调研测试部署了基于codis的redis集群。 官方地址

部署文档 参考这里http://www.cnblogs.com/shantu/p/4589798.html

维护管理

了解过codis的同志都知道codis集群组件服务启动有一定顺序的，而且基本了添加组，初始化slot ,各节点启动proxy都依懒于dashboard, 所以要遵循以下基本步骤。

start dashboard
start master and slave codis redis
add group
init slot
start proxy

下面根据自已的使用习惯写了一个codis集群的管理脚本

主要实现codis集群服务组件启动关闭、redis配置文件生成的功能，python学的实在太烂，之前写的那个更烂，这个做了些优化
使用说明:codis 本身默认配置为config.ini 放在当前home, 使用此脚本，也需要预配codis.ini配置文件，供脚本解析调用，

如下:

[codis_group]

group_1 = 192.168.5.14:6381 master,192.168.5.15:6381 slave

group_2 = 192.168.5.15:6382 master,192.168.5.14:6382 slave

group_3 = 192.168.5.16:6383 master,192.168.5.44:6383 slave

group_4 = 192.168.5.44:6384 master,192.168.5.16:6384 slave

cat redis_manage.py

        #!/usr/bin/env python

        # coding:utf8

        #author:shantuwqk@163.com

        import os,sys,commands,time

        import ConfigParser

        from subprocess import Popen,PIPE

        from mako.template import Template

        codis_config = "config.ini"

        codis_root = "/data/setup/codis/data"

        codis_dbdir = "%s/db"%codis_root

        def pars_config():

        codis_group_dict = {}

        cf = ConfigParser.ConfigParser()

        cf.read("codis.ini")

        codis_group  = cf.items("codis_group")

        return codis_group

        def codis_dashboard(opt):

        if opt == "start":

            #os.chdir(codis_root)

            exec_cmd = "cd %s; nohup ../bin/codis-config -c %s -L ./log/dashboard.log dashboard --addr=:18087 --http-log=./log/http.log &>/dev/null &" %(codis_root,codis_config)

            s = os.system(exec_cmd)

            if s == 0:

            print "\033[32;1m codis dashboard start .... OK\033[0m"

            else:

            print "\033[31;1m codis dashboard start .... Error\033[0m"

        elif opt == "stop":

            dashboard_id = "ps aux |grep \"codis-config\"| grep \"dashboard\"|awk '{print $2}'"

            s,v = commands.getstatusoutput("kill -9 `%s`" %dashboard_id)

            if s == 0:

            print "\033[32;1m KILL codis dashboard id:[%s] OK\033[0m"%dashboard_id

            else:

            print "\033[31;1m KILL codis dashboard id:[%s] Error\033[0m"%dashboard_id

        else:

            pass

        def INIT_config_master(port,memsize,datadir,gtag):

        os.chdir(codis_root)

        template_file = Template(filename="./conf/redis.master.conf.template",module_directory='tmp/test').render(port="%s"%port,memsize=memsize,datadir=datadir)

        conf = open('./conf/%s.conf'%gtag,'w')

        conf.write(template_file)

        conf.close()

        dirisnot = os.path.exists(datadir)

        redisconf = os.path.isfile("%s/conf/%s.conf"%(codis_root,gtag))

        if not dirisnot and  redisconf:

            print dirisnot,"create %s ...."%datadir

            os.makedirs(datadir)

            print commands.getstatusoutput("ls -l %s/conf/redis_%s.conf"%(codis_root,gtag))[1]

            return True

        else:

            print "%s is exists !!"%datadir

            return False

        def INIT_config_slave(port,memsize,datadir,slaveof,gtag):

        os.chdir(codis_root)

        template_file = Template(filename="./conf/redis.slave.conf.template",module_directory='tmp/test').render(port="%s"%port,memsize=memsize,datadir=datadir,slaveof="%s %s"%(slaveof.split(":")[0],slaveof.split(":")[1]))

        conf = open('./conf/%s.conf'%gtag,'w')

        conf.write(template_file)

        conf.close()

        dirisnot = os.path.exists(datadir)

        redisconf = os.path.isfile("./conf/%s.conf"%(gtag))

        if not dirisnot and  redisconf:

            print dirisnot,"create %s ...."%datadir

            os.makedirs(datadir)

            print commands.getstatusoutput("ls -l %s/conf/redis_%s.conf"%(codis_root,gtag))[1]

            return True

        else:

            print "%s is exists !!"%datadir

            return False

        def codis_redis(opt,cf):

        print "###################33",cf,

        cname = cf[:-5]

        print cname,

        port = cf[-9:-5]

        print "#########",port,

        redis_id = "ps aux | grep codis-server| grep -v grep |grep %s|awk '{print $2}'"%port

        print redis_id

        if opt == "start":

            ids,idv = commands.getstatusoutput(redis_id)

            print ids,type(idv)

            if ids == 0 and  idv != "":

            print "\033[32;1m%s redis process Already exists pid is:[%s]\033[0m"%(port,idv)

            sys.exit(1)

            else:

            exec_cmd = "cd %s; nohup ../bin/codis-server ./conf/%s.conf &> ./log/%s.log &" %(codis_root,cname,cname)

            print exec_cmd

            print os.system(exec_cmd)

            s = os.system(exec_cmd)

            if s == 0:

                print "\033[32;1m start redis %s server is OK \033[0m"%(port)

            else:

                print "\033[31;1m start redis %s server is Error\033[0m"%(port)

        elif opt == "stop":

            s,v = commands.getstatusoutput("kill -9 `%s`" %redis_id)

            if s == 0:

            print "\033[32;1m KILL redis port:[%s] pid:[%s] is OK\033[0m"%(port,redis_id)

            print v,

            else:

            print "\033[31;1m KILL redis port:[%s] pid:[%s] is Error\033[0m"%(port,redis_id)

            print v,

        else:

            pass

        def codis_group():

        print "change codis.ini exists?"

        if os.path.isfile("codis.ini") and pars_config() != "":

            print "\033[32;1mparsing the codis.ini configuration file\033[0m"

            for i in sorted(pars_config()):

            #print i[1],'\n'

            for j in i[1].split(","):

                #print j,'\n',

                exec_cmd = "cd %s; ../bin/codis-config -c %s -L ./log/%s_addgroup.log server add %s %s %s" %(codis_root,codis_config,j.split()[0].split(":")[1],i[0].split("_")[1],j.split()[0],j.split()[1])

                print exec_cmd,'\n',

                s,v = commands.getstatusoutput(exec_cmd)

                print s,v,

                if s == 0:

                print "\033[32;1m add group:[%s] with a gtag:[%s] addr:(%s) OK\033[0m" %(i[0].split("_")[1],j.split()[1],j.split()[0])

                else:

                print "\033[31;1m add group:[%s] with a gtag:[%s] addr:(%s) Error\033[0m" %(i[0].split("_")[1],j.split()[1],j.split()[0])

        else:

            print "config not exist!!"

        def remove_fenc():

        remove_fenc = "cd %s;../bin/codis-config -c %s action remove-fence"%(codis_root,codis_config)

        s,v = commands.getstatusoutput(remove_fenc)

        print "remove fenc proxy info",v,

        def slot_init():

        init_cmd = "cd %s; ../bin/codis-config -c %s slot init -f" %(codis_root,codis_config)

        print "INIT SLOT ........","\n",commands.getstatusoutput(init_cmd)[1]

        def codis_initslot(gid,slot_range):

        exec_cmd = "cd %s; ../bin/codis-config -c %s slot range-set %s %s %s online"%(codis_root,codis_config,slot_range[0],slot_range[1],gid)

        print exec_cmd

        s,v = commands.getstatusoutput(exec_cmd)

        print v,

        if s == 0:

            print "\033[32;1m slot init:[%s],gid:[%s] OK\033[0m"%(slot_range,gid)

        else:

            print "\033[31;1m slot init:[%s],gid:[%s] Error\033[0m"%(slot_range,gid)

        def offline_proxy():

        proxy_tag = commands.getstatusoutput("cat %s|grep proxy_id"%codis_config)[1].split('=')[1]

        exec_down_cmd = "cd %s; ../bin/codis-config -c %s proxy offline %s" %(codis_root,codis_config,proxy_tag)

        print "Shutdown %s offline....." %proxy_tag

        print exec_down_cmd

        commands.getstatusoutput(exec_down_cmd)

        def codis_proxy(opt):

        proxy_tag = commands.getstatusoutput("cat %s|grep proxy_id"%codis_config)[1].split('=')[1]

        if opt == "start":

            exec_new_cmd = "cd %s; nohup ../bin/codis-proxy --log-level info -c %s -L ./log/%s.log --cpu=8 --addr=0.0.0.0:19000 --http-addr=0.0.0.0:11000 &" %(codis_root,codis_config,proxy_tag)

            print exec_new_cmd

            s = os.system(exec_new_cmd)

            if s == 0:

            print "\033[32;1m codis proxy tag:[%s]start OK\033[0m"%(proxy_tag)

            else:

            print "\033[31;1m codis proxy tag:[%s]start Error\033[0m"%(proxy_tag)

            time.sleep(5)

            exec_online_cmd = "cd %s;../bin/codis-config -c %s proxy online %s" %(codis_root,codis_config,proxy_tag)

            print "Set %s online .....!!" %(proxy_tag)

            print exec_online_cmd

            print commands.getstatusoutput(exec_online_cmd)[1]

        elif opt == "stop":

            print "Shutdown %s offline....." %proxy_tag

            exec_down_cmd = "cd %s; ../bin/codis-config -c %s proxy offline %s" %(codis_root,codis_config,proxy_tag)

            print commands.getstatusoutput(exec_down_cmd)[1]

            exec_proxy_id = "ps aux |grep codis-proxy| grep -v grep|awk '{print $2}'"

            print commands.getstatusoutput("kill -9 `%s`"%exec_proxy_id)

        def get_client_ip():

        eth_inter= commands.getstatusoutput("ifconfig -a| awk '/^em/ {;a=$1;FS=\":\"; nextline=NR+1; next}{ if (NR==nextline) { split($2,b,\" \")}{ if ($2 ~ /[0-9]\./) {print a,b[1]}; FS=\" \"}}'|uniq -c|awk '{print $2,$3}'")

        if eth_inter[0] == 0:

            return eth_inter[1].split()[1]

        else:

            print "get client ip error"

        def slot_range(n,group):

        slot_dict = {}

        num=0

        l = range(n)

        per = len(l) / group

        #改变i的索引值

        for i in l[::per]:

            avg=l[i:i+per]

            tupv01 = avg[0],avg[-1]

            #print type(tupv01),tupv01

            #指定当前添加的组数

            num+=1

            slot_dict[num] = []

            slot_dict[num].append(tupv01)

            #如果只剩下一组，则将剩余元素全部追加至列表

            if num==group-1:

            tupv02 = l[i+per:][0],l[i+per:][-1]

            #print tupv02

            v02key = num + 1

            slot_dict[v02key] = []

            slot_dict[v02key].append(tupv02)

            break

        #for k,port in redis_instance_port.items():

        #    if k in slot_dict.keys():

        #        slot_dict[k].append(port)

        #print slot_dict

        return slot_dict

        def handle_slot():

        n = len(pars_config())

        return  slot_range(1024,n)

        def init_config():

        for i in sorted(pars_config()):

            for j in i[1].split(","):

                if j.split()[1] == "master":

                confname = i[0] + "_" + j.split()[1] + "_" + j.split()[0].split(":")[0] + "_" + j.split()[0].split(":")[1]

                INIT_config_master(j.split()[0].split(":")[1],6,codis_dbdir,confname)

                mastername = j.split()[0]

                elif j.split()[1] == "slave":

                confname = i[0] + "_" + j.split()[1] + "_" + j.split()[0].split(":")[0] + "_" + j.split()[0].split(":")[1]

                INIT_config_slave(j.split()[0].split(":")[1],6,codis_dbdir,mastername,confname)

                else:

                pass

        def start_master():

        mcfile_cmd = "cd %s/conf;ls -lrt| egrep \"%s\"| grep \"master\"|awk '{print $9}'"%(codis_root,get_client_ip())

        print mcfile_cmd

        mcfile = commands.getstatusoutput(mcfile_cmd)[1]

        codis_redis('start',mcfile)

        def stop_master():

        mcfile_cmd = "cd %s/conf;ls -lrt| egrep \"%s\"| grep \"master\"|awk '{print $9}'"%(codis_root,get_client_ip())

        print mcfile_cmd

        mcfile = commands.getstatusoutput(mcfile_cmd)[1]

        codis_redis('stop',mcfile)

        def start_slave():

        mcfile_cmd = "cd %s/conf;ls -lrt| egrep \"%s\"| grep \"slave\"|awk '{print $9}'"%(codis_root,get_client_ip())

        print mcfile_cmd

        mcfile = commands.getstatusoutput(mcfile_cmd)[1]

        codis_redis('start',mcfile)

        def stop_slave():

        mcfile_cmd = "cd %s/conf;ls -lrt| egrep \"%s\"| grep \"slave\"|awk '{print $9}'"%(codis_root,get_client_ip())

        print mcfile_cmd

        mcfile = commands.getstatusoutput(mcfile_cmd)[1]

        codis_redis('stop',mcfile)

        def assign_slot():

        remove_fenc()

        slot_init()

        for k,v in handle_slot().items():

            codis_initslot(k,v[0])

        def help_prompt():

        print """

        This program prints files to the standard output.

        Options include:

        --version : Prints the version number

        --help    : Helpful tips

        sample    : python codis_manage.py init

            : python codis_manage.py start_master/stop_master start_slave/stop_slave start_web/stop_web add_group initslot start_proxy/stop_proxy

        """

        if __name__ == "__main__":

        if len(sys.argv) < 2:

            print "no argument",'\n',help_prompt()

            sys.exit(1)

        if sys.argv[1] == "--help":

            help_prompt()

        elif sys.argv[1] == "--version":

            print "Version 0.1"

        elif sys.argv[1] == "init":

            init_config()

        elif sys.argv[1] == "start_master":

            start_master()

        elif sys.argv[1] == "stop_master":

            stop_master()

        elif sys.argv[1] == "start_slave":

            start_slave()

        elif sys.argv[1] == "stop_slave":

            stop_slave()

        elif sys.argv[1] == "start_web":

            codis_dashboard('start')

        elif sys.argv[1] == "stop_web":

            codis_dashboard('stop')

        elif sys.argv[1] == "add_group":

            codis_group()

        elif sys.argv[1] == "initslot":

            assign_slot()

        elif sys.argv[1] == "start_proxy":

            codis_proxy('start')

        elif sys.argv[1] == "stop_proxy":

            codis_proxy('stop')

        elif sys.argv[1] == "stopall":

            codis_proxy('stop')

            commands.getstatusoutput("killall codis-server")

            codis_dashboard("stop")

        else:

            help_prompt()

        print get_client_ip()

这里还有一个基于redis sentinel方式的集群redis服务启停脚本

sentinel.conf配置示例

 port 26330

sentinel monitor lashou 192.168.5.14 6479 2

sentinel down-after-milliseconds lashou 3000

sentinel failover-timeout lashou 4000

sentinel notification-script lashou /data/setup/redis-2.8.19/bin/get_redis_master.py

 cat get_redis_master.py

#!/usr/bin/env python

# coding:utf8

#author:shantuwqk@163.com

import os,sys,commands,time

from subprocess import Popen,PIPE

from mako.template import Template

from get_redis_master import get_redis_master_info

REDIS_ROOT = "/data/setup/redis-2.8.19"

REDIS_DBDIR = "%s/data"%REDIS_ROOT

def INIT_config_master(port,memsize,datadir):

    os.chdir(REDIS_ROOT)

    template_file = Template(filename="./conf/redis.master.conf.template",module_directory='tmp/test').render(port="%s"%port,memsize=memsize,datadir=datadir)

    conf = open('./conf/redis_%s.conf'%port,'w')

    conf.write(template_file)

    conf.close()

    dirisnot = os.path.exists(datadir)

    redisconf = os.path.isfile("%s/conf/redis_%s.conf"%(REDIS_ROOT,port))

    if not dirisnot and  redisconf:

        print dirisnot,"create %s ...."%datadir

        os.makedirs(datadir)

        print commands.getstatusoutput("ls -l %s/conf/redis_%s.conf"%(REDIS_ROOT,port))[1]

        return True

    else:

        print "%s is exists !!"%datadir

        return False

def INIT_config_slave(port,memsize,datadir,slaveof):

    os.chdir(REDIS_ROOT)

    template_file = Template(filename="./conf/redis.slave.conf.template",module_directory='tmp/test').render(port="%s"%port,memsize=memsize,datadir=datadir,slaveof="%s %s"%(slaveof.split(":")[0],slaveof.split(":")[1]))

    conf = open('./conf/redis_%s.conf'%port,'w')

    conf.write(template_file)

    conf.close()

    dirisnot = os.path.exists(datadir)

    redisconf = os.path.isfile("%s/conf/redis_%s.conf"%(REDIS_ROOT,port))

    if not dirisnot and  redisconf:

        print dirisnot,"create %s ...."%datadir

        os.makedirs(datadir)

        print commands.getstatusoutput("ls -l %s/conf/redis_%s.conf"%(REDIS_ROOT,port))[1]

        return True

    else:

        print "%s is exists !!"%datadir

        return False

def start_redis(redis_root,port):

    start_redis_cmd = "cd %s ; nohup ./src/redis-server conf/redis_%s.conf &> ./logs/%s_redis.log & "%(REDIS_ROOT,port,port)

    s = os.system(start_redis_cmd)

    if s == 0:

        print " redis start instence:[%s] ok"%(port)

    else:

        print " redis start instence:[%s] faild"%(port)

def stop_redis(port):

    stop_redis_pid = "ps aux | grep redis| grep %s|awk '{print $2}'"%(port)

    stop_redis_cmd = "kill -9 `%s`"%stop_redis_pid

    s,v = commands.getstatusoutput(stop_redis_cmd)

    if s == 0:

        print " redis stop instence:[%s] ok"%(port),"\n",v

    else:

        print " redis stop instence:[%s] faild"%(port)

def help_prompt():

    print """

This program prints files to the standard output.

Options include:

--version : Prints the version number

--help    : Helpful tips

--task    : To operate on missions:

            [start_redis|stop_redis] [port]

sample    : python redis_manage.py start/stop master/slave  port

"""

if len(sys.argv) < 2:

    print "no argument"

    help_prompt()

    sys.exit()

if sys.argv[1].startswith('--'):

    option = sys.argv[1][2:]

    if option == 'version':

        print 'Version 0.1'

    elif option == 'help':

        help_prompt()

elif sys.argv[1] == "start" and sys.argv[2] == "master" and sys.argv[3] is not None:

    INIT_config_master(sys.argv[3],6,REDIS_DBDIR)

    time.sleep(2)

    start_redis(REDIS_ROOT,sys.argv[3])

elif sys.argv[1] == "start" and sys.argv[2] == "slave" and sys.argv[3] is not None:

    INIT_config_slave(sys.argv[3],6,REDIS_DBDIR,get_redis_master_info())

    time.sleep(2)

    start_redis(REDIS_ROOT,sys.argv[3])

elif sys.argv[1] == "stop" and sys.argv[2] is not None:

    print commands.getstatusoutput("ps aux | grep redis-server|grep -v grep")[1]

    stop_redis(sys.argv[2])

else:

    help_prompt()

触发更新haproxy配置文件的脚本

 #!/usr/bin/env python

# coding:utf8

#author:shantuwqk@163.com

import os,sys,commands,time

from subprocess import Popen,PIPE

from mako.template import Template

#init_redis_master =

REDIS_ROOT = "/data/setup/redis-2.8.19"

def get_redis_master_info():

    cmd = "cd %s;./src/redis-cli -h 192.168.5.15 -p 26329 sentinel masters| grep -A 4 \"ip\"|head -4"%REDIS_ROOT

    s,v = commands.getstatusoutput(cmd)

    #print v,

    redis_master = "%s:%s"%(v.split("\n")[1],v.split("\n")[3])

    return redis_master

def handle_redis_m():

    redis_master_dict = {}

    redis_master_dict[get_redis_master_info()] = "redis_master_%s_%s"%(get_redis_master_info().split(":")[0].split(".")[3],get_redis_master_info().split(":")[1])

    return redis_master_dict

def handle_redis_s():

    redis_slave_dict = {}

    cmd = "cd %s;./src/redis-cli -h 192.168.5.15 -p 26329 sentinel slaves lashou| grep -A 1 \"name\"| grep -v \"\-\-\\|name\""%REDIS_ROOT

    s,v = commands.getstatusoutput(cmd)

    slave_list =  v.split("\n")

    for i in slave_list:

        redis_slave_dict[i] = "redis_slave_%s_%s"%(i.split(":")[0].split(".")[3],i.split(":")[1])

    return redis_slave_dict

def update_ha_conf(redis_slave_dict,redis_master_dict):

    os.chdir(REDIS_ROOT)

    template_file = Template(filename="./conf/haproxy.cfg.template",module_directory='tmp/test').render(redis_slave_dict=redis_slave_dict,redis_master_dict=redis_master_dict)

    conf = open('./conf/haproxy.cfg','w')

    conf.write(template_file)

    conf.close()

    haproxy_conf = os.path.isfile("%s/conf/haproxy.cfg"%(REDIS_ROOT))

    if haproxy_conf:

        print "update haproxy config info master:[%s] and slave:[%s] ...."%(redis_master_dict,redis_slave_dict)

        print commands.getstatusoutput("ls -l %s/conf/haproxy.cfg"%(REDIS_ROOT))[1]

        return True

    else:

        print "%s is not exists !!"%haproxy_conf

        return False

remote_ha_root = "/data/setup/haproxy/conf"

localhafile = "/data/setup/redis/conf/haproxy.cfg"

def reload_ha_service(addr):

    rsync_cmd = "rsync -avz -e \"ssh -p 22\" %s root@%s:%s/"%(localhafile,addr,remote_ha_root)

    remote_cmd = "ssh -p 22 root@%s \" /usr/local/sbin/haproxy -f /data/setup/haproxy-1.5.10/conf/haproxy.cfg -p /var/run/haproxy.pid -sf $(cat /var/run/haproxy.pid)\""

    rs,rv = commands.getstatusoutput(rsync_cmd)

    remotes,remotev = commands.getstatusoutput(remote_cmd)

    if rs == 0 and remotes == 0:

        print "haproxy.cfg update ok"

    else:

        print "haproxy.cfg update faild!!"

    time.sleep(5)

if __name__ == "__main__":

   print get_redis_master_info()

   print update_ha_conf(handle_redis_s(),handle_redis_m())

如果需要迁移现有 redis 数据到 codis，该如何操作？

先搭建好 codis 集群并让 codis-proxy 正确运行起来
对线上每一个 redis 实例运行一个 redis-port 来向 codis 导入数据，例如：

for port in {6379,6380,6479,6480}; do

    nohup redis-port sync --ncpu=4 --from=redis-server:${port} \

        --target=codis-proxy:19000 > ${port}.log 2>&1 &

    sleep 5

done

tail -f *.log

 每个 redis-port 负责将对应的 redis 数据导入到 codis

 多个 redis-port 之间不互相干扰，除非多个 redis 上的 key 本身出现冲突

单个 redis-port 可以将负责的数据并行迁移以提高速度，通过 --ncpu 来指定并行数

导入速度受带宽以及 codis-proxy 处理速度限制(本质是大量的 slotsrestore 操作)

完成数据迁移，在适当的时候将服务指向 Codis，并将原 redis 下线

旧 redis 下线时，会导致 reids-port 链接断开，于是自动退出