Python自动化部署

# -*- coding: utf-8 -*-

#!/bin/env python

'''

#Auth: karl

#Function: released version

#Date:2017/6/27

#Version:V1.0

'''

import  sys,re,os,time,datetime

import  paramiko

import logging

import Socket

import ConfigParser

import traceback

from progressbar import *

import Auto_Mysql_release

import platfORM

import smtplib

import email.mime.multipart

import email.mime.text

import JSON

import os

import struct

import requests

receivers = "XXX@.com"

receiver = "XXX@com"

#reg 为0时正常执行命令,为1时开始检查服务启动是否正常,为3时不用再备份原文件

def ssh2(host, port, username, passWord, cmds,reg=0):

  #  链接远程服务器并执行命令p

    try:

        paramiko.util.log_to_file('./../log/exec_cmd_' + time_str + '.log')

        ssh = paramiko.SSHClient()

        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        ssh.connect(host, int(port), username, password, timeout=5)

        # 执行命令文件中的命令

        if reg == 3:

            stdin, stdout, stderr = ssh.exec_command('ls /tmp/backup/old/ADMIN/integration.properties')

            if stdout.readline() != '':

                pass

            else:

                for cmd in cmds:

                    logging.info('running: ' + cmd)

                    print 'running: ' + cmd

                    time.sleep(2)

                    stdin, stdout, stderr = ssh.exec_command(cmd)

                    for out_msg in stdout.readlines():

                        print out_msg

        else:

            for cmd in cmds:

                print "=======>>",cmd

                logging.info('running: ' + cmd)

                print 'running: ' + cmd

                time.sleep(5)

                stdin, stdout, stderr = ssh.exec_command(cmd)

                if reg == 1:

                    out_msg=str("".join(stdout.readlines()))

                    print "000000------>>",out_msg

                    if "[main] INFO  org.eclipse.jetty.server.Server.doStart(Server.java:379)" in out_msg and "error" not in out_msg and "Failed startup" not in out_msg:

                        print  "THIS IS ok..."

                        print "------->> %s"%(host)

                    else:

                        #回滚到当前版本

#                        repair_cmd_file = conf.get(section, 'repair_cmd_file')

#                        exec_file_cmd(conf, section, repair_cmd_file)

                        copy_cmd_file(conf, section, host,1)

                        for ip_f in host_ip[1:]:

                            num_f = host_ip.index(ip_f)

                            num_f += 1

                            process_cmd_file(conf, section,ip_f, num_f)

                        for ip_s in host_ip:

                            num_s = host_ip.index(ip_s)

                            num_s += 1

                            print "+++++++ %s" % (ip_s)

                            start_server(conf, section, ip_s, num_s)

                        exit(1)

                else:

                    print "++++++++++++++",reg

                    for out_msg in stdout.readlines():

                        print "----", out_msg

            for err_msg in stderr.readlines():

                print err_msg

                exit(1)

        ssh.close()

        print 'command execute successful!'

        logging.info('command execute successful!')

    except Exception, e:

        print '%s\tError\n' % (host)

        print traceback.format_exc()

        __err_exit_show_msg(str(e))

def upload(conf, section):

    '''

    上传文件到远程服务器

    '''

    host = conf.get(section, 'host')

    port = conf.get(section, 'port')

    username = conf.get(section, 'username')

    password = conf.get(section, 'password')

    local_file = conf.get(section, 'local_file')

    remote_file = conf.get(section, 'remote_file')

    try:

        paramiko.util.log_to_file('../log/upload_' + time_str + '.log')

        logging.info('paramiko log created')

        t = paramiko.Transport((host, int(port)))

        t.connect(username=username, password=password)

        logging.info('connected host <' + host + '> successful')

        logging.info('Upload file SUCCESSFUL %s ' % datetime.datetime.now())

        sftp = paramiko.SFTPClient.from_transport(t)

        logging.info('Upload file SUCCESSFUL %s ' % datetime.datetime.now())

        print 'Beginning to upload file to %s  %s ' % (host, datetime.datetime.now())

        # 定义上传进度条样式

        widgets = ['File: ', Percentage(), ' ',

                   Bar(marker='#', left='[', right=']'),

                   ' ', ETA(), ' ', FileTransferSpeed()]

        file_size = os.path.getsize(local_file)

        pbar = ProgressBar(widgets=widgets, maxval=file_size)

        # 开始进度条

        pbar.start()

        # 使用匿名方法接收上传返回值，并且显示进度条

        progress_bar = lambda transferred, toBeTransferred: pbar.update(transferred)

        sftp.put(local_file,remote_file, callback=progress_bar)

        pbar.finish()

        logging.info('Upload file SUCCESSFUL %s ' % datetime.datetime.now())

        print 'Upload file SUCCESSFUL %s ' % datetime.datetime.now()

        t.close()

        logging.info('sftp closed!')

        cmd="tar -xvf %s  -C /home/appdeploy/version/ >/dev/null " % local_file

        if "linux" == platform.system():

            os.system(cmd)

            report_cmd_file = __checke_conf_key_value_empty(conf, section, 'test_report')

    except Exception, e:

        logging.error('host: <' + host + '> connect error!')

        print host, 'connect error!'

        print traceback.format_exc()

        __err_exit_show_msg(str(e))

def email_send(section,version):

    evn=section

    V=version

    msg = email.mime.multipart.MIMEMultipart()

    msg['Subject'] = '版本发布通知邮件'

    msg['From'] = '18682147058@163.com'

    msg['To'] = ','.join(receivers)

    content = '''

        你好，各位同事：

                    本次%s版本(%s)发布:   发布成功,祝贺！！！

    '''%(evn,V)

    txt = email.mime.text.MIMEText(content)

    msg.attach(txt)

    smtp = smtplib.SMTP()

    smtp.connect('smtp.163.com', '25')

    smtp.login('18682147058@163.com', 'passwd')

    smtp.sendmail(msg['From'], receivers, msg.as_string())

    smtp.quit()

    print('邮件发送成功email has send out !')

def email_linux(receivers, subject=None, bodyhtml=None,attachments=None):

    '''

对接统一通知平台，发邮件样例

receivers 收件人邮箱

subject 主题

bodyhtml 邮件内容

attachment即附件路径默认为空，如有附件传入文件路径'''

    file_name = attachments.split("/")[-1]

    lis = ''

    time_str =str(time.strftime('%Y-%m-%d',time.localtime(time.time())))

    # 把附件内容转换为字符列表

    if attachments != None:

        file_name = os.path.basename(attachments)

        file = open(attachments, 'rb')

        _content = file.read()

        lis = struct.unpack('%db' % len(_content),_content)

    # 对应渠道模板中`message`中参数

    templatejson = {

        'version'  : bodyhtml,

        'time'     :time_str,

        'question' :"详情请审阅附件，谢谢！"

    }

    data = {

        'userId': receivers, #收件人邮箱,支持多收件人,分号隔离;如下面的抄送人格式

        'ccId': receiver, #抄送人,如果没有可以屏蔽该语句

        'templateCode': 'version_release_code', #业务模板code

        'templateParam': templateJson, #如果对应渠道模板中没有类{{}}格式的参数，可以屏蔽该语句

        'subject': subject, #邮件主题，默认是渠道模板名称

        'attachmentName' : file_name, #邮件附件名称， 如果没有附件可以屏蔽该语句

        'attachmentArray': lis, #邮件附件内容,如果没有附件可以屏蔽该语句

        'msgType': 'txt', #消息类型，目前只支持txt

        'accessId': '52HX1CYE', #通知平台接入Id

        'accessToken': 'ebf1dd3140cf4f0abd79872d7d237c3D' #通知平台接入Token

    }

    json_str = json.dumps(data)

    url = "Http://public-int-gw.int.sfdc.com.cn:1080/unp/notice/single"

    headers = {'content-type': 'application/json; charset=UTF-8'}

    try:

        response = requests.post(url, data=json_str, headers=headers)

        print response.text

        print "-------------"

        result_json = json.loads(response.text)

        print(result_json) #打印返回内容

    except Exception as e:

        print('调用统一通知平台接口失败:',str(e))

def copy_cmd_file(conf, section, ip,reg):

    if reg == 0:

        filep="/home/appdeploy/version/Version_3.0"

    else:

        filep="/tmp/backup/old"

    host=ip

    port = conf.get(section, 'port')

    username = conf.get(section, 'username')

    password = conf.get(section, 'password')

    remote_file = conf.get(section, 'remote_file')

    print "----------------------------copy files--- %s--------------------" % (host) 

    cmd_c = [

        '\cp -vr {files}/ADMIN/*.war  {path}_ADMIN_01/deploy/WEBapps/'.format(files=filep, path=Filepath),

        '\cp -vr {files}/ADMIN/integration.properties {path}_ADMIN_01/deploy/resources/'.format(files=filep,path=Filepath),

        '\cp -vr {files}/TRAPP/*.war  {path}_TRAPP_01/deploy/webapps/'.format(files=filep, path=Filepath),

        '\cp -vr {files}/TRAPP/integration.properties  {path}_TRAPP_01/deploy/resources/'.format(files=filep,path=Filepath),

        '\cp -vr {files}/TRTS/*.war  {path}_TRTS_01/deploy/webapps/'.format(files=filep, path=Filepath),

        '\cp -vr {files}/TRTS/integration.properties  {path}_TRTS_01/deploy/resources/'.format(files=filep, path=Filepath),

    ]

    ssh2(host, port, username, password, cmd_c)

#针对脚本进行参数化设置

def process_cmd_file(conf, section,ip_f, num_f):

    host = ip_f

    port = conf.get(section, 'port')

    username = conf.get(section, 'username')

    password = conf.get(section, 'password')

    remote_file = conf.get(section, 'remote_file')

    print "----------------------------copy remote files--- %s--------------------" % (host)

    print "-------->>>>>",host_ip[0],port,username,password,remote_file

    cmd=[

        'scp {path}_ADMIN_01/deploy/webapps/*.war  {user}@{ip}:{path}_ADMIN_0{num}/deploy/webapps/'.format(path=Filepath,user=username,ip=host,num=num_f),

#       'scp {path}_ADMIN_01/deploy/resources/integration.properties  {user}@{ip}:{path}_ADMIN_0{num}/deploy/resources/integration.properties'.format(path=Filepath,user=username,ip=host,num=num_f),

        'scp {path}_TRTS_01/deploy/webapps/*.war  {user}@{ip}:{path}_TRTS_0{num}/deploy/webapps/'.format(path=Filepath,user=username,ip=host,num=num_f),

#       'scp {path}_TRTS_01/deploy/resources/integration.properties  {user}@{ip}:{path}_TRTS_0{num}/deploy/resources/integration.properties'.format(path=Filepath,user=username,ip=host,num=num_f),

        'scp {path}_TRAPP_01/deploy/webapps/*.war  {user}@{ip}:{path}_TRAPP_0{num}/deploy/webapps/'.format(path=Filepath,user=username,ip=host,num=num_f),

#        'scp {path}_TRAPP_01/deploy/resources/integration.properties  {user}@{ip}:{path}_TRAPP_0{num}/deploy/resources/integration.properties'.format(path=Filepath,user=username,ip=host,num=num_f)

    ]

    ssh2(host_ip[0], port, username, password, cmd)

def exec_file_cmd(conf, section, cmd_file):

    '''

    执行文件中的命令

    '''

    host = conf.get(section, 'host')

    port = conf.get(section, 'port')

    username = conf.get(section, 'username')

    password = conf.get(section, 'password')

    cmds = __get_cmds(cmd_file)

    ssh2(host, port, username, password, cmds)

def backup_ori(conf, section):

    '''

    备份远程原文件

    '''

    host = conf.get(section, 'host')

    port = conf.get(section, 'port')

    username = conf.get(section, 'username')

    password = conf.get(section, 'password')

    remote_file = conf.get(section, 'remote_file')

    remote_ori_backup_dir = conf.get(section, 'remote_ori_backup_dir')

    # 获得备份后缀

    suffix_time =  time.strftime('%Y-%m-%d',time.localtime(time.time()))

    backup_ori_cmd = [

        'mkdir -p {dir}/ADMIN {dir}/TRTS {dir}/TRAPP'.format(dir=remote_ori_backup_dir),

        'cp -vr {path}_TRAPP_01/deploy/webapps/*.war  {dir}/TRAPP'.format(path=Filepath,dir=remote_ori_backup_dir),

        'cp -vr {path}_ADMIN_01/deploy/webapps/*.war  {dir}/ADMIN'.format(path=Filepath,dir=remote_ori_backup_dir),

        'cp -vr {path}_TRTS_01/deploy/webapps/*.war  {dir}/TRTS'.format(path=Filepath,dir=remote_ori_backup_dir),

        'cp -vr {path}_TRAPP_01/deploy/resources/integration.properties  {dir}/TRAPP'.format(path=Filepath,dir=remote_ori_backup_dir),

        'cp -vr {path}_ADMIN_01/deploy/resources/integration.properties  {dir}/ADMIN'.format(path=Filepath,dir=remote_ori_backup_dir),

        'cp -vr {path}_TRTS_01/deploy/resources/integration.properties  {dir}/TRTS'.format(path=Filepath,dir=remote_ori_backup_dir)

    ]

    ssh2(host, port, username, password, backup_ori_cmd,3)

def backup_new(conf, section):

    '''

    备份远程新上传的文件

    '''

    host = conf.get(section, 'host')

    port = conf.get(section, 'port')

    username = conf.get(section, 'username')

    password = conf.get(section, 'password')

    remote_file = conf.get(section, 'remote_file')

    remote_backup_dir = conf.get(section, 'remote_backup_dir')

    # 获得备份后缀

    suffix_time = time.strftime('%Y-%m-%d_%H-%M-%S',time.localtime(time.time()))

    backup_new_cmd = [

        'mkdir -p {dir}'.format(dir=remote_backup_dir),

        'cp -vr {new_file} {dir}/{new_bak_file}_{time}'.format(new_file=remote_file,

                                                           dir=remote_backup_dir,

                                                           new_bak_file=os.path.basename(remote_file),

                                                           time=str(suffix_time))

    ]

    ssh2(host, port, username, password, backup_new_cmd)

def select_section(conf_file_name):

    '''

    选择指定读取的配置文件项

    例如：*.conf配置文件中有多个配置项 a 和 b:

      [a]

      xxxxx

      [b]

      yyyyy

    '''

    # 检测指定的配置文件是否存在

    __check_file_exists(conf_file_name)

    # 读取配置文件

    conf = ConfigParser.ConfigParser()

    conf.read(conf_file_name)

    sections = conf.sections()

    # 选择配置文件选项界面

    print 'please choose confit item:'

    for index, value in enumerate(sections):

        print  '  ', index, ':', value

    while True:

        sec_index = raw_input('please choose one item default [0]:')

        if not sec_index.isdigit() or int(sec_index) >= len(sections):

            print 'choose invalid!'

            continue

        return conf, sections[int(sec_index)]

        return conf, sections[0]

def check_config(conf, section):

    '''

    检测配置文件的正确性

    '''

    logging.info('check config starting...')

    print 'check config starting...'

    # 检测配置文件中值是否都填写

    host = __checke_conf_key_value_empty(conf, section, 'host')  # 检测配置文件中主机名

    port = __checke_conf_key_value_empty(conf, section, 'port')  # 检测配置文件中端口

    username = __checke_conf_key_value_empty(conf, section, 'username')  # 检测配置文件用户名

    password = __checke_conf_key_value_empty(conf, section, 'password')  # 检测配置文件密码

    local_file = __checke_conf_key_value_empty(conf, section, 'local_file')  # 检测配置文件本地需要上传文件

    remote_file = __checke_conf_key_value_empty(conf, section, 'remote_file')  # 检测配置文件上传到远程的文件

    remote_backup_dir = __checke_conf_key_value_empty(conf, section, 'remote_backup_dir')  # 检测配置文件远程备份目录

    remote_ori_backup_dir = __checke_conf_key_value_empty(conf, section, 'remote_ori_backup_dir')  # 检测配置文件远程临时备份目录

    start_cmd_file = __checke_conf_key_value_empty(conf, section, 'start_cmd_file')  # 检测配置文件启动服务文件

    report_cmd_file = __checke_conf_key_value_empty(conf, section, 'test_report')  # 检测配置文件停止服务文件

    # 检测配置文件中的网络是否可用

    __check_network_ping(host)

    # 检测ssh链接是否成功

    __check_ssh(host, int(port), username, password)

    # 检测本地需要上传的文件是否存在

    __check_file_exists(local_file)

    # 检测命令文件是否存在

    __check_file_exists(start_cmd_file)

    print 'check config successful!!'

    logging.info('check config successful!!')

def __valid_ip(address):

    '''

    检测IP是否合法IP

    '''

    try:

        socket.inet_aton(address)

        return True

    except:

        print traceback.format_exc()

        return False

def __check_file_exists(conf_file_name):

    '''

    检测指定的配置文件是否存在

    '''

    if not os.path.exists(conf_file_name):

        logging.error('can not find config file: ' + conf_file_name)

        __err_exit_show_msg('can not find config file: ' + conf_file_name)

    return conf_file_name

def __checke_conf_key_value_empty(conf, section, key):

    '''

    检测配置文件的key是否存在

    '''

    try:

        value = conf.get(section, key)

        # 检测配置文件中的值是否为空

        if value:

            return value

        else:

            msg = '''

      ERROR  The key:{key} value is empty in conf file

      '''.format(key=key)

            __err_exit_show_msg(msg)

    except ConfigParser.NoOptionError:

        print traceback.format_exc()

        msg = '''

      ERROR  cannot find key:{key} in conf file

    '''.format(key=key)

        __err_exit_show_msg(msg)

def __check_network_ping(host):

    if not __valid_ip(host):

        __err_exit_show_msg('host: ' + host + ' invalid')

    if "Linux" == platform.system():

        if 0 <> os.system('ping -c 3 ' + host):

            __err_exit_show_msg('host: ' + host + ' cannot ping...')

    else:

        if 0 <> os.system('ping -n 1 -w 5 ' + host):

            __err_exit_show_msg('host: ' + host + ' cannot ping...')

def __check_ssh(host, port, username, password):

    try:

        ssh = paramiko.SSHClient()

        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        ssh.connect(host, port, username, password, timeout=5)

        ssh.close()

    except Exception as e:

        print traceback.format_exc()

        msg = '''

    SSH connect failure. 

    please check your host/port/username/password

    host    :  {host}

    port    :  {port}

    username:  {username}

    password:  {password}

    '''.format(host=host, port=port,

               username=username,

               password=password)

        __err_exit_show_msg(msg)

def __get_cmds(cmd_file):

    '''

    文件中获取执行命令

    '''

    with open(cmd_file, 'r') as cmd_f:

        pattern = re.compile('(^\s*#|^\s*$)')

        func = lambda x: x if not re.match(pattern, x) else None

        cmds = [cmd for cmd in cmd_f]

        return filter(func, cmds)

def check_server(conf, section,ip,num):

    host = ip

    port = conf.get(section, 'port')

    username = conf.get(section, 'username')

    password = conf.get(section, 'password')

    suffix_time = time.strftime('%Y%m%d', time.localtime(time.time()))

    print "----------------------------check result--- %s--------------------"%(ip)

    checkserver_cmd=[

    'cat {path}_ADMIN_0{num}/logs/novatar_{time}.0.log |grep  "error\|org.eclipse.jetty.server.Server.doStart(Server.java:379)\|Failed startup"'.format(path=Filepath,time=suffix_time,num=num),

    'cat {path}_TRAPP_0{num}/logs/novatar_{time}.0.log |grep  "error\|org.eclipse.jetty.server.Server.doStart(Server.java:379)\|Failed startup"'.format(path=Filepath,time=suffix_time,num=num),

    'cat {path}_TRTS_0{num}/logs/novatar_{time}.0.log |grep  "error\|org.eclipse.jetty.server.Server.doStart(Server.java:379)\|Failed startup"'.format(path=Filepath,time=suffix_time,num=num)

    ]

    ssh2(host, port, username, password, checkserver_cmd,1)

def start_server(conf, section, ip, num):

    host = ip

    port = conf.get(section, 'port')

    username = conf.get(section, 'username')

    password = conf.get(section, 'password')

    print "----------------------------restart server --- %s--------------------" % (ip)

    checkserver_cmd = [

     'sh {path}_ADMIN_0{num}_run.sh stop; sh {path}_ADMIN_0{num}_run.sh start'.format(path=Restartfile,num=num),

     'sh {path}_TRAPP_0{num}_run.sh stop; sh {path}_TRAPP_0{num}_run.sh start'.format(path=Restartfile,num=num),

     'sh {path}_TRTS_0{num}_run.sh stop; sh {path}_TRTS_0{num}_run.sh start'.format(path=Restartfile,num=num)

    ]

    ssh2(host, port, username, password, checkserver_cmd)

def __err_exit_show_msg(msg):

    '''

    发生错误的时候显示相关错误信息并且退出程序

    '''

    print 'ERROR:  ' + msg

    logging.error('ERROR:  ' + msg)

    os.system('pause')

    sys.exit()

if __name__ == '__main__':

    try:

        start = time.clock()

        # 设置日志文件

        time_str = time.strftime('%Y-%m-%d',time.localtime(time.time()))

        log_file = '../log/upload_distribution_' + str(time_str) + '.log'

        logging.basicConfig(level=logging.INFO,

                            format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',

                            filename=log_file,

                            filemode='w',

                            datefmt='%Y-%m-%d %X')

        # 定义配置文件路径

        conf_file_name = '../conf/release.conf'

        # 选择配置文件section

        conf, section = select_section(conf_file_name)

        if section == "STG":

            Filepath="/app/jetty/server/SCS_ATP_CNSZ99_JETTY_APP"

            Restartfile="/app/jetty/logs/SCS_ATP_CNSZ99_JETTY_APP" 

            host_ip=[]

        elif section == "DEVTEST":

            Filepath="/app/jetty/server/SCS_ATP_CORE_CNSZ22_JETTY_APP"

            Restartfile="/app/jetty/logs/SCS_ATP_CORE_CNSZ22_JETTY_APP"

            host_ip=[]

        else :

            Filepath="/app/jetty/server/SCS_ATP_CORE_CNSZ17_JETTY_APP"

            Restartfile="/app/jetty/logs/SCS_ATP_CNSZ17_JETTY_APP"

            host_ip=[]

        # 检测配置文件正确性

        check_config(conf, section)

        print('\033[1;35m You can view the configuration file in the version number:\"/home/appdeploy/version/Version_\033[1;32m3.0\033[1;35m.tar" \033[0m!')

        VERSION=raw_input("Please Enter Version: eg 3.0 \n")

        # 备份原文件

        backup_ori(conf, section)

        # 上传文件

        upload(conf, section)

        # 备份新上传的文件

        backup_new(conf, section)

        # 解压文件

        start_cmd_file = conf.get(section, 'start_cmd_file')

        exec_file_cmd(conf, section, start_cmd_file)

        # 上传新文件到应用目录

        copy_cmd_file(conf, section, host_ip[0],0)

        # 拷贝文件到远程服务器

        if len(host_ip) > 1:

            for ip_f in host_ip[1:]:

                num_f = host_ip.index(ip_f)

                num_f += 1

                process_cmd_file(conf, section,ip_f, num_f)

        # 执行拷贝excel 等文件,记录到数据库中

        agre=["3306", "XXX", "root",host_ip[0],VERSION]

        print "agre :",agre

        stg=Auto_mysql_release.Mysql_connect(*agre)

        stg.File_get()

        stg.File_deal()

        #启动所有服务器的服务

        for ip_s in host_ip:

            num_s = host_ip.index(ip_s)

            num_s += 1

            start_server(conf, section,ip_s,num_s)

        # 监听服务是否启动成功

        time.sleep(90)

        for ip in host_ip:

            num=host_ip.index(ip)

            num+=1

            check_server(conf, section,ip,num)

        # 实行完毕退出

        cmd_f = ["rm -fr /tmp/backup/old/*"]

        ssh2(host_ip[0],"22","mwopr","XXX",cmd_f)

        os.system('exit')

        end = time.clock()

        print "version release SUCCESSFULL.......共耗时%f  s" %(end - start)

        # 邮件群发到相关责任人

        if section == "ONLINE":

            testReport = conf.get(section, 'test_report')

            text="%s版本发布通知邮件"% (VERSION)

            email_linux(receivers,"版本发布通知邮件",VERSION,testReport)

        elif section == "DEVTEST":

            email_send(section,VERSION)

        else:

            pass

        exit(0)

    except Exception as e:

        print traceback.format_exc()

        __err_exit_show_msg(str(e))

具体流程如下：