Log日志汇总工具

Zss 发表于:

客户需要一个工具来统计数据同步信息的汇总,报错失败了多少数据,成功了多少

那么我们可以从日志中来提取这些信息,做一个汇总,把失败的数据都列举出来

之前使用的cmd命令的方式,但是对用户似乎不大友好,虽然也方便,之后做成了带有ui的界面的形式来统计输出汇总

#coding:gbk
import os,re,sys,time
from Tkinter import *
reload(sys)
sys.setdefaultencoding('gbk')

class Log_split():
    def __init__(self,day):
        self.log_path = os.getcwd() + r'\log'
        self.log_list = []
        self.fail_list = []
        self.day = day
        if os.path.exists(self.log_path+r'\%d天详细汇总_log.txt'%day):
            os.remove(self.log_path+r'\%d天详细汇总_log.txt'%day)
        if os.path.exists(self.log_path + r'\%d天错误详情_log.txt'%day):
            os.remove(self.log_path + r'\%d天错误详情_log.txt'%day)
        self.time = int(time.time())

    def print_log(self,text):
        with open(self.log_path+r'\%d天详细汇总_log.txt'%self.day,'a') as f:
            f.write(text+'\n')
        # print text

    #添加发送端的所有日志路径
    def get_log_name(self,day):
        for i in os.listdir(self.log_path):
            if '_T_' not in i:
                if '_sys_' not in i:
                    datatime = os.stat(self.log_path + '\\' +i.decode('gbk')).st_mtime
                    if self.time -int(datatime) <= day*60*60*24:
                        if '汇总' not in i.decode('gbk'):
                            self.log_list.append(self.log_path + '\\' +i.decode('gbk'))

    #整理业务日志输出新文件 返回业务具体操作 [(源端,目的端)]
    def split_log(self,path):
        with open(path,'r') as f:
            data = f.readlines()
        dst_log = []
        error_log = []

        for i in range(0,len(data)):
            if '[DST]' in data[i].decode('gbk'):
                if '条记录'.decode('gbk') in data[i].decode('gbk'):
                    text = data[i].split(',')
                    dst_log.append(text)
            if '失败 'in data[i].decode('gbk'):
                error_log.append(data[i].decode('gbk'))
                error_log.append(data[i-2].decode('gbk'))
                error_log.append(data[i-1].decode('gbk'))

        return (error_log,dst_log)

    def add(self,number_list):
        i = 0
        for t in number_list:
            i = i+int(t)
        return i

    def statistics_log(self,info,path,day):
        dst_insert_cout = 0
        dst_delete_cout = 0
        dst_update_cout = 0
        dst_insert_fail = []
        dst_delete_fail = []
        dst_update_fail = []

        if info != ([],[]):
            index1 = 1
            for t in range(0,len(info[0]),3):
                if '失败 '.decode('gbk') in info[0][t].decode('gbk'):
                    try:
                        t_info= info[0][t].split(',')
                        error1 = info[0][t+1].split(',')[0][0:5]+info[0][t+1].split(',')[9]
                        error2 = info[0][t+2].split(',')[0][0:5]+info[0][t+2].split(',')[9]
                        err_info = '     时间 --> %s\n     失败操作 --> %s\n     源IP:%s 源数据库:%s 源db库:%s 源表:%s\n     目的IP:%s 目的数据库:%s 目的db库:%s 目的表:%s \n\n     错误信息:\n     %s     \n     %s\n'.decode('gbk')%(t_info[8],t_info[9],t_info[2],t_info[1],t_info[0].strip('[DST]'),t_info[6],t_info[5],t_info[4],t_info[3],t_info[7],error1,error2)
                        self.fail_list.append(err_info)
                        with open(self.log_path + r'\%d天错误详情_log.txt'%day,'a') as f:
                            f.write('(%d)\n'%index1)
                            f.write(err_info)
                            f.write('\n\n')
                        index1+=1
                    except:pass

            for i_info in info[1]:
                i = i_info[-2]
                if '插入'.decode('gbk') in i.decode('gbk'):
                    if '失败' in i.decode('gbk'):
                        dst_insert_fail.append(i_info)
                    else:
                        dst_insert_cout = dst_insert_cout + int(re.findall('\d{1,50}',i.decode('gbk'))[0])
                elif '删除'.decode('gbk') in i.decode('gbk'):
                    if '失败' in i.decode('gbk'):
                        dst_delete_fail.append(i_info)
                    else:
                        dst_delete_cout = dst_delete_cout + int(re.findall('\d{1,50}',i.decode('gbk'))[0])
                elif '修改'.decode('gbk') in i.decode('gbk'):
                    if '失败' in i.decode('gbk'):
                        dst_update_fail.append(i_info)
                    else:
                        dst_update_cout = dst_update_cout + int(re.findall('\d{1,50}',i.decode('gbk'))[0])
                else:pass

            pass_text = '     插入成功:%d 删除成功:%d 修改成功:%d'%(dst_insert_cout,dst_delete_cout,dst_update_cout)
            self.print_log('-'*50)
            self.print_log('文件:%s 目的端业务操作统计:\n'%path.split('\\')[-1])
            fail_table_info = []
            fail_table_list = []
            fail_table_info1 = []
            fail_table_list1 = []
            fail_table_info2 = []
            fail_table_list2 = []
            fail_inset_number_list = []
            fail_update_number_list = []
            fail_delete_number_list = []

            index = 1
            if len(dst_insert_fail) != 0:
                for i in dst_insert_fail:
                    fail_table_info.append((i[7],int(re.findall('\d{1,50}',i[9])[0])))
                    fail_table_list.append(i[7])
                for i in set(fail_table_list):
                    number = 0
                    for t in fail_table_info:
                        if t[0] == i:
                            number = number+t[1]
                    self.print_log('(%d).目的表名:%s 插入失败总数:%d'.decode('gbk')%(index,i,number))
                    fail_inset_number_list.append(number)
                    index+=1
            if len(dst_delete_fail) != 0:
                for i in dst_delete_fail:
                    fail_table_info1.append((i[7],int(re.findall('\d{1,50}',i[9])[0])))
                    fail_table_list1.append(i[7])
                for i in set(fail_table_list1):
                    number1 = 0
                    for t in fail_table_info1:
                        if t[0] == i:
                            number1 = number1+t[1]
                    self.print_log('(%d).目的表名:%s 删除失败总数:%d'.decode('gbk')%(index,i,number1))
                    fail_delete_number_list.append(number1)
                    index+=1
            if len(dst_update_fail) != 0:
                for i in dst_update_fail:
                    fail_table_info2.append((i[7],int(re.findall('\d{1,50}',i[9])[0])))
                    fail_table_list2.append(i[7])
                for i in set(fail_table_list2):
                    number2 = 0
                    for t in fail_table_info2:
                        if t[0] == i:
                            number2 = number2+t[1]
                    self.print_log('(%d).目的表名:%s 更新失败总数:%d'.decode('gbk')%(index,i,number2))
                    fail_update_number_list.append(number2)
                    index+=1
            self.print_log('\n')
            self.print_log(pass_text.decode('gbk'))
            fail_text = '     插入失败:%d 删除失败:%d 修改失败:%d'%(self.add(fail_inset_number_list),self.add(fail_delete_number_list),self.add(fail_update_number_list))
            self.print_log(fail_text.decode('gbk'))
            self.print_log('-' * 50)
            self.print_log('\n')

    def read_all(self,day):
        insert_pass = []
        update_pass = []
        delete_pass = []
        insert_fail = []
        delete_fail = []
        update_fail = []
        with open(self.log_path + r'\%d天详细汇总_log.txt'%day, 'r') as f:
            data = f.readlines()
            for i in data:
                if '成功:' in i.decode('gbk'):
                    info = re.findall('(\d{1,10})(?:\s删除成功:)(\d{1,10})(?:\s修改成功:)(\d{1,10})',i)[0]
                    insert_pass.append(info[0])
                    delete_pass.append(info[1])
                    update_pass.append(info[2])
                if '失败:' in i.decode('gbk'):
                    info2 = re.findall('(\d{1,10})(?:\s删除失败:)(\d{1,10})(?:\s修改失败:)(\d{1,10})',i)[0]
                    insert_fail.append(info2[0])
                    delete_fail.append(info2[1])
                    update_fail.append(info2[2])

        if os.path.exists(self.log_path + r'\%d天汇总_log.txt'%day):
            os.remove(self.log_path + r'\%d天汇总_log.txt'%day)
        with open(self.log_path + r'\%d天汇总_log.txt'%day, 'a') as f:
            f.write('\n')
            f.write(' '*37+'%d天时间内的增删改操作汇总:\n'%day)
            f.write(' '*30+'插入成功:%d 删除成功:%d 修改成功:%d\n'%(self.add(insert_pass),self.add(delete_pass),self.add(update_pass)))
            f.write(' '*30+'插入失败:%d 删除失败:%d 修改失败:%d\n'%(self.add(insert_fail),self.add(delete_fail),self.add(update_fail)))

    def get_error_info(self,i):
        for i in self.log_list:
            pass


    def main(self,day):
        self.get_log_name(day)
        for i in self.log_list:
            info = self.split_log(i)
            self.statistics_log(info,i,day)
        if self.log_list != []:
            self.read_all(day)
            with open(self.log_path + r'\%d天汇总_log.txt' % day, 'r') as f:
                return (f.read().decode('gbk'),self.fail_list,info[0])
        else:return (' '*339 +'未查询到符合条件的日志'.decode('gbk'),self.fail_list)

def _1day():
    index = 1
    log = Log_split(1)
    txt = log.main(1)
    t.delete("0.0", "end")
    t.insert('insert',txt[0])
    text.delete("0.0", "end")

    for i in txt[1]:
        text.insert('insert', '(%d)\n'%index)
        index+=1
        text.insert('insert',i+'\n')


def _7day():
    index = 1
    log = Log_split(7)
    txt = log.main(7)
    t.delete("0.0", "end")
    t.insert('insert', txt[0])
    text.delete("0.0", "end")
    for i in txt[1]:
        text.insert('insert', '(%d)\n'%index)
        index+=1
        text.insert('insert',i+'\n')

def _30day():
    index = 1
    log = Log_split(30)
    txt = log.main(30)
    t.delete("0.0", "end")
    t.insert('insert', txt[0])
    text.delete("0.0", "end")
    for i in txt[1]:
        text.insert('insert', '(%d)\n'%index)
        index+=1
        text.insert('insert',i+'\n')

def _day():
    index = 1
    day = t1.get("0.0","end")
    log = Log_split(int(day))
    txt = log.main(int(day))
    t.delete("0.0", "end")
    text.delete("0.0", "end")
    t.insert('insert', txt[0])
    for i in txt[1]:
        text.insert('insert', '(%d)\n'%index)
        index+=1
        text.insert('insert',i+'\n')

if __name__ == '__main__':
    root = Tk()
    root.resizable(False, False)
    root.title('log日志汇总工具'.decode('gbk'))
    root.geometry('752x620')

    Button(root,text='统计近一天的日志'.decode('gbk'),width=15,height=2,command=_1day).place(x=115, y=30)
    Button(root,text='统计近七天的日志'.decode('gbk'),width=15,height=2,command=_7day).place(x=315, y=30)
    Button(root,text='统计近三十天的日志'.decode('gbk'),width=18,height=2,command=_30day).place(x=515, y=30)
    Button(root,text='统计自定义天数的日志'.decode('gbk'),width=20,height=2,command=_day).place(x=450, y=105)


    l1 = Label(root,text="输入需要统计的天数:".decode('gbk'), width=15, height=1)
    l1.place(x=280, y=120)

    l1 = Label(root,text="日志位置:%s"%(os.getcwd() + r'\log\*天汇总_log.txt   *天详细汇总_log.txt   *天错误详情_log.txt').decode('gbk'), width=90, height=1)
    l1.place(x=80, y=593)

    l = Label(root,text="汇总如下".decode('gbk'), width=10, height=1)
    l.place(x=330, y=170)

    l2 = Label(root,text="失败详情如下".decode('gbk'), width=10, height=1)
    l2.place(x=332, y=275)

    t1 = Text(root,width=3,height=1)
    t1.place(x=410, y=123)

    t = Text(root,width=100,height=5)
    t.place(x=20,y=200)    #这里设置文本框高,可以容纳两行

    text = Text(root,width=100,height=22)
    text.place(x=20,y=300)

    mainloop()