python处理nginx访问日志处理脚本

亦凉 2022-10-11 13:45 238阅读 0赞

1、统计nginx过去五分钟日志访问量（试用与python2和python3）

#!/usr/bin/env python
    # -*- coding:utf-8 -*-
    # Author:Random_lee
    import time
    import os
    import re
     
     
    class StatusPV(object):
        def __init__(self,bf_name,name):
            self.log_path = '/usr/local/nginx/logs'
            self.log_time = time.strftime("%Y%m%d")
            self.bf_name = bf_name
            self.name = name
            self.log_name = self.bf_name + '.'+self.name+'_access.log' #-%s'%(self.log_time)
            self.logfile = os.path.join(self.log_path, self.log_name)
     
        def get_filesize(self):
            file_size = os.path.getsize(self.logfile)
            f = open(self.logfile, 'r')
            if file_size > 1000000000:
                # 文件大小超过1G从中间读取
                f.seek(0, 2)
                self.seek= f.tell()/5
            else:
                self.seek=0
            f.close()
        def count_pv(self):
            self.get_filesize()
            if not os.path.exists(self.logfile):
                print('error:' + self.logfile + ' not existed.')
                return 0
            else:
                f = open(self.logfile, 'r')
                f.seek(self.seek,0)
                num = 0
                for line in f:
                    data = time.strftime('%d/%b/%Y:%H:', time.localtime())
                    if data in line:
                        expr = re.compile('^(?P<RemoteIP>.*) - - (?P<datatime>.*) (?P<request>".+") (?P<status>\d{3}) (?P<web_size>\d{1,10})')
                        # 日志格式 10.116.201.71 - - [02/Sep/2018:09:44:13 +0800] "POST /servlet/UpdateJavaFXServlet HTTP/1.0" 200 268
                        try:
                            log_info = expr.search(line)
                            log_info = log_info.groupdict()
     
                            # 解析日志信息
                            datatime = log_info["datatime"]
                            # 取出日志信息中的datatime
                            datatime = datatime.replace('[', '')
                            datatime = datatime.replace(']', '')
                            # 去掉[]
                            data_time = datatime.split(' ')[0]
                            # 取出日期时间
                            time_zone = datatime.split(' ')[1]
                            # 取出时区
                            if time_zone == '+0800':
                                # print(time_zone)
                                #print(data_time)
                                ctime = time.strptime(data_time, '%d/%b/%Y:%H:%M:%S')
                                # 转换为格式化时间 24/Aug/2018:15:42:08
                                #print(ctime)
                                time_stamp = time.mktime(ctime)
                                # 转换为时间戳
                                #print(time_stamp)
                                if time.time() - time_stamp <= 300:
                                    # 观测的时间间隔
                                    num += 1
                                    #print(datatime)
                                else:
                                     #print("error data_time:%s"%datatime)
                                     pass
                            else:
                                print("log format error")
                        except:
                            pass
                    else:
                        pass
                f.close()
                print(num)
                time_file = time.strftime('%Y%m%d',ctime)
                date_time = time.strftime('%Y-%-m-%d:%H:%M:%S',ctime)
                f = open(self.bf_name +"-" + self.name + "-"+ time_file + ".log","a+")
                f.write(str(date_time) + " " + str(num)+"\n")
                f.close()
                 
     
    if __name__ == '__main__':
        obj_StatusPV1 = StatusPV("api","test1.com")
        obj_StatusPV2 = StatusPV("api","test2.com")
        obj_StatusPV1.count_pv()
        obj_StatusPV2.count_pv()

输出结果如下：

2021-6-30:11:50:10  12698
    2021-6-30:11:55:09  11504
    2021-6-30:12:00:05  195
    2021-6-30:12:05:06  12675
    2021-6-30:12:10:06  14757
    2021-6-30:12:15:06  13104
    2021-6-30:12:20:08  12986
    2021-6-30:12:25:08  13865
    2021-6-30:12:30:08  13234
    2021-6-30:12:35:09  12716
    2021-6-30:12:40:09  12113
    2021-6-30:12:45:09  12296
    2021-6-30:12:50:10  12421
    2021-6-30:12:55:11  12022
    2021-6-30:13:00:06  267
    2021-6-30:13:05:06  12205
    2021-6-30:13:10:07  10138
    2021-6-30:13:15:07  11754
    2021-6-30:13:20:07  12331
    2021-6-30:13:25:08  11333
    2021-6-30:13:30:08  10782

2、分接口统计过去十分钟各接口访问量（只适合python3）

#!/usr/bin/env python3
    # -*- coding:utf-8 -*-
    # Author:Random_lee
    import time
    import os
    import re
     
     
    class StatusPV(object):
        def __init__(self,bf_name,name):
            self.log_path = '/usr/local/nginx/logs'
            self.log_time = time.strftime("%Y%m%d")
            self.bf_name = bf_name
            self.name = name
            self.log_name = self.bf_name + '.'+self.name+'_access.log' #-%s'%(self.log_time)
            self.logfile = os.path.join(self.log_path, self.log_name)
     
        def get_filesize(self):
            file_size = os.path.getsize(self.logfile)
            f = open(self.logfile, 'r')
            if file_size > 1000000000:
                # 文件大小超过1G从中间读取
                f.seek(0, 2)
                self.seek= f.tell()/5
            else:
                self.seek=0
            f.close()
        def count_pv(self):
            self.get_filesize()
            if not os.path.exists(self.logfile):
                print('error:' + self.logfile + ' not existed.')
                return 0
            else:
                f = open(self.logfile, 'r')
                f.seek(self.seek,0)
                num = 0
                dic={ }
                for line in f:
                    data = time.strftime('%d/%b/%Y:%H:', time.localtime())
                    if data in line:
                        expr = re.compile('^(?P<RemoteIP>.*) - - (?P<datatime>.*) (?P<request>".+") (?P<status>\d{3}) (?P<web_size>\d{1,10})')
                        # 日志格式 10.116.201.71 - - [02/Sep/2018:09:44:13 +0800] "POST /servlet/UpdateJavaFXServlet HTTP/1.0" 200 268
                        try:
                            log_info = expr.search(line)
                            log_info = log_info.groupdict()
     
                            # 解析日志信息
                            datatime = log_info["datatime"]
                            # 取出日志信息中的datatime
                            datatime = datatime.replace('[', '')
                            datatime = datatime.replace(']', '')
                            # 去掉[]
                            data_time = datatime.split(' ')[0]
                            # 取出日期时间
                            time_zone = datatime.split(' ')[1]
                            #api接口筛选
                            reg=re.compile('^(?P<Mode>.*) (?P<request>.*) (?P<Tcpmode>.*)')
                            request_mes = log_info["request"]
                            request_api = reg.search(request_mes)
                            request_api = request_api.groupdict()
                            api = request_api["request"]
                            # 取出时区
                            if time_zone == '+0800':
                                # print(time_zone)
                                #print(data_time)
                                ctime = time.strptime(data_time, '%d/%b/%Y:%H:%M:%S')
                                # 转换为格式化时间 24/Aug/2018:15:42:08
                                #print(ctime)
                                time_stamp = time.mktime(ctime)
                                # 转换为时间戳
                                #print(time_stamp)
                                if time.time() - time_stamp <= 600:
                                    # 观测的时间间隔
                                    if api in dic.keys():
                                        dic[api] += 1
                                    else:
                                        dic[api] = 1
                                    #num += 1
                                    #print(datatime)
                                else:
                                     #print("error data_time:%s"%datatime)
                                     pass
                            else:
                                print("log format error")
                        except:
                            pass
                    else:
                        pass
                f.close()
                print(dic)
                if dic.keys():
                    for k,v in dic.items():
                        time_file = time.strftime('%Y%m%d%H',ctime)
                        date_time = time.strftime('%Y-%-m-%d:%H:%M:%S',ctime)
                        f = open("new" + self.bf_name +"-" + self.name + "-"+ time_file + ".log","a+")
                        f.write(str(date_time) + " " + str(v) +" "+ str(k) +"\n")
                    f.close()
                 
     
    if __name__ == '__main__':
        obj_StatusPV1 = StatusPV("api","test.com")
        obj_StatusPV2 = StatusPV("api","test2.com")
        obj_StatusPV1.count_pv()
        obj_StatusPV2.count_pv()

输出结果如下：

2021-7-01:09:54:28  7324  /api/user/account?apiVersion=v1&langCode=zh_TW&sourceType=app
    2021-7-01:09:54:28  2  /api/entrust/user/list?apiVersion=v1&type=history&limit=10&offset=0&typeone=0&querypath=center&langCode=zh_TW&sourceType=app
    2021-7-01:09:54:28  618  /api/user/account?apiVersion=v1&langCode=zh_CN&sourceType=app
    2021-7-01:09:54:28  148  /api/user/getUser?apiVersion=v1&langCode=zh_CN&sourceType=app
    2021-7-01:09:54:28  146  /otc/view/otc/user/accountList?apiVersion=v1&langCode=zh_TW&sourceType=app
    2021-7-01:09:54:28  155  /api/entrust/user/rlist?apiVersion=v1&langCode=zh_TW&sourceType=app
    2021-7-01:09:54:28  238  /api/config/getAppFileVersion?appType=0&langCode=tw
    2021-7-01:09:54:28  1593  /deep
    2021-7-01:09:54:28  1918  /api/article/getNovice?apiVersion=v1&categoryId=116&langCode=zh_TW&sourceType=app
    2021-7-01:09:54:28  1285  /api/user/getUser?apiVersion=v1&langCode=zh_TW&sourceType=app
    2021-7-01:09:54:28  1974  /api/config/banner?source=1&version=v1&langCode=zh_TW&sourceType=app
    2021-7-01:09:54:28  1345  /api/msgAndWorkOrder/user/userMessageCount?apiVersion=v1&langCode=zh_TW&sourceType=app
    2021-7-01:09:54:28  5825  /check
    2021-7-01:09:54:28  1971  /api/article/loadNoticeInfo?apiVersion=v1&typeName=pingtaigonggao&langCode=zh_TW&sourceType=app