myproject/pyproject/RedisTools/Agent_redis_stats.py

from __future__ import division
import redis
import time
import psutil
import socket
import pymongo
from pymongo import MongoClient

redis_report_ip = '192.168.4.94'
redis_report_port = 6379
mongodb_stats = "mongodb://192.168.4.97:27017"
mongodb_info = "mongodb://192.168.4.98:27017"


class DockerStats(object):
    def __init__(self, redis_report_ip, redis_report_port, redis_port, container):
        # self.cli = Client(base_url='tcp://%s:2375' % docker_ip)
        self.cli = Client(base_url='unix://var/run/docker.sock')
        self.redis_report_ip = redis_report_ip
        self.redis_report_port = redis_report_port
        self.redis_port = redis_port
        self.container = container
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(('192.168.4.254', 0))
        self.host = s.getsockname()[0]

    @property
    def container_stat(self):
        self.stats_obj = self.cli.stats(self.container, stream=False)
        return self.stats_obj

    @property
    def docker_conns(self):
        for conn in psutil.net_connections():
            try:
                if conn[3][1] == self.redis_port:
                    redis_pid = conn[-1]
                    break
            except:
                pass
        redis_ins = psutil.Process(redis_pid)
        return len(redis_ins.connections())

    def redis_info(self):
        r = redis.StrictRedis(host=self.host, port=self.redis_port, db=0)
        info = r.info()
        redis_key = '%s:%s' % (self.host, self.redis_port)
        info['time'] = int(time.time())
        return redis_key, info

    def write_redis(self):
        r = redis.StrictRedis(host=self.redis_report_ip, port=self.redis_report_port, db=0)
        now_result = self.container_stat  # Dict
        memory_usage = now_result["memory_stats"]["usage"]
        ##pid connections###
        cpuPercent = 0.0
        TotalUsage = now_result["cpu_stats"]["cpu_usage"]["total_usage"]
        SystemUsage = now_result["cpu_stats"]["system_cpu_usage"]
        previous = '%s:%s:%s' % (self.host, self.redis_port, 'previous')
        get_previous = r.hget('redis_stats_docker', previous)
        if not get_previous:
            r.hset('redis_stats_docker', previous, [TotalUsage, SystemUsage])
        else:
            previousCPU, previousSystem = eval(r.hget('redis_stats_docker', previous))
            cpuDelta = TotalUsage - previousCPU
            systemDelta = SystemUsage - previousSystem
            r.hset('redis_stats_docker', previous, [TotalUsage, SystemUsage])
            cpuPercent = '%.2f' % (
                (cpuDelta / systemDelta) * len(self.stats_obj["cpu_stats"]["cpu_usage"]["percpu_usage"]) * 100.0)
            ######################################{'conns': 33, 'cpu_usage': '1.20', 'mem_usage': 2788642816}
        redis_key = '%s:%s' % (self.host, self.redis_port)
        redis_value = {'conns': self.docker_conns,
                       'cpu_usage': cpuPercent,
                       'mem_usage': memory_usage,
                       'time': int(time.time())
                       }
        r.hset('redis_stats', redis_key, redis_value)
        return redis_key, redis_value


class RedisStats(object):
    def __init__(self, redis_report_ip, redis_report_port, psinfo):
        self.redis_report_ip = redis_report_ip
        self.redis_report_port = redis_report_port
        self.redis = psinfo
        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        s.connect(('192.168.4.254', 0))
        self.host = s.getsockname()[0]

    @property
    def cpu_usage(self):
        self.cpu = '%.2f' % self.redis.cpu_percent(interval=0.8)
        return self.cpu

    @property
    def mem_usage(self):
        self.memory = self.redis.memory_info().rss
        return self.memory

    @property
    def conns(self):
        self.connections = self.redis.connections()
        return self.connections

    @property
    def get_redis_port(self):
        self.conns
        for conn in self.connections:
            if conn.status == 'LISTEN':
                self.port = conn.laddr[1]
                return self.port

    def redis_info(self):
        self.port = self.get_redis_port
        r = redis.StrictRedis(host=self.host, port=self.port, db=0)
        info = r.info()
        redis_key = '%s:%s' % (self.host, self.port)
        info['time'] = int(time.time())
        return redis_key, info

    def write_redis(self):
        self.cpu_usage
        self.mem_usage
        self.conns
        self.get_redis_port
        r = redis.StrictRedis(host=self.redis_report_ip, port=self.redis_report_port, db=0)
        key = '%s:%s' % (self.host, self.port)
        value = {'cpu_usage': self.cpu,
                 'mem_usage': self.memory,
                 'conns': len(self.connections),
                 'time': int(time.time())
                 }
        r.hset('redis_stats', key, value)
        return key, value


def get_container():
    contianers = []
    cli = Client(base_url='unix://var/run/docker.sock')
    for contianer in cli.containers():
        try:
            if contianer['State'] == 'running' and 'dyredis' in contianer['Image']:
                contianers.append(contianer['Names'][0][1:])
        except:
            pass
    return contianers


def get_instance_ps():
    pslist = []
    ps = psutil.process_iter()
    for psinfo in ps:
        if psinfo.name() == 'redis-server':
            pslist.append(psutil.Process(psinfo.pid))
    return pslist


def write_mongodb(redis_key, redis_value, type, isindex):
    if type == 'stats':
        client = MongoClient(mongodb_stats)
    elif type == 'info':
        client = MongoClient(mongodb_info)
    dbname = 'redis_stats' + str(redis_value['time'] // (60 * 60 * 24 * 30))
    collection_name = 'info' + str(redis_value['time'] // (60 * 60 * 24))
    redis_value.update({'instance': redis_key})
    json_data = redis_value
    db = eval('%s.%s' % ("client", dbname))
    collection = eval('%s.%s' % ("db", collection_name))
    if isindex:
        if  "instance_1" in collection.index_information().keys():
            collection.ensure_index([("instance", pymongo.ASCENDING)])
        if "time_1" in collection.index_information().keys():
            collection.ensure_index([("time", pymongo.ASCENDING)])
    collection.insert_one(json_data)
    return collection_name


def run(redis_report_ip, redis_report_port, last_collection_name, last_time, param):
    if param:
        loop = get_container()
    else:
        loop = get_instance_ps()
    try:
        for contianer in loop:
            if param:
                contianer_port = int(contianer.split('_')[-1])
                stats = DockerStats(redis_report_ip, redis_report_port, contianer_port, contianer)
            else:
                stats = RedisStats(redis_report_ip, redis_report_port, contianer)
            stats_key, stats_val = stats.write_redis()
            collection_name = 'info' + str(stats_val['time'] // (60 * 60 * 24))
            if int(time.time()) // 3600 > last_time:
                issharptime = True
                info_key, info_val = stats.redis_info()
            if collection_name == last_collection_name:
                last_collection_name = write_mongodb(stats_key, stats_val, 'stats', 0)
                last_time = int(time.time()) // 3600
                if issharptime:
                    last_collection_name = write_mongodb(info_key, info_val, 'info', 0)
            else:
                last_collection_name = write_mongodb(stats_key, stats_val, 'stats', 1)
                last_time = int(time.time()) // 3600
                if issharptime:
                    last_collection_name = write_mongodb(info_key, info_val, 'info', 1)
    except:
        pass
    return last_collection_name, last_time


if __name__ == '__main__':
    last_collection_name = ""
    last_time = 0
    try:
        from docker import Client
    except:
        sys_env = 0
    else:
        sys_env = 1
    while 1:
        last_collection_name, last_time = run(redis_report_ip, redis_report_port, last_collection_name, last_time,
                                                  sys_env)
        time.sleep(3)