myproject/pyproject/AliyunTools/ECSMonotor/ECSMonitorApi.py

# coding: utf-8

import json
import time
from datetime import datetime, timezone
from pprint import pprint
from config import Config
from word_generater import decodestr
from influxdb import InfluxDBClient
from aliyunsdkcore.client import AcsClient
from aliyunsdkcore.acs_exception.exceptions import ClientException
from aliyunsdkcore.acs_exception.exceptions import ServerException
from aliyunsdkcms.request.v20190101.DescribeMetricLastRequest import DescribeMetricLastRequest
from aliyunsdkecs.request.v20140526.DescribeInstancesRequest import DescribeInstancesRequest


class ECSData2InfluxDB():
    def __init__(self, region):
        self.region = region
        self.client = AcsClient(decodestr(Config.accessKeyId), decodestr(
            Config.accessSecret), self.region)
        self.influxclient = InfluxDBClient(
            '127.0.0.1', 8086, 'root', 'root', 'aliyun')

    @property
    def doAaction(self):
        self.request.set_accept_format('json')
        response = self.client.do_action_with_exception(self.request)
        return json.loads(str(response, encoding='utf-8'))

    ''' data['Datapoints']
     {'Average': 0,
  'Maximum': 0,
  'Minimum': 0,
  'instanceId': 'i-2ze551ld3zvhp56r24vy',
  'timestamp': 1575360720000,
  'userId': '1655840174173842'},
 {'Average': 0,
  'Maximum': 0,
  'Minimum': 0,
  'instanceId': 'i-2ze551ld3zvhp56r24uw',
  'timestamp': 1575360720000,
  'userId': '1655840174173842'}]
    '''
    @property
    def DescribeECSInternetOutRate(self):
        self.request = DescribeMetricLastRequest()
        self.request.set_MetricName(self.action)
        self.request.set_Period("60")
        self.request.set_Namespace("acs_ecs_dashboard")
        self.request.set_Express("{\"groupby\":[\"instanceId\"]}")
        self.request.set_Length("10000")
        data = self.doAaction
        # with open("DescribeECSInternetOutRate.log", 'w') as f:
        #     f.write(str(data))
        return eval(data.get('Datapoints'))

    def DescribeInstances(self, InstanceIds):
        self.request = DescribeInstancesRequest()
        self.request.set_PageSize(100)
        # InstanceIds ["i-2ze551ld3zvhp56r2504"]
        self.request.set_InstanceIds(InstanceIds)
        Data = self.doAaction
        InstancesInfo = {}
        for instance in Data['Instances']["Instance"]:
            # pprint(instance)
            if instance.get("NetworkInterfaces"):
                PrivateIpAddress = instance['NetworkInterfaces']["NetworkInterface"][0]["PrimaryIpAddress"]
            else:
                PrivateIpAddress = instance["InnerIpAddress"]["IpAddress"][0]
            if instance.get('PublicIpAddress') and len(instance.get('PublicIpAddress')) > 1:
                PublicIpAddress = instance["PublicIpAddress"]["IpAddress"][0]
            elif instance.get('EipAddress'):
                IpAddress = instance["EipAddress"]["IpAddress"]
                if isinstance(IpAddress, str):
                    PublicIpAddress = instance["EipAddress"]["IpAddress"]
                else:
                    PublicIpAddress = instance["EipAddress"]["IpAddress"][0]
            else:
                PublicIpAddress = ""
            #PublicIpAddress = instance["PublicIpAddress"]["IpAddress"][0]
            InstanceId = instance["InstanceId"]
            InstanceChargeType = instance["InstanceChargeType"]
            InstancesInfo[InstanceId] = {"PublicIpAddress": PublicIpAddress, "PrivateIpAddress": PrivateIpAddress,
                                         "InstanceChargeType": InstanceChargeType, 'InstanceId': InstanceId}
            # InstancesInfo.append({'InstanceId':InstanceId, "PublicIpAddress": PublicIpAddress, "PrivateIpAddress":PrivateIpAddress, "InstanceChargeType":InstanceChargeType})
        return InstancesInfo

    @property
    def RearrangeData(self):
        ecsdatalist = self.DescribeECSInternetOutRate
        ecsdatadict = {}
        for data in ecsdatalist:
            ecsdatadict[data['instanceId']] = data
        instanceids = [x['instanceId'] for x in ecsdatalist]
        instanceids = [instanceids[i:i+100]
                       for i in range(0, len(instanceids), 100)]
        for arr in instanceids:
            Ipinfos = self.DescribeInstances(arr)
            for data in Ipinfos.keys():
                ecsdatadict[data]['PrivateIpAddress'] = Ipinfos[data]['PrivateIpAddress']
                ecsdatadict[data]['PublicIpAddress'] = Ipinfos[data]['PublicIpAddress']
                #instanceid = data['instanceId']
                # print(instanceid)
                #PrivateIpAddress = Ipinfos[instanceid]['PrivateIpAddress']
                #PublicIpAddress = Ipinfos[instanceid]['PublicIpAddress']
                #data['PrivateIpAddress'] = PrivateIpAddress
                #data['PublicIpAddress'] = PublicIpAddress
        return ecsdatadict

    @property
    def Save2Influx(self):
        data = self.RearrangeData
        influx_jsons = self.GenInfluxJson(data)
        # pprint(influx_jsons)
        res = self.influxclient.write_points(influx_jsons)
        return res
    
    def GenInfluxJson(self, data):
        influx_jsons = []
        table = self.action
        n = 0.01
        if self.action == "VPC_PublicIP_InternetOutRate":
            for k, v in data.items():
                if v.get("PublicIpAddress"):
                    timestamp = int(str(v['timestamp'])[:10])
                    if int(str(timestamp+n)[-1]) == 9:
                        n += 0.02
                    jsonbody = {"measurement": table,
                            "time": datetime.fromtimestamp(timestamp+n, timezone.utc).isoformat("T"),
                            "tags": {"PrivateIpAddress": v["PrivateIpAddress"]
                                    },
                            "fields": {"value": int(v['Average'])
                                    }
                            }
                    n += 0.01
                    influx_jsons.append(jsonbody)
#        if self.action == "cpu_total" or self.action == "IntranetOutRate":
        else:
            for k, v in data.items():
                if v.get("PrivateIpAddress") and "10.0." in v.get("PrivateIpAddress"):
                    timestamp = int(str(v['timestamp'])[:10])
                    if int(str(timestamp+n)[-1]) == 9:
                        n += 0.02
                    jsonbody = {"measurement": table,
                            "time": datetime.fromtimestamp(timestamp+n, timezone.utc).isoformat("T"),
                            "tags": {"PrivateIpAddress": v["PrivateIpAddress"]
                                    },
                            "fields": {"value": int(v['Average'])
                                }
                        }
                    n += 0.01
                    influx_jsons.append(jsonbody)
        return influx_jsons

    def Run(self):
        actions = ["VPC_PublicIP_InternetOutRate", "IntranetOutRate", "cpu_total", "memory_usedutilization", "net_tcpconnection","load_5m"]
        for action in actions:
            self.action = action
            self.Save2Influx

if __name__ == "__main__":

    while 1:
        ins = ECSData2InfluxDB("cn-beijing")
        ins.Run()
        time.sleep(30)