import requests from requests.auth import HTTPBasicAuth import json from credentials import username, password, apm_server from influxdb import InfluxDBClient import argparse import time import datetime from api_fns import * import socket datato = (int(time.time()))-(1*60) datafrom = int(datato - (7*60)) def process(influxhost, influxport,myResponse): # load a dictionary with the applianceGuid to human readable r1 = get_org() appliancelookup = {} if r1.status_code == requests.codes.ok: for organization in r1.json(): r2 = get_appliance(organization['id']) if r2.status_code == requests.codes.ok: for appliance in r2.json(): appliancelookup[appliance['guid']]=appliance['name'] else: print_err(r2) else: print_err(r1) webpathconvert = {} r1 = get_web_app_group() if r1.status_code == requests.codes.ok: for group in r1.json(): r2 = get_web_path(group['id']) if r2.status_code == requests.codes.ok: for web_path in r2.json(): webpathconvert[web_path['id']]=web_path['target']['url'] else: print_err(r2) else: print_err(r1) # Load dictionary with full data query for dns # For successful API call, response code will be 200 (OK) if(myResponse.ok): jData = json.loads(myResponse.content) print("The response contains {0} properties".format(len(jData))) print("\n") # Open up each measurement set for key in jData: # clear tags tags={} fields = {} if (key): series = key.get("series") if (series): for key2 in series: data = key2.get("data") print "key : ",key,"\n" print "series : ",series,"\n" print "data : ",data,"\n" if (data): tags={} tags["interface"]=key.get("interface") tags["targetDomain"]=key.get("targetDomain") tags["applianceGuid"]=key.get("applianceGuid") tags["applianceName"]=appliancelookup.get(tags["applianceGuid"]) tags["webPathIds"]=key.get("webPathIds") testeroni = key2.get("timestamp") for record in data: fields={} recordcontentsAAAA = record.get("AAAA") tags["resolvedIp"]=recordcontentsAAAA.get("resolvedIp") tags["responseCode"]=recordcontentsAAAA.get("responseCode") tags["resolutionTime"]=recordcontentsAAAA.get("resolutionTime") tempdnsserver=socket.gethostbyaddr(record.get("dnsServer")) tags["dnsServer"]=tempdnsserver[0] tags["recordtype"]="AAAA" # fields[record.get("dnsServer")+"-"+tags["recordtype"]]=recordcontentsAAAA.get("resolutionTime") fields["AAAA"] = recordcontentsAAAA.get("resolutionTime") # print "record contents AAAA : ",recordcontentsAAAA,"\n" for keys,vals in fields.items(): print keys, "=>", vals for keys,vals in tags.items() : print keys, "=>", vals if recordcontentsAAAA.get("resolutionTime"): writeinflux ("dns",tags,datetime.datetime.utcfromtimestamp(testeroni/1000).strftime('%Y-%m-%dT%H:%M:%S%Z'),fields,influxhost,influxport) print fields={} recordcontentsA = record.get("A") tags["resolvedIp"]=recordcontentsA.get("resolvedIp") tags["responseCode"]=recordcontentsA.get("responseCode") tags["resolutionTime"]=recordcontentsA.get("resolutionTime") tempdnsserver=socket.gethostbyaddr(record.get("dnsServer")) tags["dnsServer"]=tempdnsserver[0] tags["recordtype"]="A" # fields[record.get("dnsServer")+"-"+tags["recordtype"]]=recordcontentsA.get("resolutionTime") fields["A"] = recordcontentsA.get("resolutionTime") # print "record contents A : ",recordcontentsA,"\n" for keys,vals in fields.items(): print keys, "=>", vals for keys,vals in tags.items() : print keys, "=>", vals if recordcontentsA.get("resolutionTime"): writeinflux ("dns",tags,datetime.datetime.utcfromtimestamp(testeroni/1000).strftime('%Y-%m-%dT%H:%M:%S%Z'),fields,influxhost,influxport) print data = {} series = {} fields = {} tags = {} print else: # If response code is not ok (200), print the resulting http error code with description myResponse.raise_for_status() def main(influxhost, influxport,myResponse): url = "https://"+apm_server+"/api/v3/dns/webPath/data?from="+str(datafrom)+"&to="+str(datato)+"&page=1&api_key=v3" url2 = "https://"+apm_server+"/api/v3/dns/webPath/data?from="+str(datafrom)+"&to="+str(datato)+"&page=2&api_key=v3" url3 = "https://"+apm_server+"/api/v3/dns/webPath/data?from="+str(datafrom)+"&to="+str(datato)+"&page=3&api_key=v3" url4 = "https://"+apm_server+"/api/v3/dns/webPath/data?from="+str(datafrom)+"&to="+str(datato)+"&page=4&api_key=v3" url5 = "https://"+apm_server+"/api/v3/dns/webPath/data?from="+str(datafrom)+"&to="+str(datato)+"&page=5&api_key=v3" myResponse = requests.get(url,auth=HTTPBasicAuth(username, password), verify=True) myResponse2 = requests.get(url2,auth=HTTPBasicAuth(username, password), verify=True) myResponse3 = requests.get(url3,auth=HTTPBasicAuth(username, password), verify=True) #print (len(myResponse("targetDomain")) #if (len(myResponse) == 100): process(influxhost,influxport,myResponse) process(influxhost,influxport,myResponse2) process(influxhost,influxport,myResponse3) # myResponse4 = requests.get(url4,auth=HTTPBasicAuth(username, password), verify=True) # process(influxhost,influxport,myResponse4) # myResponse5 = requests.get(url5,auth=HTTPBasicAuth(username, password), verify=True) # process(influxhost,influxport,myResponse5) #myResponse = dict(myResponse.items() + myResponse2.items() + myResponse3.items() + myResponse4.items() + myResponse5.items()) #myResponse.update(requests.get(url2,auth=HTTPBasicAuth(username, password), verify=True)) #myResponse.update(requests.get(url3,auth=HTTPBasicAuth(username, password), verify=True)) #myResponse.update(requests.get(url4,auth=HTTPBasicAuth(username, password), verify=True)) #myResponse.update(requests.get(url5,auth=HTTPBasicAuth(username, password), verify=True)) #print (myResponse.status_code) # labels, tags, time, count def writeinflux(measurement,tags,time,fields,influxhost,influxport): influxuser = 'CMWrite' influxpass = 'CMWrite' databasename = 'ASPM' client = InfluxDBClient(host=influxhost, port=influxport, username=influxuser, password=influxpass, database=databasename) json_body = [ { "measurement": measurement, "tags": tags, "time": time, "fields": fields } ] # client.write_points(json_body,time_precision='ms') if client.write_points(json_body): print("Write points: {0}".format(json_body,time_precision='s')) #,time_precision='s' else: print("ERROR--> Write points: {0}".format(json_body)) # Allow command line overrides for host and port influxDB def parse_args(): parser = argparse.ArgumentParser( description='Write DNS data from Appneta into InfluxDB database') parser.add_argument('--host', type=str, required=False, default='itmetricsdb.mathworks.com', help='hostname of InfluxDB http API') parser.add_argument('--port', type=int, required=False, default=8086, help='port of InfluxDB http API') return parser.parse_args() # grab arguments if __name__ == '__main__': args = parse_args() main(influxhost=args.host, influxport=args.port, myResponse={})