From e5cdf48914966ef1db78a2ab4550195675dff344 Mon Sep 17 00:00:00 2001 From: admin Date: Sun, 3 Aug 2025 15:52:04 +0000 Subject: [PATCH] Upload files to "/" --- apdex_new.py | 147 +++++++++++++++++++++++++++++++++++++++++++++ influx2mssqlnew.py | 53 ++++++++++++++++ 2 files changed, 200 insertions(+) create mode 100644 apdex_new.py create mode 100644 influx2mssqlnew.py diff --git a/apdex_new.py b/apdex_new.py new file mode 100644 index 0000000..ec7f50c --- /dev/null +++ b/apdex_new.py @@ -0,0 +1,147 @@ +import requests +from requests.auth import HTTPBasicAuth +import json +from credentials import username, password, apm_server +from influxdb import InfluxDBClient +import argparse +import time +import datetime +from api_fns import * +import re + +datato = (int(time.time()))-(1*60) +datafrom = int(datato - (7*60)) + + +def process(influxhost, influxport,debug,myResponse): +# get the organiztion id + r1 = get_org() + tags={} + fields ={} + if r1.status_code == requests.codes.ok: + for organization in r1.json(): + print('organization:', organization ) + #get the web apps for the particular organization + r2 = get_web_app_group(organization['id']) + if r2.status_code == requests.codes.ok: + for group in r2.json(): + #get the monitoring points for the web app + r3 = get_web_path(group['id']) + if r3.status_code == requests.codes.ok: + for web_path in r3.json(): + #for analysis considering only events without any error messages + #if (web_path['status']=='OK' and web_path['statusWithMuted']=='OK' and web_path['errorMsg']=='' ): + + tags['appliance_name'] = web_path['location']['applianceName'] + location = web_path['location']['applianceName'] + + if 'appneta' in location: + for m in re.finditer('-', location): + last_index = m.start() + tags['location'] = location[last_index+1:].upper() + else: + tags['location'] = location + + tags['web_App_id'] = web_path['webPathConfig']['webAppId'] + tags['target_url']=web_path['target']['url'] + milestones = web_path['userFlow']['milestones'] + tags['web_Path_id'] = web_path['id'] + web_AppName = web_path['webPathConfig']['webAppName'] + if '_' in web_AppName and 'WIP' not in web_AppName: + for m in re.finditer('_', web_AppName): + match = m.start() + tags['Web_Application'] = web_AppName[:match] + else: + tags['Web_Application'] = web_AppName + + tags['web_AppName_Region'] = web_path['webPathConfig']['webAppName'] + #fields['Apdex_score'] = web_path['latestMeasurements']['apdexScore']['value'] + #fields['Response_time'] = web_path['latestMeasurements']['responseTime']['value'] + # fields['Total_time_latest'] = web_path['latestMeasurements']['totalTime']['value'] + metrics = 'networktiming,servertiming,browsertiming,apdexscore' + try: + #get breakdown of network, server & browser timing ,apdex score for each milestone + r4 = get_web_path_stats_id(group['id'], web_path['id'], metric=metrics) + if r4.status_code == requests.codes.ok: + for i in range ((len(milestones))): + + tags['Milestone'] = milestones[i] + network_timings = r4.json()['milestones'][i]['networkTiming'] + server_timings = r4.json()['milestones'][i]['serverTiming'] + browser_timings = r4.json()['milestones'][i]['browserTiming'] + apdex_score = r4.json()['milestones'][i]['apdexScore'] + + for i in range((len(network_timings))): + start_time = network_timings[i]['start'] + # s, ms = divmod(start_time, 1000) # (1236472051, 807) + # fields['timestamp'] = '%s.%03d' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(s)), ms) + + #time conversion to format acceptable by InfluxDb + timestamp = datetime.datetime.utcfromtimestamp(start_time/1000).strftime('%Y-%m-%dT%H:%M:%S%Z') + fields['apdexScore'] = apdex_score[i]['value'] + fields['networkTiming'] = network_timings[i]['value'] + fields['serverTiming'] = server_timings[i]['value'] + fields['browserTiming'] = browser_timings[i]['value'] + fields['Total_time_permilestone'] = fields['networkTiming'] + fields['serverTiming'] + fields['browserTiming'] + + if (debug):print(tags) + if (debug):print(fields) + writeinflux ("apdex_responseTime",tags,timestamp,fields,influxhost,influxport) + else: + print_err(r4) + except: + print('Error Message: ', web_path['status'] ) + + # except: + # print('Error message') + + tags={} + fields ={} + + else: + print_err(r3) + else: + print_err(r2) + else: + print_err(r1) + + +def main(influxhost, influxport,debug,myResponse,): + url = "https://"+apm_server+"/api/v3/webApplication?from="+str(datafrom)+"&to="+str(datato)+"&api_key=v3" + myResponse = requests.get(url,auth=HTTPBasicAuth(username, password), verify=True) + process(influxhost,influxport,debug,myResponse) + + +# labels, tags, time, count +def writeinflux(measurement,tags,time,fields,influxhost,influxport): + influxuser = 'CMWrite' + influxpass = 'CMWrite' + databasename = 'ASPM' + client = InfluxDBClient(host=influxhost, port=influxport, username=influxuser, password=influxpass, database=databasename) + json_body = [ + { + "measurement": measurement, + "tags": tags, + "time": time, + "fields": fields + } + ] +# client.write_points(json_body,time_precision='ms') + if client.write_points(json_body): print("Write points: {0}".format(json_body,time_precision='s')) #,time_precision='s' + else: print("ERROR--> Write points: {0}".format(json_body)) + +# Allow command line overrides for host and port influxDB +def parse_args(): + parser = argparse.ArgumentParser( + description='Write WebApplication data from Appneta into InfluxDB database') + parser.add_argument('--host', type=str, required=False, default='itmetricsdb.mathworks.com', + help='hostname of InfluxDB http API') + parser.add_argument('--port', type=int, required=False, default=8086, + help='port of InfluxDB http API') + parser.add_argument('--debug', action='store_true') + return parser.parse_args() + +# grab arguments +if __name__ == '__main__': + args = parse_args() + main(influxhost=args.host, influxport=args.port,debug=args.debug, myResponse={}) \ No newline at end of file diff --git a/influx2mssqlnew.py b/influx2mssqlnew.py new file mode 100644 index 0000000..b7c6b1b --- /dev/null +++ b/influx2mssqlnew.py @@ -0,0 +1,53 @@ +import sys +import time +import datetime +import argparse +import os.path +from influxdb import InfluxDBClient +import pandas as pd +import pymssql +from sqlalchemy import create_engine, MetaData, Table, select +from datetime import date, timedelta + +def parse_args(): + parser = argparse.ArgumentParser( + description='Sync MSSQL for PowerBI with InfluxDB database') + parser.add_argument('--influxhost', type=str, required=False, default='itmetricsdb.mathworks.com', + help='hostname of InfluxDB http API') + parser.add_argument('--influxport', type=int, required=False, default=8086, + help='port of InfluxDB http API') + parser.add_argument('--mssqlhost', type=str, required=False, default='itmr-00-ah.mathworks.com', + help='hostname of Microsoft SQL server holding PowerBI data') + parser.add_argument('--mssqldb', type=str, required=False, default='ASPM', + help='MSSQL Table Name of data to be synced from InfluxDB') + parser.add_argument('--mssqltable', type=str, required=False, default='ApdexScore', + help='MSSQL Table Name of data to be synced from InfluxDB') + parser.add_argument('--days', type=str, required=False, default=1, + help='Number of days to sync - will go from midnight UTC prior day back this many days') + return parser.parse_args() + +# Main +def main(influxhost, influxport): + yesterday = str(date.today() - timedelta(90)) + today = str(date.today()) + # create a configuration + # Set influxDB data and connect + influxuser = 'CMWrite' + influxpass = 'CMWrite' + mssqluser = 'apdex' + mssqlpass = 'apdex' + databasename = mssqldb + client = InfluxDBClient(host=influxhost, port=influxport, username=influxuser, password=influxpass, database=databasename) + query = "select * from ApdexScore where time >= \'" + yesterday + "\' and time < \'" + today + "\'" + points = client.query(query, chunked=True, chunk_size=10000).get_points() + df = pd.DataFrame(points) + print df + ## instance a python db connection object- same form as psycopg2/python-mysql drivers also + engine = create_engine('mssql+pyodbc://apdex:apdex@' + 'itmr-00-ah' + '/' + 'apdex?driver=SQL+Server+Native+Client+11.0') + conn = engine.connect() + df.to_sql(con=engine, name='apdex', if_exists='replace') + +# grab arguments +if __name__ == '__main__': + args = parse_args() + main(influxhost=args.influxhost, influxport=args.influxport, mssqlhost=args.mssqlhost, mssqltable=args.mssqltable, days=args.days, mssqldb=args.mssqldb) \ No newline at end of file