Upload files to "/"

apdex_new.py

@@ -0,0 +1,147 @@
+import requests
+from requests.auth import HTTPBasicAuth
+import json
+from credentials import username, password, apm_server
+from influxdb import InfluxDBClient
+import argparse
+import time
+import datetime
+from api_fns import *
+import re
+
+datato = (int(time.time()))-(1*60)
+datafrom = int(datato - (7*60))
+
+
+def process(influxhost, influxport,debug,myResponse):
+# get the organiztion id
+    r1 = get_org()
+    tags={}
+    fields ={}
+    if r1.status_code == requests.codes.ok:
+        for organization in r1.json():
+            print('organization:', organization )
+            #get the web apps for the particular organization
+            r2 = get_web_app_group(organization['id'])
+            if r2.status_code == requests.codes.ok:
+                for group in r2.json():
+                    #get the monitoring points for the web app
+                    r3 = get_web_path(group['id'])
+                    if r3.status_code == requests.codes.ok:
+                        for web_path in r3.json():
+                            #for analysis considering only events without any error messages 
+                            #if (web_path['status']=='OK' and web_path['statusWithMuted']=='OK' and web_path['errorMsg']=='' ):
+
+                            tags['appliance_name'] = web_path['location']['applianceName']
+                            location = web_path['location']['applianceName']
+                            
+                            if 'appneta' in location:
+                                for m in re.finditer('-', location):
+                                    last_index = m.start()
+                                tags['location'] = location[last_index+1:].upper()
+                            else:
+                                tags['location'] = location
+
+                            tags['web_App_id'] = web_path['webPathConfig']['webAppId']
+                            tags['target_url']=web_path['target']['url']
+                            milestones = web_path['userFlow']['milestones']
+                            tags['web_Path_id'] = web_path['id']
+                            web_AppName = web_path['webPathConfig']['webAppName']
+                            if '_' in web_AppName and 'WIP' not in web_AppName:
+                                for m in re.finditer('_', web_AppName):
+                                     match = m.start()
+                                tags['Web_Application'] = web_AppName[:match]
+                            else:
+                                tags['Web_Application'] = web_AppName
+                            
+                            tags['web_AppName_Region'] = web_path['webPathConfig']['webAppName']
+                            #fields['Apdex_score'] = web_path['latestMeasurements']['apdexScore']['value']
+                            #fields['Response_time'] = web_path['latestMeasurements']['responseTime']['value']
+                            # fields['Total_time_latest'] = web_path['latestMeasurements']['totalTime']['value']
+                            metrics = 'networktiming,servertiming,browsertiming,apdexscore'
+                            try:
+                                #get breakdown of network, server & browser timing ,apdex score for each milestone 
+                                r4 = get_web_path_stats_id(group['id'], web_path['id'], metric=metrics)
+                                if r4.status_code == requests.codes.ok:
+                                    for i in range ((len(milestones))):
+                                        
+                                        tags['Milestone'] = milestones[i]
+                                        network_timings = r4.json()['milestones'][i]['networkTiming']
+                                        server_timings = r4.json()['milestones'][i]['serverTiming']
+                                        browser_timings = r4.json()['milestones'][i]['browserTiming']
+                                        apdex_score = r4.json()['milestones'][i]['apdexScore']
+                                    
+                                        for i in range((len(network_timings))):
+                                            start_time = network_timings[i]['start'] 
+                                            # s, ms = divmod(start_time, 1000)  # (1236472051, 807)
+                                            # fields['timestamp'] = '%s.%03d' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(s)), ms)
+
+                                            #time conversion to format acceptable by InfluxDb 
+                                            timestamp = datetime.datetime.utcfromtimestamp(start_time/1000).strftime('%Y-%m-%dT%H:%M:%S%Z')
+                                            fields['apdexScore'] = apdex_score[i]['value']
+                                            fields['networkTiming'] = network_timings[i]['value']
+                                            fields['serverTiming'] = server_timings[i]['value']
+                                            fields['browserTiming'] = browser_timings[i]['value']
+                                            fields['Total_time_permilestone'] = fields['networkTiming'] + fields['serverTiming'] + fields['browserTiming']
+                                            
+                                            if (debug):print(tags)
+                                            if (debug):print(fields)
+                                            writeinflux ("apdex_responseTime",tags,timestamp,fields,influxhost,influxport)
+                                else:
+                                    print_err(r4)
+                            except:
+                                print('Error Message: ', web_path['status'] )
+
+                            # except:
+                            #     print('Error message')
+                            
+                            tags={}
+                            fields ={}
+
+                    else:
+                        print_err(r3)
+            else:
+                print_err(r2)       
+    else:
+        print_err(r1)
+
+
+def main(influxhost, influxport,debug,myResponse,):
+    url = "https://"+apm_server+"/api/v3/webApplication?from="+str(datafrom)+"&to="+str(datato)+"&api_key=v3"    
+    myResponse = requests.get(url,auth=HTTPBasicAuth(username, password), verify=True)
+    process(influxhost,influxport,debug,myResponse)
+
+    
+# labels, tags, time, count
+def writeinflux(measurement,tags,time,fields,influxhost,influxport):
+    influxuser = 'CMWrite'
+    influxpass = 'CMWrite'
+    databasename = 'ASPM'
+    client = InfluxDBClient(host=influxhost, port=influxport, username=influxuser, password=influxpass, database=databasename)
+    json_body = [
+    {
+        "measurement": measurement,
+        "tags": tags, 
+        "time": time,
+        "fields": fields
+    }
+    ]
+#    client.write_points(json_body,time_precision='ms')
+    if client.write_points(json_body): print("Write points: {0}".format(json_body,time_precision='s')) #,time_precision='s'
+    else: print("ERROR--> Write points: {0}".format(json_body))
+
+# Allow command line overrides for host and port influxDB
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='Write WebApplication data from Appneta into InfluxDB database')
+    parser.add_argument('--host', type=str, required=False, default='itmetricsdb.mathworks.com',
+                        help='hostname of InfluxDB http API')
+    parser.add_argument('--port', type=int, required=False, default=8086,
+                        help='port of InfluxDB http API')
+    parser.add_argument('--debug', action='store_true')
+    return parser.parse_args()
+
+# grab arguments
+if __name__ == '__main__':
+    args = parse_args()
+    main(influxhost=args.host, influxport=args.port,debug=args.debug, myResponse={})

influx2mssqlnew.py

@@ -0,0 +1,53 @@
+import sys
+import time
+import datetime
+import argparse
+import os.path
+from influxdb import InfluxDBClient
+import pandas as pd
+import pymssql
+from sqlalchemy import create_engine, MetaData, Table, select
+from datetime import date, timedelta
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='Sync MSSQL for PowerBI with InfluxDB database')
+    parser.add_argument('--influxhost', type=str, required=False, default='itmetricsdb.mathworks.com',
+                        help='hostname of InfluxDB http API')
+    parser.add_argument('--influxport', type=int, required=False, default=8086,
+                        help='port of InfluxDB http API')
+    parser.add_argument('--mssqlhost', type=str, required=False, default='itmr-00-ah.mathworks.com',
+                        help='hostname of Microsoft SQL server holding PowerBI data')
+    parser.add_argument('--mssqldb', type=str, required=False, default='ASPM',
+                        help='MSSQL Table Name of data to be synced from InfluxDB')
+    parser.add_argument('--mssqltable', type=str, required=False, default='ApdexScore',
+                        help='MSSQL Table Name of data to be synced from InfluxDB')
+    parser.add_argument('--days', type=str, required=False, default=1,
+                        help='Number of days to sync - will go from midnight UTC prior day back this many days')
+    return parser.parse_args()
+
+#  Main 
+def main(influxhost, influxport):
+    yesterday = str(date.today() - timedelta(90))
+    today = str(date.today())
+    # create a configuration
+    # Set influxDB data and connect
+    influxuser = 'CMWrite'
+    influxpass = 'CMWrite'
+    mssqluser = 'apdex'
+    mssqlpass = 'apdex'
+    databasename = mssqldb
+    client = InfluxDBClient(host=influxhost, port=influxport, username=influxuser, password=influxpass, database=databasename)  
+    query = "select * from ApdexScore where time >= \'" + yesterday + "\' and time < \'" + today + "\'"
+    points = client.query(query, chunked=True, chunk_size=10000).get_points()
+    df = pd.DataFrame(points)
+    print df
+    ## instance a python db connection object- same form as psycopg2/python-mysql drivers also
+    engine = create_engine('mssql+pyodbc://apdex:apdex@' + 'itmr-00-ah' + '/' + 'apdex?driver=SQL+Server+Native+Client+11.0')
+    conn = engine.connect()
+    df.to_sql(con=engine, name='apdex', if_exists='replace')
+    
+# grab arguments
+if __name__ == '__main__':
+    args = parse_args()
+    main(influxhost=args.influxhost, influxport=args.influxport, mssqlhost=args.mssqlhost, mssqltable=args.mssqltable, days=args.days, mssqldb=args.mssqldb)