#!/usr/bin/env python3
"""
gpcheckperf -- Check the hardware for Apache Cloudberry

Usage: gpcheckperf <options>

    --version  : print version information
    -d segdir  : where segdir is a directory on which to test I/O performance.
                 Multiple -d flags may be specified for multiple directories
                 on each host. These will typically be your Cloudberry primary
                 (and/or mirror) data directories.
                 e.g. -d /dbfast1 -d /dbfast2
    -?         : print this help text
    -v         : enable verbose
    -V         : enable very verbose
    -D         : print per host disk stats on disk read/write tests
    -r flags   : where flags specifies the tests to run [default=dsn].
                 Supported test flags are:
                       d: runs only disk tests
                       s: runs only stream benchmark
                       n: runs only netperf tests (serial)
                       N: runs only netperf tests (parallel)
                       M: runs only netperf tests (full matrix)
    -B size    : where size specifies the block size for disk performance tests
                 [default=32KB] e.g. 1KB, 4MB
    -S size    : where size specifies the file size for disk performance tests
                 [default=2X Memory] e.g. 500MB, 16GB
    -h host    : the host to connect to (multiple -h is okay)
    -f file    : a file listing all hosts to connect to
    --duration : how long to run network test (default 5 seconds)
    --netperf  : use netperf instead of gpnetbenchServer/gpnetbenchClient
    --buffer-size : the size of the send buffer in kilobytes ( default 8 kilobytes)
"""

import datetime
import os
import sys
import re

sys.path.append(sys.path[0] + '/lib')
try:
    import getopt, math, io, stat, subprocess, signal
    from gppylib.gpparseopts import OptParser
    from gppylib.util import ssh_utils
    from functools import reduce
except ImportError as e:
    sys.exit('Error: unable to import module: ' + str(e))

GPHOME = os.getenv('GPHOME')
if GPHOME is None:
    sys.exit('Please set GPHOME environment variable')

USER = os.getenv('USER')
if USER is None or USER == ' ':
    sys.exit('Please set USER environment variable')


def usage(exitarg):
    parser = OptParser()
    try:
        parser.print_help()
    except:
        print(__doc__)
    sys.exit(exitarg)


class Global():
    script_name = os.path.split(__file__)[-1]
    opt = {'-d': [], '-D': False, '-v': False, '-V': False, '-r': '',
           '-B': 1024 * 32, '-S': 0, '-h': [], '-f': None,
           '--duration': 15, '--net': None, '--netserver': 'gpnetbenchServer',
           '--netclient': 'gpnetbenchClient', '--buffer-size': 0}


GV = Global()


def killall(procname):
    return '(pkill %s || killall -9 %s) > /dev/null 2>&1 || true' % (procname[0:15], procname)


def strcmd(cmd):
    return reduce(lambda x, y: x + ' ' + y, map(lambda x: x.find(' ') > 0 and "'" + x + "'" or x, cmd))


def gpssh(cmd, verbose):
    c = ['%s/bin/gpssh' % GPHOME]
    if verbose:
        c.append('-v')
    if GV.opt['-f']:
        c.append('-f')
        c.append(GV.opt['-f'])
    else:
        for h in GV.opt['-h']:
            c.append('-h')
            c.append(h)
    c.append(cmd)

    if GV.opt['-v']:
        print('[Info]', strcmd(c))
    p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    out = p.stdout.read(-1).decode()
    rc = p.wait()
    return not rc, out


def gpsync(src, dst):
    c = ['%s/bin/gpsync' % GPHOME]
    if GV.opt['-V']:
        c.append('-v')
    if GV.opt['-f']:
        c.append('-f')
        c.append(GV.opt['-f'])
    else:
        for h in GV.opt['-h']:
            c.append('-h')
            c.append(h)
    c.append(src)
    c.append(dst)
    if GV.opt['-v']:
        print ('[Info]', strcmd(c))
    p = subprocess.Popen(c, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    out = p.stdout.read(-1)
    rc = p.wait()
    return not rc, out


def getPlatform():
    if sys.platform.find('linux') >= 0:
        return 'linux'
    if sys.platform.find('darwin') >= 0:
        return 'darwin'
    return '?'


def getMemory():
    platform = getPlatform()

    if platform == 'linux':
        rc, out = run("sh -c 'cat /proc/meminfo | grep MemTotal'")
        if rc != 0:
            return None
        word_list = out.strip().split(' ')
        val = int(word_list[len(word_list) - 2])
        factor = word_list[len(word_list) - 1]
        if factor == 'kB':
            return val * 1024 if val else None

    if platform == 'darwin':
        rc, out = run("/usr/sbin/sysctl hw.physmem")
        if rc != 0:
            return None
        word_list = out.strip().split(' ')
        val = int(word_list[1])
        return val if val else None

    return None


def parseMemorySize(line):
    factor = 1
    try:
        line = line.strip().upper()
        if line.endswith('B'):
            line = line[:-1]
        if line.endswith('G'):
            factor = 1024 * 1024 * 1024
        elif line.endswith('M'):
            factor = 1024 * 1024
        elif line.endswith('K'):
            factor = 1024
        if factor > 1:
            line = line[:-1]
        return int(line) * factor
    except Exception as err:
        print ('Error: ' + err)
        sys.exit(-1)


def parseDuration(val):
    factor = 1
    try:
        val = val.strip().upper()
        if val.endswith('S'):
            val = val[:-1]
        if val.endswith('M'):
            factor = 60
        if val.endswith('H'):
            factor = 3600
        if val.endswith('D'):
            factor = 86400
        if factor > 1:
            val = val[:-1]
        return int(val) * factor
    except Exception as err:
        print("exception: %s" % err)
        sys.exit(-1)


def print_version():
    print('%s version $Revision$' % GV.script_name)
    sys.exit(0)


def parseCommandLine():
    try:
        (options, args) = getopt.getopt(sys.argv[1:], '?vVDd:r:B:S:p:h:f:', ['duration=', 'version', 'netperf', 'buffer-size='])
    except Exception as e:
        usage('Error: ' + str(e))
        exit(1)

    for (switch, val) in options:
        if switch == '-?':
            usage(0)
        elif switch[1] in 'vVD':
            GV.opt[switch] = True
        elif switch[1] in 'dh':
            GV.opt[switch].append(val)
        elif switch[1] in 'rf':
            GV.opt[switch] = val
        elif switch[1] in 'BS':
            GV.opt[switch] = parseMemorySize(val)
        elif switch == '--duration':
            GV.opt[switch] = parseDuration(val)
        elif switch == '--version':
            print_version()
        elif switch == '--netperf':
            GV.opt['--netserver'] = 'netserver'
            GV.opt['--netclient'] = 'netperf'
        elif switch == '--buffer-size':
            GV.opt[switch] = int(val)

    # run default tests (if not specified)
    if GV.opt['-r'] == '':
        GV.opt['-r'] = 'dsN'

    # parse netperf test type, error if more than one specified
    netcount = 0
    if GV.opt['-r'].find('n') >= 0:
        GV.opt['--net'] = 'netperf'
        netcount += 1
    if GV.opt['-r'].find('N') >= 0:
        GV.opt['--net'] = 'parallel'
        netcount += 1
    if GV.opt['-r'].find('M') >= 0:
        GV.opt['--net'] = 'matrix'
        netcount += 1
    if netcount > 1:
        usage('Error: please only specify one network test at a time (-r [n|M|N])')

    if GV.opt['-V']:
        GV.opt['-v'] = True

    if len(GV.opt['-d']) == 0:
        usage('Error: please specify at least one segdir (-d switch)')

    if not (0 <= GV.opt['-B'] and 0 <= GV.opt['-S']):
        usage('Error: please enter valid memory sizes for -B or -S switches')

    hf = (len(GV.opt['-h']) and 1 or 0) + (GV.opt['-f'] and 1 or 0)
    if hf != 1:
        usage('Error: please specify at least one of -h or -f args, but not both')

    if (GV.opt['-B']) > 1024 * 1024:
        usage('Error: maximum size for -B parameter is 1MB')

    if GV.opt['-S'] == 0:
        system_mem_size = getMemory()
        if system_mem_size is not None:
            GV.opt['-S'] = 2 * system_mem_size / len(GV.opt['-d'])
        else:
            sys.exit('[Error] could not get system memory size. Instead, you can use the -S option to provide the file size value')

    else:
        GV.opt['-S'] /= len(GV.opt['-d'])

    if GV.opt['--duration'] <= 0:
        GV.opt['--duration'] = 15
        print('[INFO] Invalid network duration specified.  Using default (15 seconds)')

    if GV.opt['--netclient'].find('netperf') >= 0:
        if GV.opt['--buffer-size']:
            print('[Warning] --buffer-size option will be ignored when the --netperf option is enabled')
    else:
        if GV.opt['--buffer-size'] <= 0:
            print('[INFO] --buffer-size value is not specified or invalid. Using default (8 kilobytes)')
            GV.opt['--buffer-size'] = 8

    # strip the last '/' from the dir
    dd = []
    for d in GV.opt['-d']:
        if d and d[-1] == '/':
            d = d[:-1]
        dd.append(d)
    GV.opt['-d'] = dd

    def strcmd(cmd):
        return reduce(lambda x, y: x + ' ' + y, map(lambda x: x.find(' ') > 0 and "'" + x + "'" or x, cmd))

    print(strcmd(sys.argv))


def run(cmd):
    try:
        if GV.opt['-v']:
            print('[Info]', cmd)
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
        out = p.stdout.read(-1).decode()
        ok = p.wait()
    except KeyboardInterrupt:
        raise

    if GV.opt['-v'] and out and out != '':
        print(out)

    if out and out.find('Permission denied') >= 0 and out.find('publickey') >= 0:
        if not GV.opt['-v']:
            print(out)
        print('Please use gpssh-exkeys to exchange keys ...')
        sys.exit(1)

    return ok, out


runTeardown_torndown = False


def runTeardown():
    global runTeardown_torndown
    if runTeardown_torndown:
        return
    if GV.opt['-v']:
        print('--------------------')
        print('  TEARDOWN')
        print('--------------------')
    dirs = ''
    for d in GV.opt['-d']:
        dirs = '%s %s/gpcheckperf_$USER' % (dirs, d)
    try:
        gpssh('rm -rf ' + dirs, GV.opt['-V'])
    except:
        pass

    try:
        if GV.opt['--net']:
            gpssh(killall(GV.opt['--netserver']), GV.opt['-V'])
    except:
        pass

    runTeardown_torndown = True


def runSetup():
    if GV.opt['-v']:
        print('--------------------')
        print('  SETUP %s' % datetime.datetime.now().isoformat())
        print('--------------------')
    okCount = 0
    try:
        # Verify python3 is accessible
        if GV.opt['-v']:
            print('[Info] verify python3 interpreter exists')
        (ok, out) = gpssh('python3 -c print', GV.opt['-V'])
        if not ok:
            if not GV.opt['-v']:
                print(out)
            sys.exit("[Error] unable to find python3 interpreter on some hosts\n"
                     + "        verify PATH variables on the hosts")

            # mkdir cperf
        if GV.opt['-v']:
            print('[Info] making gpcheckperf directory on all hosts ... ')
        dirs = ''
        for d in GV.opt['-d']:
            dirs = '%s %s/gpcheckperf_$USER' % (dirs, d)

        cmd = 'rm -rf %s ; mkdir -p %s' % (dirs, dirs)
        (ok, out) = gpssh(cmd, GV.opt['-V'])
        if not ok:
            print('failed gpssh: %s' % out)
            sys.exit("[Error] unable to make gpcheckperf directory. \n"
                     + "        command failed: " + cmd)

    except:
        runTeardown()
        raise


def copyExecOver(fname):
    # copy to this dir
    d = GV.opt['-d'][0]
    target = '%s/gpcheckperf_$USER/%s' % (d, fname)
    path = '%s/lib/%s' % (sys.path[0], fname)

    if not os.path.isfile(path):
        print("binary file not found: %s" % path)
        sys.exit(1)

    if GV.opt['-v']:
        print('[Info] copy local %s to remote %s' % (path, target))

    try:
        pathStat = os.stat(path)
    except:
        sys.exit('[Error] cannot stat file ' + path)
    if not stat.S_ISREG(pathStat.st_mode):
        sys.exit('[Error] invalid file ' + path)
    if not os.access(path, os.X_OK):
        sys.exit('[Exit] file not executable: ' + path)

    (ok, out) = gpsync(path, '=:%s' % target)
    if not ok:
        sys.exit('[Error] command failed: gpsync %s =:%s with output: %s' % (path, target, out))

    # chmod +x file
    (ok, out) = gpssh('chmod a+rx %s' % target, GV.opt['-V'])
    if not ok:
        sys.exit('[Error] command failed: chmod a+rx %s with output: %s' % (target, out))

    return target

def parseMultiDDResult(out):
    # parse output of "time -p"
    out = io.StringIO(out)
    result = {}
    bytes = 0
    for line in out:
        i = line.find(']')
        if i == -1:
            continue
        o = line[i + 2:]


        if o.find('multidd total bytes ') >= 0:
            h = line[1:i]
            o = o.split()
            m = re.search("(^\d+)", o[-1])
            if m is None:
                sys.exit('[Error] expected %s to be a number' % o[-1])
            bytes = int(m.group(1))
            continue

        if o.find('real') >= 0:
            h = line[1:i]
            o = o.split()
            o[1] = o[1].replace(',', '.')
            m = re.search("(^\d+.\d+)", o[1])
            if m is None:
                sys.exit('[Error] expected %s to be a floating point number' % o[1])
            time = float(m.group(1))
            if time < 0.1:
                time = 0.1  # avoid division by zero
            mbps = bytes / time / 1024 / 1024
            result[h] = (mbps, time, bytes)
            continue

    return result


def runDiskWriteTest(multidd):
    print
    print('--------------------')
    print('--  DISK WRITE TEST')
    print('--------------------')

    cmd = 'time -p ' + multidd
    for d in GV.opt['-d']:
        cmd = cmd + (' -i /dev/zero -o %s/gpcheckperf_$USER/ddfile' % d)
    if GV.opt['-B']:
        cmd = cmd + (' -B %d' % GV.opt['-B'])
    if GV.opt['-S']:
        cmd = cmd + (' -S %d' % GV.opt['-S'])
    (ok, out) = gpssh(cmd, GV.opt['-V'])
    if not ok:
        sys.exit('[Error] command failed: %s with output: %s' % (cmd, out))
    return parseMultiDDResult(out)


def runDiskReadTest(multidd):
    print
    print('--------------------')
    print('--  DISK READ TEST')
    print('--------------------')

    cmd = 'time -p ' + multidd
    for d in GV.opt['-d']:
        cmd = cmd + (' -o /dev/null -i %s/gpcheckperf_$USER/ddfile' % d)
    if GV.opt['-B']:
        cmd = cmd + (' -B %d' % GV.opt['-B'])
    if GV.opt['-S']:
        cmd = cmd + (' -S %d' % GV.opt['-S'])
    (ok, out) = gpssh(cmd, GV.opt['-V'])
    if not ok:
        sys.exit('[Error] command failed: %s with output: %s' % (cmd, out))
    return parseMultiDDResult(out)


def runStreamTest():
    print
    print('--------------------')
    print('--  STREAM TEST')
    print('--------------------')

    cmd = copyExecOver('stream')
    (ok, out) = gpssh(cmd, GV.opt['-V'])
    if not ok:
        sys.exit('[Error] command failed: %s with output: %s' % (cmd, out))
    out = io.StringIO(out)
    result = {}
    for line in out:
        i = line.find(']')
        o = line[i + 2:]
        if o.startswith('Copy:'):
            h = line[1:i]
            o = o.split()
            result[h] = (float(o[1]), 0, 0)
    return result


def startNetServer():
    port = 23000
    rmtPath = copyExecOver(GV.opt['--netserver'])

    for i in range(5):
        if i > 0:
            print('[Warning] retrying with port %d' % port)
        (ok, out) = gpssh(killall(GV.opt['--netserver']), GV.opt['-V'])

        (ok, out) = gpssh('%s -p %d > /dev/null 2>&1' % (rmtPath, port), GV.opt['-V'])
        if ok:
            return port

        if GV.opt['-v']:
            print('[Warning] start netserver with port %d failed' % port)
        port += 12

    return 0


def killProc(proc):
    if proc:
        try:
            os.kill(proc.pid, signal.SIGKILL)
        except OSError:
            pass
        return proc.wait()
    return 0


def spawnNetperfTestBetween(x, y, netperf_path, netserver_port, sec=5):
    if GV.opt['--netclient'] == 'netperf' and GV.opt['--netserver'] == 'netserver':
        cmd = ('%s -H %s -p %d -t TCP_STREAM -l %s -f M -P 0 '
               % (netperf_path, y, netserver_port, sec))
    else:
        cmd = ('%s -H %s -p %d -l %s -P 0 -b %s'
               % (netperf_path, y, netserver_port, sec, GV.opt['--buffer-size']))

    c = ['ssh', '-o', 'BatchMode yes',
         '-o', 'StrictHostKeyChecking no',
         x, cmd]
    proc = None
    try:
        if GV.opt['-v'] or GV.opt['-V']:
            print('[Info]', strcmd(c))
        proc = subprocess.Popen(c, stdout=subprocess.PIPE)
    except KeyboardInterrupt:
        killProc(proc)
        raise
    if not proc:
        print('[Warning] netperf failed on %s -> %s' % (x, y))
        return None, None, None

    return proc, x, y


def reapNetperfTest(proc, x, y):
    ok = True
    out = None
    if proc:
        try:
            rc = proc.wait()
            out = proc.stdout.read(-1).decode()
            ok = not killProc(proc)
            proc = None
        except KeyboardInterrupt as ki:
            raise
        except:
            pass
        finally:
            proc = None

    if GV.opt['-v'] and out and out != '':
        print('[Info]', out)
    if out and out.find('Permission denied') >= 0 and out.find('publickey') >= 0:
        if not GV.opt['-v']:
            print(out)
        print('[Error] Please use gpssh-exkeys to exchange keys ...')
        sys.exit(1)

    if not ok:
        print('[Warning] netperf failed on %s -> %s' % (x, y))
        return []

    for line in io.StringIO(out):
        line = line.split()
        if len(line) != 5:
            continue
        l = [x, y]
        l.extend(map(lambda x: float(x), line))
        if GV.opt['-v']:
            print('[Info] %s -> %s : %s' % (x, y, str(line)))
        return l

    print('[Warning] netperf failed on %s -> %s (invalid result)' % (x, y))
    print('[Info]', out)
    return []


def runNetperfTestBetween(x, y, netperf_path, netserver_port):
    (proc, x, y) = spawnNetperfTestBetween(x, y, netperf_path, netserver_port, GV.opt['--duration'])
    if not proc:
        return []

    return reapNetperfTest(proc, x, y)


def setupNetPerfTest():
    print
    print('-------------------')
    print('--  NETPERF TEST')
    print('-------------------')

    hostlist = ssh_utils.HostList()
    for h in GV.opt['-h']:
        hostlist.add(h)
    if GV.opt['-f']:
        hostlist.parseFile(GV.opt['-f'])

    h = hostlist.get()
    if len(h) == 0:
        usage('Error: missing hosts in -h and/or -f arguments')

    if len(h) == 1:
        print('[Warning] single host only - abandon netperf test')
        return (None, None, None)

    # start netserver
    netserver_port = startNetServer()
    if netserver_port == 0:
        print('[Error] unable to start netserver ... abort netperf test')
        return (None, None, None)

    netperf = copyExecOver(GV.opt['--netclient'])
    return (netperf, h, netserver_port)


def runNetPerfTest():
    (netperf, hostlist, netserver_port) = setupNetPerfTest()
    if not netperf:
        return None

    # make sure we have even number of hosts
    odd = None
    h = hostlist[:]
    if len(h) & 1 == 1:
        odd = h.pop()  # take one out

    # run netperf test between host[0] & host[1], host[2] & host[3], ...
    result = []
    for i in range(len(h)):
        if i & 1 == 0:
            continue
        x = h[i - 1]
        y = h[i]
        res = runNetperfTestBetween(x, y, netperf, netserver_port)
        if res and len(res) >= 6:
            result.append(res)
        res = runNetperfTestBetween(y, x, netperf, netserver_port)
        if res and len(res) >= 6:
            result.append(res)

    # run netperf test between the odd host and host 0
    if odd:
        x = odd
        y = h[0]
        res = runNetperfTestBetween(x, y, netperf, netserver_port)
        if res and len(res) >= 6:
            result.append(res)
        res = runNetperfTestBetween(y, x, netperf, netserver_port)
        if res and len(res) >= 6:
            result.append(res)

    return result


def runNetPerfTestParallel():
    (netperf, hostlist, netserver_port) = setupNetPerfTest()
    if not netperf:
        return None

    # make sure we have even number of hosts
    h = hostlist[:]
    if len(h) & 1 == 1:
        h.append(h[0])

    # run netperf test between host[0] & host[1], host[2] & host[3], ...
    result = []
    procList = []
    for i in range(len(h)):
        if i & 1 == 0:
            continue
        x = h[i - 1]
        y = h[i]
        p = spawnNetperfTestBetween(x, y, netperf, netserver_port, GV.opt['--duration'])
        if p[0]:
            procList.append(p)

    for (proc, x, y) in procList:
        res = reapNetperfTest(proc, x, y)
        if res and len(res) >= 6:
            result.append(res)

    # second round: test between host[1] & host[0], host[3] & host[2], ...
    procList = []
    for i in range(len(h)):
        if i & 1 == 0:
            continue
        x = h[i - 1]
        y = h[i]
        p = spawnNetperfTestBetween(y, x, netperf, netserver_port, GV.opt['--duration'])
        if p[0]:
            procList.append(p)

    for (proc, x, y) in procList:
        res = reapNetperfTest(proc, x, y)
        if res and len(res) >= 6:
            result.append(res)

    return result


def get_host_map(hostlist):
    '''
        Returns a seglist dictionary that maps segment name to hostname and
        a uniqhosts dictionary that maps each hostname to one segment name.

        For example:
        seglist =
            seglist['sdw1-1'] = apollo1
            seglist['sdw1-2'] = apollo1
            seglist['sdw2-1'] = apollo2
            seglist['sdw2-2'] = apollo2
        uniqhosts =
            uniqhosts['apollo1'] = 'sdw1-1'
            uniqhosts['apollo2'] = 'sdw2-1'

    '''
    seglist = dict()  # segment list
    uniqhosts = dict()  # unique host list

    '''
     Get hostnames using non-verbose mode since verbose output makes parsing difficult with extra lines as show:
        Using delaybeforesend 0.05 and prompt_validation_timeout 1.0
        [Reset ...]
        [INFO] login sdw2
        [sdw2] sdw2
        [INFO] completed successfully
        [Cleanup...]
    '''
    rc, out = gpssh('hostname', False)

    if not rc:
        raise Exception('Encountered error running hostname')

    ''' Sample output:
        [sdw1] sdw1
        [sdw2] sdw2
    '''

    # get unique hostname list
    for line in out.splitlines():
        seg, host = line.translate(str.maketrans('','','[]')).split()
        uniqhosts[host] = seg

    # get list of segments associated with each host (can't use gpssh since it de-dupes hosts)
    for host in hostlist:
        cmd = ['ssh', '-o', 'BatchMode yes', '-o', 'StrictHostKeyChecking no', host, 'hostname']

        proc = None
        try:
            if GV.opt['-v']:
                print('[Info]', strcmd(cmd))
            proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            out = proc.stdout.read(-1)
            rc = proc.wait()
            if rc:
                raise Exception('ssh error with the following command:\n%s with output: %s' % (cmd, out))
            seglist[host] = out.strip()
        except KeyboardInterrupt:
            killProc(proc)
            raise
        except Exception as err:
            print(err)
            sys.exit(-1)

    return seglist, uniqhosts


def runNetPerfTestMatrix():
    '''
        Runs full matrix network test where each host connects with
            every link on every other host.
        Returns performance results and a dictionary mapping segment_name
            to hostname
    '''
    (netperf, hostlist, netserver_port) = setupNetPerfTest()
    if not netperf:
        return None, None

    # dict() of seglist[segname] = hostname, uniqhosts[hostname] = 1 segment name
    seglist, uniqhosts = get_host_map(hostlist)

    # spawn netperf between all hosts
    result = []
    procList = []
    for host in uniqhosts:
        for segment in seglist:
            if seglist[segment] == host:
                pass
            else:
                p = spawnNetperfTestBetween(host, segment, netperf, netserver_port, GV.opt['--duration'])
                if p[0]:
                    procList.append(p)

    for (proc, srcHost, dstHost) in procList:
        res = reapNetperfTest(proc, srcHost, dstHost)
        if res and len(res) >= 6:
            result.append(res)

    return result, seglist


def printMatrixResult(result, seglist):
    print('Full matrix netperf bandwidth test')

    # sum up Rx/Tx rate for each host
    netTx = dict()
    netRx = dict()
    for h in result:
        if h[0] in netTx:
            netTx[h[0]] += float(h[6])
        else:
            netTx[h[0]] = float(h[6])

        # netRx requires that we lookup the hostname for a given segment name
        if seglist[h[1]] in netRx:
            netRx[seglist[h[1]]] += float(h[6])
        else:
            netRx[seglist[h[1]]] = float(h[6])

    # print Tx rates
    print("\nPer host transfer rates")
    for host in netTx:
        print('%s Tx rate: %.2f' % (host, netTx[host]))
    print

    # print Rx rates
    print("Per host receive rates")
    for host in netRx:
        print('%s Rx rate: %.2f' % (host, netRx[host]))
    print

    # print per link results in verbose mode
    if GV.opt['-v']:
        for h in result:
            print('%s -> %s = %f' % (h[0], h[1], h[6]))

    sum = min = max = avg = 0
    n = list(map(lambda x: float(x[6]), result))
    sum = reduce(lambda x, y: x + y, n)
    min = reduce(lambda x, y: x > y and y or x, n, 9999999999)
    max = reduce(lambda x, y: x > y and x or y, n)
    avg = float(sum) / len(result)

    copy = n[:]
    copy.sort()
    median = copy[len(copy) // 2]

    print('')
    print('Summary:')
    print('sum = %.2f MB/sec' % sum)
    print('min = %.2f MB/sec' % min)
    print('max = %.2f MB/sec' % max)
    print('avg = %.2f MB/sec' % avg)
    print('median = %.2f MB/sec' % median)
    print('')


def printNetResult(result):
    print('Netperf bisection bandwidth test')
    for h in result:
        print('%s -> %s = %f' % (h[0], h[1], h[6]))

    sum = min = max = avg = 0
    n = list(map(lambda x: float(x[6]), result))
    sum = reduce(lambda x, y: x + y, n)
    min = reduce(lambda x, y: x > y and y or x, n, 9999999999)
    max = reduce(lambda x, y: x > y and x or y, n, 0)
    avg = float(sum) / len(result)

    copy = n[:]
    copy.sort()
    median = copy[len(copy) // 2]

    print('')
    print('Summary:')
    print('sum = %.2f MB/sec' % sum)
    print('min = %.2f MB/sec' % min)
    print('max = %.2f MB/sec' % max)
    print('avg = %.2f MB/sec' % avg)
    print('median = %.2f MB/sec' % median)
    print('')

    limit = max * 0.9
    for r in result:
        if r[6] < limit:
            print('[Warning] connection between %s and %s is no good' % (r[0], r[1]))


def printResult(title, result):
    totTime = 0
    totBytes = 0
    totMBPS = 0
    min = 99999999
    max = 0
    minHost = maxHost = None
    for h in result:
        (mbps, time, bytes) = result[h]
        totTime += time
        totBytes += bytes
        totMBPS += mbps
        if min > mbps:
            min = mbps
            minHost = h
        if max < mbps:
            max = mbps
            maxHost = h

    avgTime = totTime / len(result)

    print('')
    if totTime > 0:
        print(' %s avg time (sec): %3.2f' % (title, avgTime))
    if totBytes > 0:
        print(' %s tot bytes: %d' % (title, totBytes))
    print(' %s tot bandwidth (MB/s): %3.2f' % (title, totMBPS))
    print(' %s min bandwidth (MB/s): %3.2f [%s]' % (title, min, minHost))
    print(' %s max bandwidth (MB/s): %3.2f [%s]' % (title, max, maxHost))
    if GV.opt['-D']:
        print(' -- per host bandwidth --')
        for h in result:
            (mbps, time, bytes) = result[h]
            print('    %s bandwidth (MB/s): %3.2f [%s]' % (title, mbps, h))
    print('')


def main():
    try:
        parseCommandLine()
        runSetup()
        diskWriteResult = diskReadResult = streamResult = netResult = None
        tornDown = False
        try:
            if GV.opt['-r'].find('d') >= 0:
                print('[Warning] Using %d bytes for disk performance test. This might take some time' % (GV.opt['-S'] * len(GV.opt['-d'])))
                multidd = copyExecOver('multidd')
                diskWriteResult = runDiskWriteTest(multidd)
                diskReadResult = runDiskReadTest(multidd)

            if GV.opt['-r'].find('s') >= 0:
                streamResult = runStreamTest()

            if GV.opt['--net'] == 'netperf':
                netResult = runNetPerfTest()
            elif GV.opt['--net'] == 'parallel':
                netResult = runNetPerfTestParallel()
            elif GV.opt['--net'] == 'matrix':
                netResult, seglist = runNetPerfTestMatrix()

            runTeardown()

        finally:
            print('')
            print('====================')
            print('==  RESULT %s' % datetime.datetime.now().isoformat())
            print('====================')

            if diskWriteResult:
                printResult('disk write', diskWriteResult)

            if diskReadResult:
                printResult('disk read', diskReadResult)

            if streamResult:
                printResult('stream', streamResult)

            if netResult and GV.opt['--net'] == 'matrix':
                printMatrixResult(netResult, seglist)
            elif netResult and GV.opt['--net']:
                printNetResult(netResult)

            runTeardown()

    except KeyboardInterrupt:
        print('[Abort] Keyboard Interrupt ...')

if __name__ == '__main__':
     main()
